Merge remote-tracking branch 'origin/feature/validations-classes' into feature/validations-clarion-csv-parts
This commit is contained in:
hreyes
1
.gitignore
vendored
1
.gitignore
vendored
@@ -67,6 +67,7 @@ htmlcov/
|
||||
*.dockerignore
|
||||
postgres-data/
|
||||
backend/uploads/
|
||||
backend/layouts/imports/
|
||||
docker-compose.yml
|
||||
.mypy_cache/
|
||||
|
||||
|
||||
19
README.md
19
README.md
@@ -42,14 +42,24 @@ anexo76/
|
||||
│ ├── .env.example # Variables de entorno de ejemplo
|
||||
│ ├── core/ # Módulos core
|
||||
│ │ ├── config.py # Configuración centralizada
|
||||
│ │ ├── paths.py # Rutas base y layout_path() para importación CSV
|
||||
│ │ ├── database.py # Configuración de BD multi-tenant
|
||||
│ │ ├── security.py # Autenticación y autorización
|
||||
│ │ └── middleware.py # Middlewares personalizados
|
||||
│ ├── layouts/ # Directorio de datos: temp y errors de importación CSV (creado al arranque)
|
||||
│ └── api/
|
||||
│ └── v1/
|
||||
│ ├── router.py # Router principal API v1
|
||||
│ └── modules/ # Módulos de negocio
|
||||
│ ├── auth/ # Autenticación
|
||||
│ └── a76/
|
||||
│ ├── layouts_csv/ # Lógica centralizada de cargas por CSV (routes, tasks, validaciones por proceso)
|
||||
│ │ ├── facturas/
|
||||
│ │ ├── customs_brokers/
|
||||
│ │ ├── clients_and_providers/
|
||||
│ │ └── ... # Una carpeta por proceso de importación
|
||||
│ ├── csv_templates/
|
||||
│ └── ...
|
||||
│ ├── auth/
|
||||
│ ├── tenants/ # Gestión de tenants
|
||||
│ ├── licenses/ # Control de licencias
|
||||
│ └── ...
|
||||
@@ -136,6 +146,11 @@ python -c "from core.database import init_db; init_db()"
|
||||
|
||||
```
|
||||
docker build -t dev.aduanasoft.com/anexo76/backend:latest -f ./backend/Dockerfile ./backend
|
||||
```
|
||||
|
||||
**Importación CSV y workers Celery**: La lógica de cargas por CSV está en `api/v1/modules/a76/layouts_csv/` (una carpeta por proceso). Los workers Celery deben cargar los módulos `api.v1.modules.a76.layouts_csv.<proceso>.tasks`. El directorio de trabajo del worker debe ser la raíz del backend para que `layout_path("imports", "temp")` y `layout_path("imports", "errors")` apunten a los mismos directorios que la API. Si API y worker comparten volumen, los archivos temporales y de errores se escriben en `backend/layouts/imports/temp` y `backend/layouts/imports/errors`.
|
||||
|
||||
```
|
||||
docker build \
|
||||
--build-arg VITE_API_URL=https://anexo76-dev.aduanasoft.com/api/ \
|
||||
--build-arg VITE_KEYCLOAK_URL=https://anexo76-dev.aduanasoft.com/kcauth/ \
|
||||
@@ -165,6 +180,8 @@ pip install -r requirements.txt
|
||||
uvicorn main:app --reload
|
||||
```
|
||||
|
||||
Los directorios `backend/layouts/imports/temp` y `backend/layouts/imports/errors` se crean automáticamente al arrancar el backend para la importación CSV.
|
||||
|
||||
### Frontend
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,430 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"bom_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila BOM según columnas del template. FK opcional a a76.parts."""
|
||||
parent = (row.get("NUMPARTE_PADRE") or "").strip()
|
||||
if not parent:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"}
|
||||
if len(parent) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
component = (row.get("NUMPARTE_COMPONENTE") or "").strip()
|
||||
if not component:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"}
|
||||
if len(component) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
qty = row.get("CANTIDAD")
|
||||
if qty is None or qty == "":
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"}
|
||||
try:
|
||||
val = Decimal(str(qty))
|
||||
if val < 0:
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 10:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
version_bom = row.get("VERSION_BOM")
|
||||
if not _optional_number(version_bom):
|
||||
return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"}
|
||||
|
||||
version_bill = row.get("VERSION_BILL")
|
||||
if not _optional_number(version_bill):
|
||||
return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"}
|
||||
|
||||
# Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas)
|
||||
if valid_part_numbers is not None and len(valid_part_numbers) > 0:
|
||||
if parent not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"}
|
||||
if component not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"BOMs import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts for FK validation: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"BOMs import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"BOMs import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
valid_count = 0
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
# Cuando exista la tabla de destino, aquí se hará insert/update.
|
||||
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"BOMs import cleanup failed: {cleanup_err}")
|
||||
|
||||
return response
|
||||
@@ -5,7 +5,7 @@ Mismo patrón que parts y classes: upload → scan → status → commit.
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.boms.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@@ -1,528 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
if not class_code:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
|
||||
if len(class_code) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
material_key = (row.get("CLAVEMAT") or "").strip()
|
||||
if material_key:
|
||||
if len(material_key) > 10:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
|
||||
if valid_material_keys is not None and material_key not in valid_material_keys:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom:
|
||||
if len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
if valid_uom_codes is not None and uom not in valid_uom_codes:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 20:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
sub_key = (row.get("CLAVESUB") or "").strip()
|
||||
if sub_key and len(sub_key) > 5:
|
||||
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
|
||||
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if iva_exempt and len(iva_exempt) > 4:
|
||||
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
rev_fisica = row.get("REVFISICA")
|
||||
if rev_fisica is not None and rev_fisica != "":
|
||||
try:
|
||||
v = int(rev_fisica)
|
||||
if v < -32768 or v > 32767:
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Classes import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Classes import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code: Dict[str, Class] = {}
|
||||
for c in (
|
||||
session.query(Class)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_code[c.class_code] = c
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
if existing:
|
||||
existing.description_es = desc_es
|
||||
existing.description_en = desc_en
|
||||
existing.material_key = material_key
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.sub_key = sub_key
|
||||
existing.physical_review = physical_review
|
||||
existing.iva_exempt_fraction = iva_exempt_fraction
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
class_code=class_code,
|
||||
description_es=desc_es,
|
||||
description_en=desc_en,
|
||||
material_key=material_key,
|
||||
unit_of_measure=unit_of_measure,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
sub_key=sub_key,
|
||||
physical_review=physical_review,
|
||||
iva_exempt_fraction=iva_exempt_fraction,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Classes import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -1,45 +0,0 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -12,7 +12,7 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t
|
||||
|
||||
from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse
|
||||
from .service import ClassService
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.classes.routes import router as imports_router
|
||||
|
||||
# Create a new router for custom endpoints
|
||||
router = APIRouter()
|
||||
|
||||
@@ -1,500 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientOrProviderEnum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active(val: Optional[str]) -> bool:
|
||||
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
|
||||
rfc = (row.get("RFC") or "").strip()
|
||||
if not rfc:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
|
||||
if len(rfc) > 30:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
|
||||
name = (row.get("NOMBRE") or "").strip()
|
||||
if len(name) > 256:
|
||||
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
if short_name and len(short_name) > 10:
|
||||
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
curp = (row.get("CURP") or "").strip()
|
||||
if curp and len(curp) > 19:
|
||||
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CP import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
|
||||
Upsert por (tenant_id, company_id, rfc).
|
||||
"""
|
||||
logger.info(f"CP import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CP import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
rfc = _str_or_none(row_norm.get("RFC"), 30)
|
||||
if not rfc:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
|
||||
if client_or_provider is None:
|
||||
client_or_provider = ClientOrProviderEnum.BOTH
|
||||
|
||||
is_active = _parse_active(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
|
||||
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
|
||||
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
|
||||
existing.client_or_provider = client_or_provider
|
||||
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
rfc=rfc,
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 256),
|
||||
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
|
||||
curp=_str_or_none(row_norm.get("CURP"), 19),
|
||||
client_or_provider=client_or_provider,
|
||||
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
|
||||
# Opcional: crear dirección si hay email/teléfono/dirección
|
||||
email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
|
||||
if email or phone or address_str:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_str,
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
phone=phone,
|
||||
email=email,
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
|
||||
)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CP import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CP import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -20,7 +20,7 @@ from .dto import (
|
||||
)
|
||||
from .service import ClientProviderService
|
||||
from .models import ClientProvider
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.clients_and_providers.routes import router as imports_router
|
||||
|
||||
# Create main router to add custom endpoints
|
||||
router = APIRouter(prefix="/clients-providers")
|
||||
|
||||
@@ -7,35 +7,38 @@ import io
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# Importar configs de cada módulo
|
||||
from api.v1.modules.a76.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.facturas.template_config import (
|
||||
TEMPLATE_COLUMNS as IMPORTS_TEMPLATE_COLUMNS,
|
||||
_resolve_template_columns as resolve_imports_template,
|
||||
)
|
||||
from api.v1.modules.a76.parts.imports.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.boms.imports.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.classes.imports.template_config import TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.customs_brokers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.parts.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.layouts_csv.boms.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.layouts_csv.classes.template_config import (
|
||||
TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS,
|
||||
TEMPLATE_DOWNLOAD_HEADERS as CLASSES_TEMPLATE_DOWNLOAD_HEADERS,
|
||||
)
|
||||
from api.v1.modules.a76.layouts_csv.customs_brokers.template_config import (
|
||||
TEMPLATE_COLUMNS as CUSTOMS_BROKERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.clients_and_providers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.clients_and_providers.template_config import (
|
||||
TEMPLATE_COLUMNS as CLIENTS_PROVIDERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.exchange_rate.template_config import (
|
||||
TEMPLATE_COLUMNS as EXCHANGE_RATE_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.us_tariff_fractions.template_config import (
|
||||
TEMPLATE_COLUMNS as US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.pedmientos.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.pedmientos.template_config import (
|
||||
TEMPLATE_COLUMNS as PEDIMENTOS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.vehicles.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.vehicles.template_config import (
|
||||
TEMPLATE_COLUMNS as VEHICLES_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.drivers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.drivers.template_config import (
|
||||
TEMPLATE_COLUMNS as DRIVERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.trailers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.trailers.template_config import (
|
||||
TEMPLATE_COLUMNS as TRAILERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
|
||||
@@ -63,8 +66,12 @@ def _build_registry() -> Dict[str, List[str]]:
|
||||
# boms
|
||||
registry["boms"] = _canonicals_from_columns(BOMS_TEMPLATE_COLUMNS.get("boms"))
|
||||
|
||||
# material_classes
|
||||
registry["material_classes"] = _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes"))
|
||||
# material_classes: cabeceras de descarga según plantilla usuario (CLAVE, CLASE, DESCRIPCION, etc.)
|
||||
registry["material_classes"] = (
|
||||
CLASSES_TEMPLATE_DOWNLOAD_HEADERS
|
||||
if CLASSES_TEMPLATE_DOWNLOAD_HEADERS
|
||||
else _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes"))
|
||||
)
|
||||
|
||||
# customs_brokers
|
||||
registry["customs_brokers"] = _canonicals_from_columns(CUSTOMS_BROKERS_TEMPLATE_COLUMNS.get("customs_brokers"))
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# CSV import flow for Agentes Aduanales (Customs Brokers).
|
||||
# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit.
|
||||
@@ -1,447 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con a76.imports)
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > 5:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CB import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Guardar números de línea con error en Redis para insert_valid_rows
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
|
||||
"""
|
||||
logger.info(f"CB import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CB import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
clave = (row_norm.get("CLAVE") or "").strip()[:5]
|
||||
if not clave:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
|
||||
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
|
||||
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
|
||||
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
|
||||
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
|
||||
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
|
||||
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
|
||||
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
|
||||
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
existing.license = lic[:4] if lic and lic.isdigit() else None
|
||||
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
|
||||
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
license_val = lic[:4] if lic and lic.isdigit() else None
|
||||
new_broker = CustomsBroker(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
broker_key=clave,
|
||||
type=_str_or_none(row_norm.get("TIPO"), 9),
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 80),
|
||||
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
|
||||
fax=_str_or_none(row_norm.get("FAX"), 30),
|
||||
email=_str_or_none(row_norm.get("EMAIL"), 100),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
tax_id=_str_or_none(row_norm.get("RFC"), 30),
|
||||
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
license=license_val,
|
||||
company=_str_or_none(row_norm.get("EMPRESA"), 200),
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
|
||||
)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CB import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Limpieza
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CB import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -7,7 +7,7 @@ from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from . import dto, services
|
||||
from .imports.routes import router as imports_router
|
||||
from ..layouts_csv.customs_brokers.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@@ -1,465 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if _parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
|
||||
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
|
||||
if local_raw and len(local_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
|
||||
if foreign_raw and len(foreign_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"ER import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
|
||||
"""
|
||||
logger.info(f"ER import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"ER import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
parsed_date = _parse_date(row_norm.get("FECHA"))
|
||||
if not parsed_date:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
|
||||
continue
|
||||
|
||||
try:
|
||||
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
|
||||
value_decimal = Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
|
||||
continue
|
||||
|
||||
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
|
||||
if local_currency:
|
||||
local_currency = local_currency.upper()
|
||||
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
|
||||
if foreign_currency:
|
||||
foreign_currency = foreign_currency.upper()
|
||||
|
||||
key_date = parsed_date.date()
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
|
||||
if existing:
|
||||
existing.value = value_decimal
|
||||
existing.local_currency = local_currency or None
|
||||
existing.foreign_currency = foreign_currency or None
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
date=parsed_date,
|
||||
value=value_decimal,
|
||||
local_currency=local_currency,
|
||||
foreign_currency=foreign_currency,
|
||||
)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"ER import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"ER import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -35,7 +35,7 @@ from fastapi import APIRouter
|
||||
custom_router = APIRouter(prefix="/exchange-rate", tags=[])
|
||||
|
||||
# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.exchange_rate.routes import router as imports_router
|
||||
custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"])
|
||||
|
||||
@custom_router.get("/test-ping")
|
||||
|
||||
@@ -1,474 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
|
||||
FA_IMPORT_META_PREFIX = "fa_import_meta:"
|
||||
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
|
||||
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:16] if len(s) > 16 else s
|
||||
|
||||
|
||||
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
|
||||
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
code_norm = _normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
if len(code_norm) > 16:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
|
||||
|
||||
prefix_raw = (row.get("PREFIJO") or "").strip()
|
||||
if prefix_raw and len(prefix_raw) > 10:
|
||||
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
|
||||
if um_raw and len(um_raw) > 10:
|
||||
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
|
||||
if tipo_raw and len(tipo_raw) > 10:
|
||||
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
adv_pct = row.get("ADVALOREM_PCT")
|
||||
if adv_pct is not None and str(adv_pct).strip():
|
||||
try:
|
||||
v = float(str(adv_pct).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
|
||||
|
||||
adv_dlls = row.get("ADVALOREM_DLLS")
|
||||
if adv_dlls is not None and str(adv_dlls).strip():
|
||||
try:
|
||||
v = float(str(adv_dlls).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"FA import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(str(val).strip().replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
|
||||
"""
|
||||
logger.info(f"FA import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"FA import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
|
||||
continue
|
||||
|
||||
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
|
||||
description = _str_or_none(row_norm.get("DESCRIPCION"))
|
||||
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
|
||||
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
|
||||
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.prefix = prefix
|
||||
existing.type_code = type_code
|
||||
existing.ad_valorem = ad_valorem
|
||||
existing.fixed_cost = fixed_cost
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.description = description
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_row = USTariffFraction(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
code=code,
|
||||
prefix=prefix,
|
||||
type_code=type_code,
|
||||
ad_valorem=ad_valorem,
|
||||
fixed_cost=fixed_cost,
|
||||
unit_of_measure=unit_of_measure,
|
||||
description=description,
|
||||
)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"FA import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path_clean = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path_clean):
|
||||
os.remove(meta_path_clean)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"FA import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -31,7 +31,7 @@ crud_router = TenantCRUDRoutes(
|
||||
)
|
||||
|
||||
# Master router with prefix so all routes live under /us-tariff-fractions
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.us_tariff_fractions.routes import router as imports_router
|
||||
main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"])
|
||||
main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"])
|
||||
main_router.include_router(crud_router.router)
|
||||
|
||||
1
backend/api/v1/modules/a76/layouts_csv/__init__.py
Normal file
1
backend/api/v1/modules/a76/layouts_csv/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# Lógica centralizada de cargas por CSV (routes, tasks, validaciones, template_config por proceso)
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers, fk_loader for boms CSV import
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (BOMs).
|
||||
"""
|
||||
import re
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_decimal_required_min(
|
||||
row: Dict[str, Any], col: str, line_num: int, min_val: Decimal
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
try:
|
||||
v = Decimal(str(val))
|
||||
if v < min_val:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def check_optional_number(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
if not _optional_number(row.get(col)):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
allowed: Optional[set],
|
||||
msg: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or allowed is None or len(allowed) == 0:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación de import CSV de BOMs.
|
||||
"""
|
||||
from typing import Set
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_boms_fk_sets(tenant_id: int, company_id: int) -> Set[str]:
|
||||
"""Carga valid_part_numbers (Part.part_number por tenant/company)."""
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("BOMs import: could not load parts: %s", e)
|
||||
return valid_part_numbers
|
||||
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para BOM (para cuando exista tabla BOM).
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).replace(",", "."))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_bom_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_part_numbers: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para BOM.
|
||||
Para uso futuro cuando exista tabla BOM.
|
||||
"""
|
||||
parent = _str_or_none(row_norm.get("NUMPARTE_PADRE"), 70)
|
||||
component = _str_or_none(row_norm.get("NUMPARTE_COMPONENTE"), 70)
|
||||
quantity = _decimal_or_none(row_norm.get("CANTIDAD"))
|
||||
uom = _str_or_none(row_norm.get("UNIMED"), 10)
|
||||
version_bom = _decimal_or_none(row_norm.get("VERSION_BOM"))
|
||||
version_bill = _decimal_or_none(row_norm.get("VERSION_BILL"))
|
||||
return {
|
||||
"parent_part_number": parent,
|
||||
"component_part_number": component,
|
||||
"quantity": quantity,
|
||||
"uom": uom,
|
||||
"version_bom": version_bom,
|
||||
"version_bill": version_bill,
|
||||
}
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -78,7 +79,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
168
backend/api/v1/modules/a76/layouts_csv/boms/tasks.py
Normal file
168
backend/api/v1/modules/a76/layouts_csv/boms/tasks.py
Normal file
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y cuenta filas válidas.
|
||||
Usa layouts_csv.common y common.fk_loader, validators.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_bom
|
||||
from .common.fk_loader import load_boms_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "bom"
|
||||
|
||||
# Para routes.py
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("BOMs import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("BOMs import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("BOMs import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
valid_count = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, 0, 0, skipped_details,
|
||||
)
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = (
|
||||
f"WIP: {valid_count} filas válidas. "
|
||||
"La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("BOMs import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_bom
|
||||
|
||||
__all__ = ["validate_row_bom"]
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de BOMs.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_max_length,
|
||||
check_decimal_required_min,
|
||||
check_optional_number,
|
||||
check_in_set,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_required_parent(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "NUMPARTE_PADRE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_required_component(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "NUMPARTE_COMPONENTE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_quantity(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_decimal_required_min(row, "CANTIDAD", line_num, Decimal("0"))
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "UNIMED", 10, line_num)
|
||||
|
||||
|
||||
def validate_row_optional_numbers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_optional_number(row, "VERSION_BOM", line_num)
|
||||
if err:
|
||||
return err
|
||||
return check_optional_number(row, "VERSION_BILL", line_num)
|
||||
|
||||
|
||||
def validate_row_fks(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not valid_part_numbers or len(valid_part_numbers) == 0:
|
||||
return None
|
||||
err = check_in_set(
|
||||
row, "NUMPARTE_PADRE", line_num,
|
||||
valid_part_numbers, "Parte padre no existe en catálogo",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return check_in_set(
|
||||
row, "NUMPARTE_COMPONENTE", line_num,
|
||||
valid_part_numbers, "Parte componente no existe en catálogo",
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila BOM.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required_parent,
|
||||
validate_row_required_component,
|
||||
validate_row_quantity,
|
||||
validate_row_lengths,
|
||||
validate_row_optional_numbers,
|
||||
validate_row_fks,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de BOMs.
|
||||
Encadena: requeridos (padre, componente, cantidad) → longitudes → opcionales numéricos → FKs.
|
||||
"""
|
||||
err = validate_row_required_parent(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_required_component(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_quantity(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_optional_numbers(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fks(row, line_num, valid_part_numbers)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,36 @@
|
||||
# Datos de prueba – Import CSV Clases de Materiales
|
||||
|
||||
## Archivo: `test_data_clases.csv`
|
||||
|
||||
### Cómo usar
|
||||
|
||||
1. **Catálogos necesarios**: El CSV usa códigos que deben existir en tu tenant/empresa:
|
||||
- **TIPO DE MATERIAL**: p. ej. `MAT01` (catálogo Tipos de Activo Fijo / MaterialType).
|
||||
- **U.M. COMERCIAL**: p. ej. `KG`, `MTR` (catálogo Unidades de Medida por tenant/company).
|
||||
- **FRACCION ARANCELARIA**: mínimo 8 caracteres y que exista en Fracciones Arancelarias Sifr@ o Histórico (p. ej. `12345678`, `87654321`).
|
||||
- **FRACCION AMERICANA**: que exista en catálogo Fracciones Americanas (p. ej. `1234567890123456`, `6543210987654321`).
|
||||
|
||||
Si no tienes esos códigos, crea al menos uno de cada catálogo o sustituye en el CSV por códigos reales de tu base.
|
||||
|
||||
2. **Filas válidas (para probar carga correcta)**
|
||||
- Líneas 2 y 3: `CLASE01`, `CLASE02` — completas y correctas (ajusta tipos de material, U.M. y fracciones a tus catálogos).
|
||||
|
||||
3. **Filas que disparan errores (para probar mensajes)**
|
||||
- **Línea 4**: Todas las celdas vacías → *"La columna de Clase esta vacio..."* (Col. A vacía).
|
||||
- **Línea 5**: `CLASE04` sin B,D,E,F → *"Existen campos vacios que son obligatorios..."* (validación completa).
|
||||
- **Línea 6**: Clase de más de 8 caracteres (`ABCDEFGHIJ`) → *"La Clase: ... supera la longitud de caracteres"*.
|
||||
- **Línea 7**: Tipo de material `INVALIDO` (no en catálogo) → *"(Col. D) El Tipo de Activo Fijo: INVALIDO no existe..."*.
|
||||
- **Línea 8**: U.M. `UMINE` (no en catálogo) → *"(Col. E) La Unidad de Medida Comercial: UMINE no existe..."*.
|
||||
- **Línea 9**: Fracción mexicana `1234` (< 8 caracteres) → *"La Fraccion 1234 no alcanza la longitud de 8 caracteres"*.
|
||||
- **Línea 10**: Fracción americana `INVALIDO` (no en catálogo) → *"(Col. G) La Fraccion Americana: INVALIDO no existe..."*.
|
||||
- **Línea 11**: Tasa de depreciación `150` (> 100) → *"(Col. H) La Tasa de Depreciación: 150 no puede ser mayor al 100 %"*.
|
||||
- **Línea 12**: Código producto CP `9999` (si existe catálogo CP y no está) → *"(Col. J) La clase: CLASE10 tiene asignado un código de producto inexistente"*.
|
||||
|
||||
### CSV sin cabecera (opcional)
|
||||
|
||||
Si quieres probar detección de “primera fila = datos”, usa un archivo cuya primera línea sea una fila de datos (no "CLAVE CLASE"...). El sistema usará `TEMPLATE_DOWNLOAD_HEADERS` como cabecera y la primera línea como dato.
|
||||
|
||||
### Valores mínimos para una fila válida
|
||||
|
||||
Con **Modo Actualizar** y clase ya existente, solo es obligatoria la Col. A (CLAVE CLASE).
|
||||
Con **Modo Reemplazar** o clase nueva, son obligatorios: A, B, D, E, F (mínimo 8 caracteres y en catálogo), y el resto según reglas (H ≤ 100, J en catálogo CP si aplica).
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers, fk_loader for classes CSV import
|
||||
@@ -0,0 +1,108 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (clases de materiales).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_min_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
min_len: int,
|
||||
line_num: int,
|
||||
msg: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; error si longitud menor que min_len."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) < min_len:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": msg or f"Mínimo {min_len} caracteres",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_int_range(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
min_val: int,
|
||||
max_val: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; devuelve error si no es int o está fuera de rango."""
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
v = int(val)
|
||||
if v < min_val or v > max_val:
|
||||
return {"line": line_num, "col": col, "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número entero"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
allowed: Optional[set],
|
||||
msg: str = "No existe en el catálogo",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor y allowed no es None."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or allowed is None:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
|
||||
|
||||
def check_decimal_max(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
max_val: float,
|
||||
msg: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; error si no es numérico o si es mayor que max_val (Clarion Col H)."""
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
v = float(val)
|
||||
if v > max_val:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": msg or f"El valor no puede ser mayor a {max_val}.",
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser un número."}
|
||||
return None
|
||||
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación/mapeo de import CSV de clases de materiales.
|
||||
Clarion: Tipo Activo Fijo, U.M., Fracción Mex (GFracGenSifra + histórico), Fracción Ame (GFracAme), Código Producto CP (si existe).
|
||||
"""
|
||||
from typing import Set, Tuple
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_classes_fk_sets(
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Tuple[Set[str], Set[str], Set[str], Set[str], Set[str]]:
|
||||
"""
|
||||
Carga todos los conjuntos necesarios para validación CSV de clases (paridad Clarion).
|
||||
Devuelve (valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp).
|
||||
- valid_fraction_mex_8: códigos de 8 caracteres válidos (TariffFraction + HistoricalTariffFraction).
|
||||
- valid_fraction_ame: códigos de fracción americana (USTariffFraction por tenant/company).
|
||||
- valid_product_codes_cp: códigos de producto/servicio CP (vacío si no existe catálogo).
|
||||
"""
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_fraction_mex_8: Set[str] = set()
|
||||
valid_fraction_ame: Set[str] = set()
|
||||
valid_product_codes_cp: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.a76.general_catalogs.fractions.tariff_fractions.models import TariffFraction
|
||||
from api.v1.modules.a76.general_catalogs.fractions.historical_tariff_fractions.models import HistoricalTariffFraction
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
for m in session.query(MaterialType.key).all():
|
||||
if m[0]:
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if u[0]:
|
||||
valid_uom_codes.add(u[0])
|
||||
|
||||
for row in session.query(TariffFraction.code).all():
|
||||
if row[0]:
|
||||
code = row[0].strip()
|
||||
valid_fraction_mex_8.add(code[:8])
|
||||
|
||||
for row in (
|
||||
session.query(HistoricalTariffFraction.historical_fraction)
|
||||
.filter(
|
||||
HistoricalTariffFraction.tenant_id == tenant_id,
|
||||
HistoricalTariffFraction.company_id == company_id,
|
||||
HistoricalTariffFraction.historical_fraction.isnot(None),
|
||||
)
|
||||
.distinct()
|
||||
.all()
|
||||
):
|
||||
if row[0] and row[0].strip():
|
||||
valid_fraction_mex_8.add(row[0].strip()[:8])
|
||||
|
||||
for row in (
|
||||
session.query(USTariffFraction.code)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if row[0]:
|
||||
valid_fraction_ame.add(row[0].strip())
|
||||
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("Classes import: could not load FK sets: %s", e)
|
||||
return (
|
||||
valid_material_keys,
|
||||
valid_uom_codes,
|
||||
valid_fraction_mex_8,
|
||||
valid_fraction_ame,
|
||||
valid_product_codes_cp,
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Class (clases de materiales).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def row_to_class_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_material_keys: Set[str],
|
||||
valid_uom_codes: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para Class.
|
||||
class_code se normaliza a mayúsculas (Clip(Upper) Clarion).
|
||||
"""
|
||||
raw_clase = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
class_code = raw_clase.upper() if raw_clase else None
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
return {
|
||||
"class_code": class_code,
|
||||
"description_es": desc_es,
|
||||
"description_en": desc_en,
|
||||
"material_key": material_key,
|
||||
"unit_of_measure": unit_of_measure,
|
||||
"fraction": fraction,
|
||||
"us_fraction": us_fraction,
|
||||
"sub_key": sub_key,
|
||||
"physical_review": physical_review,
|
||||
"iva_exempt_fraction": iva_exempt_fraction,
|
||||
}
|
||||
|
||||
|
||||
def row_to_class_data_merge_existing(
|
||||
row_norm: Dict[str, Any],
|
||||
existing_data: Dict[str, Any],
|
||||
valid_material_keys: Set[str],
|
||||
valid_uom_codes: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Para modo actualizar (parcial): valores del CSV si no vacíos, sino los de la clase existente (Clarion VALIDA_PARCIAL_CLASE).
|
||||
"""
|
||||
data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes)
|
||||
if not data["class_code"]:
|
||||
return data
|
||||
for key in ("description_es", "description_en", "material_key", "unit_of_measure",
|
||||
"fraction", "us_fraction", "sub_key", "physical_review", "iva_exempt_fraction"):
|
||||
if data.get(key) is None or (isinstance(data[key], str) and not data[key].strip()):
|
||||
data[key] = existing_data.get(key)
|
||||
return data
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -39,6 +40,8 @@ def _get_redis():
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
actualizar: bool = Query(False, description="Modo actualizar (ACT): validación parcial si la clase existe"),
|
||||
siempre_toda: bool = Query(False, description="Forzar siempre validación completa"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
@@ -59,6 +62,8 @@ async def upload_import_file(
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "material_classes",
|
||||
"actualizar": actualizar,
|
||||
"siempre_toda": siempre_toda,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -78,7 +83,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
309
backend/api/v1/modules/a76/layouts_csv/classes/tasks.py
Normal file
309
backend/api/v1/modules/a76/layouts_csv/classes/tasks.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template, detect_headers_or_data
|
||||
from .validators import validate_row_class, validate_row_class_partial
|
||||
from .common.mappers import row_to_class_data, row_to_class_data_merge_existing
|
||||
from .common.fk_loader import load_classes_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cls"
|
||||
|
||||
# Para routes.py
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("Classes import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header)
|
||||
try:
|
||||
total_rows = common_csv.count_csv_rows(file_path, has_header=has_header)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
actualizar = meta.get("actualizar", False)
|
||||
siempre_toda = meta.get("siempre_toda", False)
|
||||
|
||||
valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets(
|
||||
tenant_id, company_id
|
||||
)
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
existing_class_codes = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
for c in session.query(Class.class_code).filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
).all():
|
||||
if c[0]:
|
||||
existing_class_codes.add(c[0].strip().upper())
|
||||
except Exception as e:
|
||||
logger.warning("Classes import: could not load existing class codes: %s", e)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_class(
|
||||
row_norm,
|
||||
i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
actualizar=actualizar,
|
||||
siempre_toda=siempre_toda,
|
||||
existing_class_codes=existing_class_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("Classes import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("Classes import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
actualizar = meta.get("actualizar", False)
|
||||
siempre_toda = meta.get("siempre_toda", False)
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets(
|
||||
tenant_id, company_id
|
||||
)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code = {}
|
||||
for c in session.query(Class).filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
).all():
|
||||
key = (c.class_code or "").strip().upper()
|
||||
if key:
|
||||
existing_by_code[key] = c
|
||||
|
||||
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
class_code_raw = (row_norm.get("CLASE") or "").strip().upper()[:8]
|
||||
use_partial = actualizar and class_code_raw and class_code_raw in existing_by_code and not siempre_toda
|
||||
|
||||
if use_partial:
|
||||
err = validate_row_class_partial(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
else:
|
||||
err = validate_row_class(
|
||||
row_norm,
|
||||
i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
actualizar=actualizar,
|
||||
siempre_toda=siempre_toda,
|
||||
existing_class_codes=set(existing_by_code.keys()),
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
if use_partial:
|
||||
existing = existing_by_code.get(class_code_raw)
|
||||
existing_data = {
|
||||
"description_es": existing.description_es,
|
||||
"description_en": existing.description_en,
|
||||
"material_key": existing.material_key,
|
||||
"unit_of_measure": existing.unit_of_measure,
|
||||
"fraction": existing.fraction,
|
||||
"us_fraction": existing.us_fraction,
|
||||
"sub_key": existing.sub_key,
|
||||
"physical_review": existing.physical_review,
|
||||
"iva_exempt_fraction": existing.iva_exempt_fraction,
|
||||
}
|
||||
data = row_to_class_data_merge_existing(
|
||||
row_norm, existing_data, valid_material_keys, valid_uom_codes,
|
||||
)
|
||||
else:
|
||||
data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes)
|
||||
|
||||
class_code = data.get("class_code")
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
if existing:
|
||||
existing.description_es = data["description_es"]
|
||||
existing.description_en = data["description_en"]
|
||||
existing.material_key = data["material_key"]
|
||||
existing.unit_of_measure = data["unit_of_measure"]
|
||||
existing.fraction = data["fraction"]
|
||||
existing.us_fraction = data["us_fraction"]
|
||||
existing.sub_key = data["sub_key"]
|
||||
existing.physical_review = data["physical_review"]
|
||||
existing.iva_exempt_fraction = data["iva_exempt_fraction"]
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
**data,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("Classes import DB error: %s", db_err)
|
||||
return common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error=str(db_err),
|
||||
)
|
||||
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
response = common_responses.commit_result(
|
||||
"warning", 0, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
message=f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
)
|
||||
elif inserted_count == 0:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
error="No hay registros válidos en el archivo CSV",
|
||||
)
|
||||
else:
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Classes import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
Cabeceras de descarga = Clarion: CLAVE CLASE, DESCRIPCION ESPAÑOL, DESCRIPCION INGLES, etc.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
|
||||
|
||||
# Valores que indican que la primera fila es cabecera (primera columna normalizada)
|
||||
FIRST_COLUMN_HEADER_VALUES = ("CLAVE CLASE", "CLASE")
|
||||
|
||||
|
||||
def detect_headers_or_data(
|
||||
file_path: str,
|
||||
normalize_header_fn,
|
||||
encoding: str = "utf-8-sig",
|
||||
) -> Tuple[Optional[List[str]], bool]:
|
||||
"""
|
||||
Lee la primera línea del CSV y decide si es cabecera o dato.
|
||||
Devuelve (fieldnames, has_header).
|
||||
- Si la primera celda normalizada está en FIRST_COLUMN_HEADER_VALUES -> has_header=True, fieldnames=None
|
||||
(la primera fila es cabecera; iter_csv_rows sin fieldnames).
|
||||
- Si no -> has_header=False, fieldnames=TEMPLATE_DOWNLOAD_HEADERS (la primera fila es dato).
|
||||
"""
|
||||
try:
|
||||
with open(file_path, "r", encoding=encoding) as f:
|
||||
sample = f.read(2048)
|
||||
except Exception:
|
||||
return None, True
|
||||
lines = sample.splitlines()
|
||||
if not lines:
|
||||
return None, True
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = csv.excel
|
||||
reader = csv.reader(io.StringIO(lines[0]), dialect=dialect)
|
||||
first_row = next(reader, None)
|
||||
if not first_row:
|
||||
return None, True
|
||||
first_cell = (first_row[0] or "").strip()
|
||||
first_cell_norm = normalize_header_fn(first_cell)
|
||||
if first_cell_norm in FIRST_COLUMN_HEADER_VALUES:
|
||||
return None, True
|
||||
return list(TEMPLATE_DOWNLOAD_HEADERS), False
|
||||
|
||||
# Cabeceras que se escriben al descargar la plantilla CSV (igual que Clarion EstructuraCatClasesAF)
|
||||
TEMPLATE_DOWNLOAD_HEADERS: List[str] = [
|
||||
"CLAVE CLASE",
|
||||
"DESCRIPCION ESPAÑOL",
|
||||
"DESCRIPCION INGLES",
|
||||
"TIPO DE MATERIAL",
|
||||
"U.M. COMERCIAL",
|
||||
"FRACCION ARANCELARIA",
|
||||
"FRACCION AMERICANA",
|
||||
"TASA DE DEPRECIACION",
|
||||
"REVISION FISICA (1/0)",
|
||||
"CODIGO DE PRODUCTO/SERVICIO CP",
|
||||
]
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLAVE CLASE", "CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION ESPAÑOL", "DESCRIPCION", "DESCRIPCION ES", "DESCRIPCION ESPAÑOL"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION INGLES", "DESCRIPCION EN", "DESCRIPTION", "DESCRIPCION INGLES"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["TIPO DE MATERIAL", "MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["U.M. COMERCIAL", "UNIDAD MEDIDA", "UNIT", "UOM", "TIPO DE MU.M.", "U.M. COMERCIAL"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION ARANCELARIA", "FRACCION MEX", "COM FRACCION"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION AMERICANA", "FRACCION USA", "US FRACTION", "FRACCION"]},
|
||||
{"canonical": "TASADEPRECIA", "aliases": ["TASA DE DEPRECIACION", "TASA DEPRECIACIÓN", "TASA DEPRECIACION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REVISION FISICA (1/0)", "REV FISICA", "PHYSICAL REVIEW", "TASA DE REVISION", "REVISION FISICA"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["CODIGO DE PRODUCTO/SERVICIO CP", "EXENTA IVA", "FRACCION EXENTA IVA", "CODIGO PRODUCTO SERVICIO CP"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
elif key_norm.startswith("CLAVE CLASE"):
|
||||
# CSV leído con delimitador incorrecto: primera columna es "CLAVE CLASE,..." -> usar primer valor como CLASE
|
||||
if "CLASE" not in out and value:
|
||||
first_val = (value.split(",")[0] if "," in str(value) else value).strip()
|
||||
if first_val:
|
||||
out["CLASE"] = first_val
|
||||
return out
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_class, validate_row_class_partial
|
||||
|
||||
__all__ = ["validate_row_class", "validate_row_class_partial"]
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de clases de materiales.
|
||||
Paridad con Clarion: VALIDACIONES_CLASE (Col A len, Col D/E/F/G en catálogo, Col F len≥8, Col H ≤100, Col J en catálogo CP).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_max_length,
|
||||
check_int_range,
|
||||
check_decimal_max,
|
||||
)
|
||||
|
||||
MSG_CLASE_VACIO = "Error: (Col. A) La columna de Clase esta vacio y no se pueden hacer las validaciones."
|
||||
MSG_CLASE_VACIO_SOLUCION = "Capturar en la Columna A una Clase nueva o una ya existente a la cual desee actualizar campos"
|
||||
|
||||
|
||||
def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""CLASE obligatorio; mensaje Clarion si está vacío. Acepta clave 'CLASE' o 'CLAVE CLASE' (por si el CSV no se normalizó)."""
|
||||
val = (row.get("CLASE") or row.get("CLAVE CLASE") or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": "CLASE", "msg": f"{MSG_CLASE_VACIO} {MSG_CLASE_VACIO_SOLUCION}"}
|
||||
if len(val) > 8:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLASE",
|
||||
"msg": f"Error: (Col. A) La Clase: {val} supera la longitud de caracteres. Capturar en la columna A una Clase de 8 caracteres como máximo.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_required_full(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Obligatorios en validación completa: B, D, E, F (Clarion VALIDA_TODA_CLASE)."""
|
||||
cols_labels = [
|
||||
("DESCRIPCIONE", "(Col.B) Descripción Español"),
|
||||
("CLAVEMAT", "(Col.D) Tipo de Material"),
|
||||
("UNIMED", "(Col.E) U.M. Comercial"),
|
||||
("FRACCION", "(Col.F) Fraccion Arancelaria Mex."),
|
||||
]
|
||||
missing = [(col, label) for col, label in cols_labels if not (row.get(col) or "").strip()]
|
||||
if not missing:
|
||||
return None
|
||||
campos = ", ".join(label for _, label in missing)
|
||||
first_col = missing[0][0]
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": first_col,
|
||||
"msg": f"Existen campos vacios que son obligatorios, es la {campos}. Revisar la línea del archivo y capturar los campos con la información correcta.",
|
||||
}
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("DESCRIPCIONE", 500),
|
||||
("DESCRIPCIONI", 500),
|
||||
("CLAVEMAT", 10),
|
||||
("UNIMED", 5),
|
||||
("FRACCION", 20),
|
||||
("FRACCIONAME", 16),
|
||||
("CLAVESUB", 5),
|
||||
("FRACCIONEXENTAIVA", 4),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_fraction_min(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""FRACCION (Col F): si no vacío, mínimo 8 caracteres (Clarion VALIDACIONES_CLASE)."""
|
||||
val = (row.get("FRACCION") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) < 8:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCION",
|
||||
"msg": f"La Fraccion {val} no alcanza la longitud de 8 caracteres. Capturar en la columna F una Fracción de 8 caracteres.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_int_range(row, "REVFISICA", line_num, -32768, 32767)
|
||||
|
||||
|
||||
def validate_row_fks(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]],
|
||||
valid_uom_codes: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val_d = (row.get("CLAVEMAT") or "").strip()
|
||||
if val_d and valid_material_keys is not None and val_d not in valid_material_keys:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLAVEMAT",
|
||||
"msg": f"Error: (Col. D) El Tipo de Activo Fijo: {val_d} no existe en el Catálogo de Tipos de Activo Fijo. Revisar este Tipo de Activo Fijo en el archivo, en caso de ser correcto actualice los catálogos fijos.",
|
||||
}
|
||||
val_e = (row.get("UNIMED") or "").strip()
|
||||
if val_e and valid_uom_codes is not None and val_e not in valid_uom_codes:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "UNIMED",
|
||||
"msg": f"Error: (Col. E) La Unidad de Medida Comercial: {val_e} no existe en el Catálogo de U.M. Revisar esta Unidad de Medida en el archivo, en caso de ser correcta actualice los catálogos.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_fraction_mex_8(value: str) -> str:
|
||||
"""Primeros 8 caracteres si len>=10, sino hasta 8 (Clarion SUB(ColumnaF, 1, 8))."""
|
||||
if not value:
|
||||
return ""
|
||||
v = value.strip()
|
||||
if len(v) >= 10:
|
||||
return v[:8]
|
||||
return v[:8] if len(v) > 8 else v
|
||||
|
||||
|
||||
def validate_row_fraction_mex_catalog(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_fraction_mex_8: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col F: si no vacía, debe existir en catálogo Mex (GFracGenSifra) o Histórico (Clarion)."""
|
||||
val = (row.get("FRACCION") or "").strip()
|
||||
if not val or valid_fraction_mex_8 is None:
|
||||
return None
|
||||
code_8 = _normalize_fraction_mex_8(val)
|
||||
if not code_8:
|
||||
return None
|
||||
if code_8 in valid_fraction_mex_8:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCION",
|
||||
"msg": (
|
||||
f"Error: (Col. F) La Fraccion Mexicana: {val} no existe en el Catálogo de Fracciones Arancelarias Sifr@ ni en el Historico. "
|
||||
"Revisar esta Fracción Arancelaria en el archivo, en caso de ser correcta Actualizar las Fracciones Arancelarias."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def validate_row_fraction_ame_catalog(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_fraction_ame: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col G: si no vacía, debe existir en catálogo Fracciones Americanas (Clarion GFracAme)."""
|
||||
val = (row.get("FRACCIONAME") or "").strip()
|
||||
if not val or valid_fraction_ame is None:
|
||||
return None
|
||||
if val in valid_fraction_ame:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCIONAME",
|
||||
"msg": (
|
||||
f"Error: (Col. G) La Fraccion Americana: {val} no existe en el Catálogo de Fracciones Americanas. "
|
||||
"Dar de alta la Fracción Americana en el Catálogo de Fracciones Americanas."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def validate_row_tasa_depreciacion(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col H: si viene informada, no puede ser mayor al 100 % (Clarion)."""
|
||||
val = row.get("TASADEPRECIA")
|
||||
if val is None or val == "":
|
||||
return None
|
||||
err = check_decimal_max(row, "TASADEPRECIA", line_num, 100.0, msg=None)
|
||||
if err and "100" in (err.get("msg") or ""):
|
||||
v = (row.get("TASADEPRECIA") or "").strip()
|
||||
err["msg"] = f"Error: (Col. H) La Tasa de Depreciación: {v} no puede ser mayor al 100 %. Ajustar la Tasa de Depreciacion."
|
||||
return err
|
||||
|
||||
|
||||
def validate_row_codigo_producto_cp(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_product_codes_cp: Optional[Set[str]],
|
||||
class_code: str = "",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col J: si no vacía y existe catálogo CP, debe existir en GCodigosProductoCP (Clarion)."""
|
||||
val = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if valid_product_codes_cp is None or len(valid_product_codes_cp) == 0:
|
||||
return None
|
||||
if val in valid_product_codes_cp:
|
||||
return None
|
||||
cl = class_code or (row.get("CLASE") or "").strip()
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCIONEXENTAIVA",
|
||||
"msg": (
|
||||
f"Error: (Col. J) La clase: {cl} tiene asignado un código de producto inexistente. "
|
||||
"Capturar un código de producto correcto."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def validaciones_clase(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]],
|
||||
valid_uom_codes: Optional[Set[str]],
|
||||
valid_fraction_mex_8: Optional[Set[str]] = None,
|
||||
valid_fraction_ame: Optional[Set[str]] = None,
|
||||
valid_product_codes_cp: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Reglas compartidas Clarion (VALIDACIONES_CLASE): longitudes, tipos, FKs, fracciones, tasa, código CP."""
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fraction_min(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_types(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fks(row, line_num, valid_material_keys, valid_uom_codes)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fraction_mex_catalog(row, line_num, valid_fraction_mex_8)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fraction_ame_catalog(row, line_num, valid_fraction_ame)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_tasa_depreciacion(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
err = validate_row_codigo_producto_cp(
|
||||
row, line_num, valid_product_codes_cp, class_code
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila de clase de material.
|
||||
Flujo Clarion: no ACT → siempre VALIDA_TODA_CLASE; ACT y clase existe → VALIDA_PARCIAL_CLASE;
|
||||
ACT y clase no existe → VALIDA_TODA_CLASE (validación completa) y si pasa se crea en el insert (ADD).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required,
|
||||
validate_row_required_full,
|
||||
validaciones_clase,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
actualizar: bool = False,
|
||||
siempre_toda: bool = False,
|
||||
existing_class_codes: Optional[Set[str]] = None,
|
||||
valid_fraction_mex_8: Optional[Set[str]] = None,
|
||||
valid_fraction_ame: Optional[Set[str]] = None,
|
||||
valid_product_codes_cp: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Igual que Clarion:
|
||||
- No ACT (actualizar=False): siempre TODA (B,D,E,F obligatorios + validaciones_clase).
|
||||
- ACT y clase existe: PARCIAL (solo CLASE + validaciones_clase); en insert se actualiza (PUT).
|
||||
- ACT y clase no existe: TODA (validación completa); si pasa, en insert se crea (ADD).
|
||||
siempre_toda fuerza TODA en todos los casos.
|
||||
"""
|
||||
err = validate_row_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
class_code = (row.get("CLASE") or "").strip().upper()[:8]
|
||||
use_full = siempre_toda or not actualizar
|
||||
if actualizar and existing_class_codes is not None and class_code in existing_class_codes:
|
||||
use_full = False
|
||||
|
||||
if use_full:
|
||||
err = validate_row_required_full(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
# En Actualizar, si la clase no existe se valida completa y si pasa se crea en insert (como Clarion ADD).
|
||||
|
||||
return validaciones_clase(
|
||||
row,
|
||||
line_num,
|
||||
valid_material_keys,
|
||||
valid_uom_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_class_partial(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
valid_fraction_mex_8: Optional[Set[str]] = None,
|
||||
valid_fraction_ame: Optional[Set[str]] = None,
|
||||
valid_product_codes_cp: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Validación parcial (modo Act, clase existente): solo CLASE + validaciones_clase."""
|
||||
err = validate_row_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return validaciones_clase(
|
||||
row,
|
||||
line_num,
|
||||
valid_material_keys,
|
||||
valid_uom_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for clients_and_providers)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de clientes y proveedores.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum
|
||||
|
||||
RFC_MAX = 30
|
||||
NAME_MAX = 256
|
||||
SHORT_NAME_MAX = 10
|
||||
CURP_MAX = 19
|
||||
|
||||
|
||||
def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def check_tipo_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if not tipo_raw:
|
||||
return None
|
||||
if parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def parse_active(val: Optional[str]) -> bool:
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para ClientProvider y opcional ClientProviderAddress.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum
|
||||
|
||||
from .common_validators import (
|
||||
parse_client_or_provider,
|
||||
parse_active,
|
||||
)
|
||||
|
||||
MAX_LEN = {
|
||||
"rfc": 30,
|
||||
"name": 256,
|
||||
"short_name": 10,
|
||||
"curp": 19,
|
||||
"responsible": 80,
|
||||
"position": 30,
|
||||
"incoterm": 19,
|
||||
"email": 100,
|
||||
"phone": 30,
|
||||
"address": 100,
|
||||
"postal_code": 15,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"contact": 50,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_client_provider_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
|
||||
"""
|
||||
Mapea fila normalizada a datos para ClientProvider y opcional ClientProviderAddress.
|
||||
Devuelve (cp_data, address_data_or_none). address_data es para crear después del flush (necesita client_id).
|
||||
"""
|
||||
rfc = _str_or_none(row_norm.get("RFC"), MAX_LEN["rfc"])
|
||||
if not rfc:
|
||||
return ({}, None)
|
||||
|
||||
client_or_provider = parse_client_or_provider(row_norm.get("TIPO")) or ClientOrProviderEnum.BOTH
|
||||
|
||||
cp_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"rfc": rfc,
|
||||
"name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]),
|
||||
"short_name": _str_or_none(row_norm.get("SHORT_NAME"), MAX_LEN["short_name"]),
|
||||
"curp": _str_or_none(row_norm.get("CURP"), MAX_LEN["curp"]),
|
||||
"client_or_provider": client_or_provider,
|
||||
"responsible": _str_or_none(row_norm.get("RESPONSABLE"), MAX_LEN["responsible"]),
|
||||
"position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]),
|
||||
"incoterm": _str_or_none(row_norm.get("INCOTERM"), MAX_LEN["incoterm"]),
|
||||
"is_active": parse_active(row_norm.get("ACTIVO")),
|
||||
}
|
||||
|
||||
email = _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"])
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"])
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"])
|
||||
if email or phone or address_str:
|
||||
address_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"streets": address_str,
|
||||
"postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"phone": phone,
|
||||
"email": email,
|
||||
"contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]),
|
||||
}
|
||||
return (cp_data, address_data)
|
||||
return (cp_data, None)
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -81,7 +82,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_client_provider
|
||||
from .common.mappers import row_to_client_provider_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cp"
|
||||
|
||||
# Para routes.py
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("CP import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("CP import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
cp_data, address_data = row_to_client_provider_data(row_norm, tenant_id, company_id)
|
||||
if not cp_data or not cp_data.get("rfc"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
rfc = cp_data["rfc"]
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
for k, v in cp_data.items():
|
||||
if k not in ("tenant_id", "company_id", "rfc"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(**cp_data)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
if address_data:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_data.get("streets"),
|
||||
postal_code=address_data.get("postal_code"),
|
||||
city=address_data.get("city"),
|
||||
state=address_data.get("state"),
|
||||
country=address_data.get("country"),
|
||||
phone=address_data.get("phone"),
|
||||
email=address_data.get("email"),
|
||||
contact=address_data.get("contact"),
|
||||
)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("CP import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("CP import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("CP import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_client_provider
|
||||
|
||||
__all__ = ["validate_row_client_provider"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de clientes y proveedores.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
RFC_MAX,
|
||||
NAME_MAX,
|
||||
SHORT_NAME_MAX,
|
||||
CURP_MAX,
|
||||
check_required_max,
|
||||
check_max_length,
|
||||
check_tipo_client_provider,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_client_provider_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_required_max(row, "RFC", RFC_MAX, line_num)
|
||||
|
||||
|
||||
def validate_row_client_provider_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_max_length(row, "NOMBRE", NAME_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "SHORT_NAME", SHORT_NAME_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "CURP", CURP_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_client_provider_tipo(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_tipo_client_provider(row, line_num)
|
||||
|
||||
|
||||
def validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de clientes y proveedores.
|
||||
RFC requerido (max 30); TIPO opcional pero debe ser Cliente/Proveedor/Ambos; NOMBRE/SHORT_NAME/CURP longitudes.
|
||||
"""
|
||||
err = validate_row_client_provider_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_client_provider_tipo(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_client_provider_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila cliente/proveedor.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import validate_row_client_provider
|
||||
|
||||
__all__ = ["validate_row_client_provider"]
|
||||
@@ -0,0 +1 @@
|
||||
# Shared utilities for layouts_csv imports (storage, normalize, csv, meta, responses)
|
||||
41
backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py
Normal file
41
backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
Lectura de CSV con detección de delimitador (compartida por layouts_csv).
|
||||
Si se pasa fieldnames, no se usa la primera fila como cabecera y se toma como dato (CSV sin cabeceras).
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from typing import Iterator, Tuple, Dict, Any, Optional, List
|
||||
|
||||
|
||||
def iter_csv_rows(
|
||||
file_path: str,
|
||||
fieldnames: Optional[List[str]] = None,
|
||||
) -> Iterator[Tuple[int, Dict[str, Any]]]:
|
||||
"""
|
||||
Abre el CSV, detecta dialecto y devuelve (line_num, row_dict) por cada fila.
|
||||
line_num empieza en 1 (primera fila de datos).
|
||||
Si fieldnames es None: la primera fila del archivo se usa como cabecera (comportamiento por defecto).
|
||||
Si fieldnames es una lista: no se usa cabecera; la primera fila se considera dato y se usan fieldnames como columnas.
|
||||
"""
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
if fieldnames:
|
||||
reader = csv.DictReader(f, fieldnames=fieldnames, dialect=dialect)
|
||||
for i, row in enumerate(reader, start=1):
|
||||
yield i, dict(row)
|
||||
else:
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in enumerate(reader, start=1):
|
||||
yield i, row
|
||||
|
||||
|
||||
def count_csv_rows(file_path: str, has_header: bool = True) -> int:
|
||||
"""Cuenta filas del CSV. Si has_header=True (por defecto), no cuenta la cabecera."""
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_lines = sum(1 for _ in f)
|
||||
return total_lines if not has_header else max(0, total_lines - 1)
|
||||
35
backend/api/v1/modules/a76/layouts_csv/common/meta.py
Normal file
35
backend/api/v1/modules/a76/layouts_csv/common/meta.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Carga y guardado de meta (tenant_id, company_id) para imports CSV.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, Tuple
|
||||
|
||||
|
||||
def load_meta(file_path: str) -> Dict[str, Any]:
|
||||
"""Carga meta desde archivo .meta.json asociado al CSV. Devuelve dict vacío si no existe o falla."""
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if not os.path.exists(meta_path):
|
||||
return {}
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def require_tenant_context(file_path: str) -> Tuple[int, int]:
|
||||
"""
|
||||
Obtiene tenant_id y company_id del meta. Lanza ValueError si faltan.
|
||||
"""
|
||||
meta = load_meta(file_path)
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
raise ValueError("Falta contexto (tenant/company)")
|
||||
return int(tenant_id), int(company_id)
|
||||
|
||||
|
||||
def get_meta_path(file_path: str) -> str:
|
||||
"""Ruta del archivo .meta.json para un CSV."""
|
||||
return file_path.replace(".csv", ".meta.json")
|
||||
16
backend/api/v1/modules/a76/layouts_csv/common/normalize.py
Normal file
16
backend/api/v1/modules/a76/layouts_csv/common/normalize.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""
|
||||
Normalización de cabeceras CSV (compartida por todos los módulos layouts_csv).
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
"""Normaliza nombre de columna: NFKD, mayúsculas, sin acentos, espacios colapsados."""
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
47
backend/api/v1/modules/a76/layouts_csv/common/responses.py
Normal file
47
backend/api/v1/modules/a76/layouts_csv/common/responses.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Helpers para construir respuestas de scan y commit (formato unificado).
|
||||
"""
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
def scan_result(
|
||||
job_id: str,
|
||||
processed_rows: int,
|
||||
error_count: int,
|
||||
errors_detail: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Respuesta de scan_file (waiting_confirmation)."""
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def commit_result(
|
||||
status: str,
|
||||
inserted: int,
|
||||
skipped_invalid: int,
|
||||
skipped_missing_fk: int,
|
||||
skipped_duplicate: int,
|
||||
skipped_details: List[Dict[str, Any]],
|
||||
message: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Respuesta de insert_valid_rows (finished / warning / failed)."""
|
||||
out = {
|
||||
"status": status,
|
||||
"inserted": inserted,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if message:
|
||||
out["message"] = message
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
171
backend/api/v1/modules/a76/layouts_csv/common/storage.py
Normal file
171
backend/api/v1/modules/a76/layouts_csv/common/storage.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""
|
||||
Redis y rutas de archivos para imports CSV (compartido por módulos layouts_csv).
|
||||
Cada módulo usa un job_type (ej. "part", "cls", "bom") para prefijos y nombres de archivo.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, Set, List
|
||||
|
||||
from core.paths import layout_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def upload_dir() -> str:
|
||||
"""Directorio temporal para CSV en el worker."""
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def error_dir() -> str:
|
||||
"""Directorio de archivos JSONL de errores."""
|
||||
return layout_path("imports", "errors")
|
||||
|
||||
|
||||
def storage_keys(job_type: str, job_id: str) -> tuple:
|
||||
"""Prefijos Redis para un job_type y job_id. Devuelve (file_key, meta_key, error_lines_key)."""
|
||||
# Facturas usa prefijo "import_" sin tipo (compatibilidad con rutas existentes)
|
||||
if job_type == "" or job_type == "invoice":
|
||||
prefix = "import_"
|
||||
else:
|
||||
prefix = f"{job_type}_import_"
|
||||
return (
|
||||
f"{prefix}file:{job_id}",
|
||||
f"{prefix}meta:{job_id}",
|
||||
f"{prefix}error_lines:{job_id}",
|
||||
)
|
||||
|
||||
|
||||
def file_path_for_job(job_type: str, job_id: str) -> str:
|
||||
"""Ruta local del archivo CSV para un job."""
|
||||
if job_type == "" or job_type == "invoice":
|
||||
return os.path.join(upload_dir(), f"{job_id}.csv")
|
||||
return os.path.join(upload_dir(), f"{job_type}_{job_id}.csv")
|
||||
|
||||
|
||||
def error_path_for_job(job_type: str, job_id: str) -> str:
|
||||
"""Ruta del archivo JSONL de errores para un job."""
|
||||
os.makedirs(error_dir(), exist_ok=True)
|
||||
if job_type == "" or job_type == "invoice":
|
||||
return os.path.join(error_dir(), f"{job_id}.jsonl")
|
||||
return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl")
|
||||
|
||||
|
||||
def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]:
|
||||
"""
|
||||
Descarga contenido del CSV desde Redis y lo escribe en disco.
|
||||
Devuelve la ruta del archivo o None si no hay datos o falla.
|
||||
"""
|
||||
file_key, _, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(file_key)
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e)
|
||||
return None
|
||||
path = file_path_for_job(job_type, job_id)
|
||||
os.makedirs(upload_dir(), exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(raw)
|
||||
return path
|
||||
|
||||
|
||||
def ensure_meta_from_redis(job_type: str, job_id: str, file_path: str, log_prefix: str = "") -> bool:
|
||||
"""Descarga meta desde Redis y la escribe en .meta.json. Devuelve True si hubo datos."""
|
||||
_, meta_key, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(meta_key)
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode meta from Redis: %s", log_prefix or job_type, e)
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def store_error_lines(job_type: str, job_id: str, line_numbers: List[int]) -> None:
|
||||
"""Guarda la lista de números de línea con error en Redis."""
|
||||
_, _, error_key = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(error_key, json.dumps(line_numbers).encode("utf-8"), ex=IMPORT_REDIS_TTL)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to store error lines in Redis: %s", e)
|
||||
|
||||
|
||||
def get_error_lines(job_type: str, job_id: str, error_path: str) -> Set[int]:
|
||||
"""
|
||||
Obtiene el conjunto de líneas con error: primero desde Redis, si está vacío desde el JSONL.
|
||||
"""
|
||||
_, _, error_key = storage_keys(job_type, job_id)
|
||||
lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(error_key)
|
||||
if raw:
|
||||
lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug("Could not load error lines from Redis: %s", e)
|
||||
if not lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
return lines
|
||||
|
||||
|
||||
def delete_import_from_redis(job_type: str, job_id: str) -> None:
|
||||
"""Borra claves Redis del import (file, meta, error_lines)."""
|
||||
file_key, meta_key, error_key = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(file_key, meta_key, error_key)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to delete import keys from Redis: %s", e)
|
||||
|
||||
|
||||
def cleanup_import_job(
|
||||
job_type: str,
|
||||
job_id: str,
|
||||
file_path: Optional[str] = None,
|
||||
error_path: Optional[str] = None,
|
||||
meta_path: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Elimina archivos locales y claves Redis del job."""
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove file %s: %s", file_path, e)
|
||||
if error_path and os.path.exists(error_path):
|
||||
try:
|
||||
os.remove(error_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove error file %s: %s", error_path, e)
|
||||
if meta_path and os.path.exists(meta_path):
|
||||
try:
|
||||
os.remove(meta_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove meta %s: %s", meta_path, e)
|
||||
delete_import_from_redis(job_type, job_id)
|
||||
@@ -0,0 +1 @@
|
||||
# layouts_csv.customs_brokers
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for customs_brokers)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de agentes aduanales (clave, licencia).
|
||||
"""
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
BROKER_KEY_MAX = 5
|
||||
LICENSE_MAX = 4
|
||||
|
||||
|
||||
def check_required_broker_key(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > BROKER_KEY_MAX:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {BROKER_KEY_MAX} caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_license(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if not licencia:
|
||||
return None
|
||||
if len(licencia) > LICENSE_MAX or not licencia.isdigit():
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
return None
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para CustomsBroker.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
MAX_LEN = {
|
||||
"broker_key": 5,
|
||||
"type": 9,
|
||||
"name": 80,
|
||||
"address": 1500,
|
||||
"postal_code": 15,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"phone": 30,
|
||||
"fax": 30,
|
||||
"email": 100,
|
||||
"country": 3,
|
||||
"tax_id": 30,
|
||||
"personal_id": 20,
|
||||
"position": 30,
|
||||
"license": 4,
|
||||
"company": 200,
|
||||
"contact": 80,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _license_value(row_norm: Dict[str, Any]) -> Optional[str]:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
if not lic or not lic.isdigit():
|
||||
return None
|
||||
return lic[:4]
|
||||
|
||||
|
||||
def row_to_customs_broker_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for CustomsBroker model (create or update)."""
|
||||
clave = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["broker_key"])
|
||||
if not clave:
|
||||
return {}
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"broker_key": clave,
|
||||
"type": _str_or_none(row_norm.get("TIPO"), MAX_LEN["type"]),
|
||||
"name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]),
|
||||
"address": _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]),
|
||||
"postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"phone": _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]),
|
||||
"fax": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]),
|
||||
"email": _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"tax_id": _str_or_none(row_norm.get("RFC"), MAX_LEN["tax_id"]),
|
||||
"personal_id": _str_or_none(row_norm.get("PERSONAL_ID"), MAX_LEN["personal_id"]),
|
||||
"position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]),
|
||||
"license": _license_value(row_norm),
|
||||
"company": _str_or_none(row_norm.get("EMPRESA"), MAX_LEN["company"]),
|
||||
"contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]),
|
||||
}
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -81,7 +82,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
220
backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py
Normal file
220
backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py
Normal file
@@ -0,0 +1,220 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_customs_broker
|
||||
from .common.mappers import row_to_customs_broker_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cb"
|
||||
|
||||
# Para routes.py
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("CB import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("CB import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_customs_broker_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("broker_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
clave = data["broker_key"]
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
for k, v in data.items():
|
||||
if k not in ("tenant_id", "company_id", "broker_key"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
else:
|
||||
new_broker = CustomsBroker(**data)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("CB import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("CB import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("CB import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_customs_broker
|
||||
|
||||
__all__ = ["validate_row_customs_broker"]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de agentes aduanales.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_broker_key,
|
||||
check_optional_license,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de agentes aduanales.
|
||||
CLAVE requerida (max 5, alfanumérica); LICENCIA opcional (max 4 dígitos).
|
||||
"""
|
||||
err = check_required_broker_key(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_license(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila agente aduanal.
|
||||
"""
|
||||
from .common import validate_row_customs_broker
|
||||
|
||||
__all__ = ["validate_row_customs_broker"]
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers for drivers CSV import
|
||||
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (conductores).
|
||||
"""
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
MAX_LEN = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
}
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def check_int_positive(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
v = parse_int(row.get(col))
|
||||
if v is None:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser numerico"}
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser mayor a 0"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_birth_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or not str(val).strip():
|
||||
return None
|
||||
if parse_birth_date(val) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
return None
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Driver (conductores).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
MAX_LEN,
|
||||
parse_int,
|
||||
parse_birth_date,
|
||||
)
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_driver_data(
|
||||
row_norm: Dict[str, Any],
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""Mapea una fila normalizada del CSV a un diccionario para DriverCreateDTO."""
|
||||
transporter_key = _str_or_none(row_norm.get("TRANSPORTISTA"), MAX_LEN["transporter_key"])
|
||||
line = parse_int(row_norm.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
return {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row_norm.get("CLAVE CONDUCTOR"), MAX_LEN["driver_name"]),
|
||||
"license_number": _str_or_none(row_norm.get("LICENCIA"), MAX_LEN["license_number"]),
|
||||
"express_line_id": _str_or_none(row_norm.get("PERMISO LINEA EXPRESS"), MAX_LEN["express_line_id"]),
|
||||
"ace_id": _str_or_none(row_norm.get("IDENTIFICACION ACE"), MAX_LEN["ace_id"]),
|
||||
"birth_date": parse_birth_date(row_norm.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row_norm.get("SEXO"), MAX_LEN["gender"]),
|
||||
"birth_country": _str_or_none(row_norm.get("PAIS NACIMIENTO"), MAX_LEN["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row_norm.get("TRANSPORTA MAT. PELIGROSO?"), MAX_LEN["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row_norm.get("PERMISO MAT. PELIGROSO"), MAX_LEN["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row_norm.get("NOMBRE(S)"), MAX_LEN["first_name"]),
|
||||
"last_name": _str_or_none(row_norm.get("APELLIDO PATERNO"), MAX_LEN["last_name"]),
|
||||
"id_key1": _str_or_none(row_norm.get("FORMA IDENTIFICACION 1"), MAX_LEN["id_key1"]),
|
||||
"id_number1": _str_or_none(row_norm.get("NUM. IDENTIFICACION 1"), MAX_LEN["id_number1"]),
|
||||
"id_state1": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["id_state1"]),
|
||||
"id_country1": _str_or_none(row_norm.get("PAIS"), MAX_LEN["id_country1"]),
|
||||
"id_key2": _str_or_none(row_norm.get("FORMA IDENTIFICACION 2"), MAX_LEN["id_key2"]),
|
||||
"id_number2": _str_or_none(row_norm.get("NUM. IDENTIFICACION 2"), MAX_LEN["id_number2"]),
|
||||
"id_state2": _str_or_none(row_norm.get("ESTADO 2"), MAX_LEN["id_state2"]),
|
||||
"id_country2": _str_or_none(row_norm.get("PAIS 2"), MAX_LEN["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
@@ -15,6 +15,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -81,7 +82,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
344
backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py
Normal file
344
backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py
Normal file
@@ -0,0 +1,344 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Conductores.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe) y clave de estado en Redis.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_driver
|
||||
from .common.mappers import row_to_driver_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "drv"
|
||||
|
||||
# Para routes.py
|
||||
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
|
||||
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
|
||||
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
|
||||
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
|
||||
DRV_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
unique.append(name if count == 1 else f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("Drivers import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("Drivers import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.transportation.drivers.services import DriverService
|
||||
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
driver_key = (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-"
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": driver_key,
|
||||
"invoice": driver_key,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_driver_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("transporter_key") or data.get("line") is None:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
key = f"{data['transporter_key']}:{data['line']}"
|
||||
if key in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
})
|
||||
continue
|
||||
seen_keys_in_file[key] = i
|
||||
|
||||
existing = DriverService.get_driver_by_key_and_line(
|
||||
session, data["transporter_key"], data["line"], str(company_id), tenant_id
|
||||
)
|
||||
try:
|
||||
if existing:
|
||||
update_fields = {
|
||||
k: v for k, v in data.items()
|
||||
if k not in ("transporter_key", "line", "company_id", "tenant_id")
|
||||
}
|
||||
for field, value in update_fields.items():
|
||||
setattr(existing, field, value)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = DriverCreateDTO(**data)
|
||||
DriverService.create_driver(session, create_data)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i, "driver_key": key, "invoice": key, "reason": str(db_err),
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("Drivers import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Drivers import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to delete status key: %s", e)
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros validos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("Drivers import: starting commit for job %s", job_id)
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_driver
|
||||
|
||||
__all__ = ["validate_row_driver"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de conductores.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
MAX_LEN,
|
||||
check_max_length,
|
||||
check_int_positive,
|
||||
check_optional_birth_date,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_driver_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_max_length(
|
||||
row, "TRANSPORTISTA", MAX_LEN["transporter_key"], line_num, required=True
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return check_int_positive(row, "LINEA", line_num)
|
||||
|
||||
|
||||
def validate_row_driver_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("CLAVE CONDUCTOR", MAX_LEN["driver_name"]),
|
||||
("LICENCIA", MAX_LEN["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", MAX_LEN["express_line_id"]),
|
||||
("IDENTIFICACION ACE", MAX_LEN["ace_id"]),
|
||||
("SEXO", MAX_LEN["gender"]),
|
||||
("PAIS NACIMIENTO", MAX_LEN["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", MAX_LEN["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", MAX_LEN["hazardous_material_state"]),
|
||||
("NOMBRE(S)", MAX_LEN["first_name"]),
|
||||
("APELLIDO PATERNO", MAX_LEN["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", MAX_LEN["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", MAX_LEN["id_number1"]),
|
||||
("ESTADO", MAX_LEN["id_state1"]),
|
||||
("PAIS", MAX_LEN["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", MAX_LEN["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", MAX_LEN["id_number2"]),
|
||||
("ESTADO 2", MAX_LEN["id_state2"]),
|
||||
("PAIS 2", MAX_LEN["id_country2"]),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_driver_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_optional_birth_date(row, "FECHA NACIMIENTO", line_num)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila conductor.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import (
|
||||
validate_row_driver_required,
|
||||
validate_row_driver_lengths,
|
||||
validate_row_driver_date,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de conductores.
|
||||
Encadena: requeridos (TRANSPORTISTA, LINEA) → longitudes → fecha opcional.
|
||||
"""
|
||||
err = validate_row_driver_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_driver_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_driver_date(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for exchange_rate)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de tipos de cambio.
|
||||
"""
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
DATE_FORMATS: List[str] = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
CURRENCY_MAX = 7
|
||||
|
||||
|
||||
def parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def parse_decimal_positive(val: Optional[str]) -> Optional[Decimal]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
try:
|
||||
v = float(str(val).strip().replace(",", "."))
|
||||
if v <= 0:
|
||||
return None
|
||||
return Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def check_required_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
return None
|
||||
|
||||
|
||||
def check_required_value_positive(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para ExchangeRate.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
parse_date,
|
||||
parse_decimal_positive,
|
||||
)
|
||||
|
||||
CURRENCY_MAX = 7
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_exchange_rate_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for ExchangeRate model. Returns {} if FECHA or VALOR invalid."""
|
||||
parsed_date = parse_date(row_norm.get("FECHA"))
|
||||
value_decimal = parse_decimal_positive(row_norm.get("VALOR"))
|
||||
if not parsed_date or value_decimal is None:
|
||||
return {}
|
||||
local = _str_or_none(row_norm.get("MONEDA_LOCAL"), CURRENCY_MAX)
|
||||
if local:
|
||||
local = local.upper()
|
||||
foreign = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), CURRENCY_MAX)
|
||||
if foreign:
|
||||
foreign = foreign.upper()
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"date": parsed_date,
|
||||
"value": value_decimal,
|
||||
"local_currency": local,
|
||||
"foreign_currency": foreign,
|
||||
}
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -81,7 +82,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
227
backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py
Normal file
227
backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py
Normal file
@@ -0,0 +1,227 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_exchange_rate
|
||||
from .common.mappers import row_to_exchange_rate_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "er"
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
# Para routes.py
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID)
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("ER import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("ER import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_exchange_rate_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("date"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA o VALOR no válidos"})
|
||||
continue
|
||||
|
||||
key_date = data["date"].date() if hasattr(data["date"], "date") else data["date"]
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
if existing:
|
||||
existing.value = data["value"]
|
||||
existing.local_currency = data.get("local_currency")
|
||||
existing.foreign_currency = data.get("foreign_currency")
|
||||
session.add(existing)
|
||||
else:
|
||||
new_er = ExchangeRate(**data)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("ER import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("ER import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("ER import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_exchange_rate
|
||||
|
||||
__all__ = ["validate_row_exchange_rate"]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de tipos de cambio.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_date,
|
||||
check_required_value_positive,
|
||||
check_optional_max_length,
|
||||
CURRENCY_MAX,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de tipos de cambio.
|
||||
FECHA y VALOR requeridos; MONEDA_LOCAL y MONEDA_EXTRANJERA opcionales (max 7).
|
||||
"""
|
||||
err = check_required_date(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_required_value_positive(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "MONEDA_LOCAL", CURRENCY_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "MONEDA_EXTRANJERA", CURRENCY_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila tipo de cambio.
|
||||
"""
|
||||
from .common import validate_row_exchange_rate
|
||||
|
||||
__all__ = ["validate_row_exchange_rate"]
|
||||
@@ -11,6 +11,7 @@ from typing import Optional, Literal, Dict, Any
|
||||
from core.celery_app import celery_app
|
||||
from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .tasks import (
|
||||
@@ -88,7 +89,7 @@ async def upload_import_file(
|
||||
|
||||
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import base64
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import csv
|
||||
@@ -11,87 +10,37 @@ from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process
|
||||
|
||||
# We'll need schemas for validation
|
||||
# from api.v1.modules.a76.invoices.schemas import InvoiceHeaderCreate
|
||||
# But for Phase 1 we use a lighter check
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem)
|
||||
# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py)
|
||||
JOB_TYPE = ""
|
||||
|
||||
# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="")
|
||||
IMPORT_FILE_KEY_PREFIX = "import_file:"
|
||||
IMPORT_META_KEY_PREFIX = "import_meta:"
|
||||
IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:"
|
||||
IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Redis client using same URL as Celery broker (worker and API can share data)."""
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
"""Directory on the worker for temp CSV and meta (same structure as API, but local to worker)."""
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
"""
|
||||
Load file content from Redis and write to worker's upload dir.
|
||||
Returns local file_path if successful, None otherwise.
|
||||
"""
|
||||
redis_client = _get_redis()
|
||||
key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}"
|
||||
data = redis_client.get(key)
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to decode import file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
"""Usa common storage con job_type vacío (prefijo import_)."""
|
||||
return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import")
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
"""Load meta from Redis and write to worker's meta file. Returns True if meta was found and written."""
|
||||
redis_client = _get_redis()
|
||||
key = f"{IMPORT_META_KEY_PREFIX}{job_id}"
|
||||
data = redis_client.get(key)
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to decode import meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import")
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
"""Remove file, meta and error lines from Redis after commit (cleanup)."""
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
|
||||
f"{IMPORT_META_KEY_PREFIX}{job_id}",
|
||||
f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete import keys from Redis: {e}")
|
||||
common_storage.delete_import_from_redis(JOB_TYPE, job_id)
|
||||
|
||||
class ForeignKeyValidator:
|
||||
def __init__(self, session, tenant_id, company_id):
|
||||
@@ -188,9 +137,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
# 2. Setup Error Log
|
||||
error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl")
|
||||
os.makedirs(os.path.dirname(error_path), exist_ok=True)
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
error_count = 0
|
||||
@@ -211,22 +158,12 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
date_format = "yyyy-mm-dd" # Default to ISO format
|
||||
logger.info(f"No date_format specified in config, using default: {date_format}")
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f_meta:
|
||||
meta = json.load(f_meta) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read meta for job {job_id}: {e}")
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Missing context (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
template_id = meta.get("template_id") or (
|
||||
"imp_temp_header" if model_target == "invoice_header" else "imp_temp_details"
|
||||
)
|
||||
@@ -335,24 +272,11 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store error lines in Redis: {e}")
|
||||
|
||||
# 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV)
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
def validate_row_phase_1(
|
||||
row: Dict[str, Any],
|
||||
@@ -770,13 +694,23 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
# Ensure we have the file on this worker: prefer Redis (so any worker can run commit)
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
upload_dir = _worker_upload_dir()
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
if not os.path.exists(file_path):
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
from api.v1.modules.a76.invoices.models import (
|
||||
InvoiceHeader,
|
||||
@@ -798,66 +732,16 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
|
||||
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
|
||||
from api.v1.modules.a76.items.models import LineItem
|
||||
from api.v1.modules.a76.items.models import LineItem
|
||||
from api.v1.modules.a76.items.line_financials.models import LineFinancial
|
||||
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
|
||||
from api.v1.modules.a76.items.line_customs.models import LineCustom
|
||||
from api.v1.modules.a76.items.line_descriptions.models import LineDescription
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
|
||||
error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl")
|
||||
footer_config = parse_footer_config(meta.get("footer_config"))
|
||||
|
||||
# 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file)
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err['line'])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Load Metadata (Context)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
footer_config = {}
|
||||
meta = {}
|
||||
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get('tenant_id')
|
||||
company_id = meta.get('company_id')
|
||||
operation_type_raw = meta.get('operation_type', 'imp')
|
||||
footer_config = parse_footer_config(meta.get('footer_config'))
|
||||
except: pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Missing context (tenant/company)"}
|
||||
|
||||
# 2. Re-read and Map
|
||||
# Initialize counters outside the session block so they're accessible later
|
||||
headers_to_insert = []
|
||||
details_to_insert = []
|
||||
skipped_invalid = 0
|
||||
skipped_missing_invoice = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_fk_details = []
|
||||
inserted_count = 0
|
||||
response = None # Will be set inside the session block
|
||||
|
||||
date_format = footer_config.get("dateFormat")
|
||||
|
||||
# Validate and set default date_format if not provided
|
||||
if not date_format:
|
||||
date_format = "yyyy-mm-dd" # Default to ISO format
|
||||
@@ -871,6 +755,15 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
|
||||
logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}")
|
||||
|
||||
headers_to_insert = []
|
||||
details_to_insert = []
|
||||
skipped_invalid = 0
|
||||
skipped_missing_invoice = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_fk_details = []
|
||||
inserted_count = 0
|
||||
response = None
|
||||
|
||||
with CoreSessionLocal() as session:
|
||||
invoice_id_cache = {}
|
||||
cleared_invoices = set() # Track invoices where we've already cleared items in this job
|
||||
@@ -1459,11 +1352,12 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
|
||||
# 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# common validators for parts CSV import
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user