diff --git a/.gitignore b/.gitignore index 8628496d..539bbaeb 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ wheels/ backend/.env frontend/.env backend/SCRIPTS/ +.cursor/ # IDEs .vscode/ @@ -68,6 +69,7 @@ htmlcov/ *.dockerignore postgres-data/ backend/uploads/ +backend/layouts/imports/ docker-compose.yml .mypy_cache/ diff --git a/README.md b/README.md index d6d16209..b5b129cf 100644 --- a/README.md +++ b/README.md @@ -42,14 +42,24 @@ anexo76/ │ ├── .env.example # Variables de entorno de ejemplo │ ├── core/ # Módulos core │ │ ├── config.py # Configuración centralizada +│ │ ├── paths.py # Rutas base y layout_path() para importación CSV │ │ ├── database.py # Configuración de BD multi-tenant │ │ ├── security.py # Autenticación y autorización │ │ └── middleware.py # Middlewares personalizados +│ ├── layouts/ # Directorio de datos: temp y errors de importación CSV (creado al arranque) │ └── api/ │ └── v1/ │ ├── router.py # Router principal API v1 │ └── modules/ # Módulos de negocio -│ ├── auth/ # Autenticación +│ └── a76/ +│ ├── layouts_csv/ # Lógica centralizada de cargas por CSV (routes, tasks, validaciones por proceso) +│ │ ├── facturas/ +│ │ ├── customs_brokers/ +│ │ ├── clients_and_providers/ +│ │ └── ... # Una carpeta por proceso de importación +│ ├── csv_templates/ +│ └── ... +│ ├── auth/ │ ├── tenants/ # Gestión de tenants │ ├── licenses/ # Control de licencias │ └── ... @@ -136,6 +146,11 @@ python -c "from core.database import init_db; init_db()" ``` docker build -t dev.aduanasoft.com/anexo76/backend:latest -f ./backend/Dockerfile ./backend +``` + +**Importación CSV y workers Celery**: La lógica de cargas por CSV está en `api/v1/modules/a76/layouts_csv/` (una carpeta por proceso). Los workers Celery deben cargar los módulos `api.v1.modules.a76.layouts_csv..tasks`. El directorio de trabajo del worker debe ser la raíz del backend para que `layout_path("imports", "temp")` y `layout_path("imports", "errors")` apunten a los mismos directorios que la API. Si API y worker comparten volumen, los archivos temporales y de errores se escriben en `backend/layouts/imports/temp` y `backend/layouts/imports/errors`. + +``` docker build \ --build-arg VITE_API_URL=https://anexo76-dev.aduanasoft.com/api/ \ --build-arg VITE_KEYCLOAK_URL=https://anexo76-dev.aduanasoft.com/kcauth/ \ @@ -165,6 +180,8 @@ pip install -r requirements.txt uvicorn main:app --reload ``` +Los directorios `backend/layouts/imports/temp` y `backend/layouts/imports/errors` se crean automáticamente al arrancar el backend para la importación CSV. + ### Frontend ```bash diff --git a/backend/api/v1/modules/a76/boms/imports/tasks.py b/backend/api/v1/modules/a76/boms/imports/tasks.py deleted file mode 100644 index ac72ad0d..00000000 --- a/backend/api/v1/modules/a76/boms/imports/tasks.py +++ /dev/null @@ -1,430 +0,0 @@ -""" -Tareas Celery para importación CSV de BOMs. -Flujo: scan_file (validación) → insert_valid_rows (commit). -Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista. -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from decimal import Decimal, InvalidOperation -from typing import Dict, Any, Optional, List, Set - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -BOM_IMPORT_FILE_PREFIX = "bom_import_file:" -BOM_IMPORT_META_PREFIX = "bom_import_meta:" -BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:" -BOM_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"BOMs import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"bom_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"BOMs import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{BOM_IMPORT_FILE_PREFIX}{job_id}", - f"{BOM_IMPORT_META_PREFIX}{job_id}", - f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"BOMs import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_bom( - row: Dict[str, Any], - line_num: int, - valid_part_numbers: Optional[Set[str]] = None, -) -> Optional[Dict[str, Any]]: - """Valida una fila BOM según columnas del template. FK opcional a a76.parts.""" - parent = (row.get("NUMPARTE_PADRE") or "").strip() - if not parent: - return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"} - if len(parent) > 70: - return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"} - - component = (row.get("NUMPARTE_COMPONENTE") or "").strip() - if not component: - return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"} - if len(component) > 70: - return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"} - - qty = row.get("CANTIDAD") - if qty is None or qty == "": - return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"} - try: - val = Decimal(str(qty)) - if val < 0: - return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"} - except (InvalidOperation, ValueError, TypeError): - return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"} - - uom = (row.get("UNIMED") or "").strip() - if uom and len(uom) > 10: - return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"} - - def _optional_number(val: Any) -> bool: - if val is None: - return True - s = re.sub(r"\s+", "", str(val).strip()) - if not s: - return True - try: - float(s.replace(",", ".")) - return True - except (ValueError, TypeError): - return False - - version_bom = row.get("VERSION_BOM") - if not _optional_number(version_bom): - return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"} - - version_bill = row.get("VERSION_BILL") - if not _optional_number(version_bill): - return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"} - - # Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas) - if valid_part_numbers is not None and len(valid_part_numbers) > 0: - if parent not in valid_part_numbers: - return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"} - if component not in valid_part_numbers: - return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - logger.info(f"BOMs import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"BOMs import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_part_numbers: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.parts.models import Part - for p in ( - session.query(Part.part_number) - .filter( - Part.tenant_id == tenant_id, - Part.company_id == company_id, - ) - .all() - ): - valid_part_numbers.add(p[0]) - except Exception as e: - logger.warning(f"BOMs import: could not load parts for FK validation: {e}") - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"BOMs import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=BOM_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"BOMs import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _decimal_or_none(val: Any) -> Optional[Decimal]: - if val is None or val == "": - return None - try: - return Decimal(str(val)) - except (InvalidOperation, ValueError, TypeError): - return None - - -def _int_or_none(val: Any) -> Optional[int]: - if val is None or val == "": - return None - try: - return int(val) - except (ValueError, TypeError): - return None - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - logger.info(f"BOMs import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"BOMs import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_part_numbers: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.parts.models import Part - for p in ( - session.query(Part.part_number) - .filter( - Part.tenant_id == tenant_id, - Part.company_id == company_id, - ) - .all() - ): - valid_part_numbers.add(p[0]) - except Exception as e: - logger.warning(f"BOMs import: could not load parts: {e}") - - inserted_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - valid_count = 0 - - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue - - valid_count += 1 - # Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas. - # Cuando exista la tabla de destino, aquí se hará insert/update. - - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_duplicate": 0, - "skipped_details": skipped_details, - } - if valid_count > 0 and inserted_count == 0: - response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada." - - except Exception as e: - logger.error(f"BOMs import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - response = { - "status": "failed", - "error": str(e), - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_duplicate": 0, - "skipped_details": skipped_details, - } - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"BOMs import cleanup failed: {cleanup_err}") - - return response diff --git a/backend/api/v1/modules/a76/boms/routes.py b/backend/api/v1/modules/a76/boms/routes.py index ddd64d15..98f5a529 100644 --- a/backend/api/v1/modules/a76/boms/routes.py +++ b/backend/api/v1/modules/a76/boms/routes.py @@ -5,7 +5,7 @@ Mismo patrón que parts y classes: upload → scan → status → commit. from fastapi import APIRouter -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.boms.routes import router as imports_router router = APIRouter() diff --git a/backend/api/v1/modules/a76/classes/imports/tasks.py b/backend/api/v1/modules/a76/classes/imports/tasks.py deleted file mode 100644 index ca754d06..00000000 --- a/backend/api/v1/modules/a76/classes/imports/tasks.py +++ /dev/null @@ -1,528 +0,0 @@ -""" -Tareas Celery para importación CSV de Clases de Materiales. -Flujo: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List, Set - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -CLS_IMPORT_FILE_PREFIX = "cls_import_file:" -CLS_IMPORT_META_PREFIX = "cls_import_meta:" -CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" -CLS_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Classes import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"cls_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Classes import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{CLS_IMPORT_FILE_PREFIX}{job_id}", - f"{CLS_IMPORT_META_PREFIX}{job_id}", - f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Classes import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_class( - row: Dict[str, Any], - line_num: int, - valid_material_keys: Optional[Set[str]] = None, - valid_uom_codes: Optional[Set[str]] = None, -) -> Optional[Dict[str, Any]]: - class_code = (row.get("CLASE") or "").strip() - if not class_code: - return {"line": line_num, "col": "CLASE", "msg": "Requerido"} - if len(class_code) > 8: - return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} - - desc_es = (row.get("DESCRIPCIONE") or "").strip() - if desc_es and len(desc_es) > 500: - return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} - desc_en = (row.get("DESCRIPCIONI") or "").strip() - if desc_en and len(desc_en) > 500: - return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} - - material_key = (row.get("CLAVEMAT") or "").strip() - if material_key: - if len(material_key) > 10: - return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"} - if valid_material_keys is not None and material_key not in valid_material_keys: - return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"} - - uom = (row.get("UNIMED") or "").strip() - if uom: - if len(uom) > 5: - return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} - if valid_uom_codes is not None and uom not in valid_uom_codes: - return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"} - - fraction = (row.get("FRACCION") or "").strip() - if fraction and len(fraction) > 20: - return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"} - us_fraction = (row.get("FRACCIONAME") or "").strip() - if us_fraction and len(us_fraction) > 16: - return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} - sub_key = (row.get("CLAVESUB") or "").strip() - if sub_key and len(sub_key) > 5: - return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"} - iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip() - if iva_exempt and len(iva_exempt) > 4: - return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"} - - rev_fisica = row.get("REVFISICA") - if rev_fisica is not None and rev_fisica != "": - try: - v = int(rev_fisica) - if v < -32768 or v > 32767: - return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"} - except (ValueError, TypeError): - return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - logger.info(f"Classes import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Classes import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_material_keys: Set[str] = set() - valid_uom_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.public.reference_data.material_types.models import MaterialType - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - for m in session.query(MaterialType.key).all(): - valid_material_keys.add(m[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - except Exception as e: - logger.warning(f"Classes import: could not load FK sets: {e}") - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_class( - row_norm, i, - valid_material_keys=valid_material_keys, - valid_uom_codes=valid_uom_codes, - ) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"Classes import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=CLS_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Classes import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _int_or_none(val: Any) -> Optional[int]: - if val is None or val == "": - return None - try: - return int(val) - except (ValueError, TypeError): - return None - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - logger.info(f"Classes import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Classes import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.classes.models import Class - - valid_material_keys: Set[str] = set() - valid_uom_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.public.reference_data.material_types.models import MaterialType - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - for m in session.query(MaterialType.key).all(): - valid_material_keys.add(m[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - except Exception as e: - logger.warning(f"Classes import: could not load FK sets: {e}") - - inserted_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - existing_by_code: Dict[str, Class] = {} - for c in ( - session.query(Class) - .filter( - Class.tenant_id == tenant_id, - Class.company_id == company_id, - ) - .all() - ): - existing_by_code[c.class_code] = c - - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_class( - row_norm, i, - valid_material_keys=valid_material_keys, - valid_uom_codes=valid_uom_codes, - ) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue - - class_code = _str_or_none(row_norm.get("CLASE"), 8) - if not class_code: - skipped_invalid += 1 - continue - - existing = existing_by_code.get(class_code) - desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) - desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) - material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) - if material_key and material_key not in valid_material_keys: - material_key = None - unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) - if unit_of_measure and unit_of_measure not in valid_uom_codes: - unit_of_measure = None - fraction = _str_or_none(row_norm.get("FRACCION"), 20) - us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) - sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) - physical_review = _int_or_none(row_norm.get("REVFISICA")) - iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) - - if existing: - existing.description_es = desc_es - existing.description_en = desc_en - existing.material_key = material_key - existing.unit_of_measure = unit_of_measure - existing.fraction = fraction - existing.us_fraction = us_fraction - existing.sub_key = sub_key - existing.physical_review = physical_review - existing.iva_exempt_fraction = iva_exempt_fraction - session.add(existing) - inserted_count += 1 - else: - new_class = Class( - tenant_id=tenant_id, - company_id=company_id, - class_code=class_code, - description_es=desc_es, - description_en=desc_en, - material_key=material_key, - unit_of_measure=unit_of_measure, - fraction=fraction, - us_fraction=us_fraction, - sub_key=sub_key, - physical_review=physical_review, - iva_exempt_fraction=iva_exempt_fraction, - ) - session.add(new_class) - existing_by_code[class_code] = new_class - inserted_count += 1 - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"Classes import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"Classes import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Classes import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/classes/imports/template_config.py b/backend/api/v1/modules/a76/classes/imports/template_config.py deleted file mode 100644 index 4a389bd8..00000000 --- a/backend/api/v1/modules/a76/classes/imports/template_config.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls). -""" - -from typing import Dict, List, Any, Optional - -TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { - "material_classes": [ - {"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]}, - {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]}, - {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]}, - {"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]}, - {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]}, - {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, - {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, - {"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]}, - {"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]}, - {"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]}, - ], -} - - -def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: - cols = TEMPLATE_COLUMNS.get("material_classes") - if not cols: - return {} - lookup: Dict[str, str] = {} - for item in cols: - canonical = item["canonical"] - lookup[normalize_header_fn(canonical)] = canonical - for alias in item.get("aliases") or []: - lookup[normalize_header_fn(alias)] = canonical - return lookup - - -def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: - lookup = build_normalized_lookup(normalize_header_fn) - if not lookup: - return {normalize_header_fn(k): v for k, v in row.items()} - out: Dict[str, Any] = {} - for csv_header, value in row.items(): - key_norm = normalize_header_fn(csv_header) - if key_norm in lookup: - out[lookup[key_norm]] = value - return out diff --git a/backend/api/v1/modules/a76/classes/routes.py b/backend/api/v1/modules/a76/classes/routes.py index 2166610f..389bbbd5 100644 --- a/backend/api/v1/modules/a76/classes/routes.py +++ b/backend/api/v1/modules/a76/classes/routes.py @@ -12,7 +12,7 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse from .service import ClassService -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.classes.routes import router as imports_router # Create a new router for custom endpoints router = APIRouter() diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py deleted file mode 100644 index 34fea87c..00000000 --- a/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py +++ /dev/null @@ -1,500 +0,0 @@ -""" -Tareas Celery para importación CSV de Clientes y Proveedores. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template -from api.v1.modules.a76.clients_and_providers.models import ( - ClientProvider, - ClientProviderAddress, - ClientOrProviderEnum, -) - -logger = logging.getLogger(__name__) - -# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports) -CP_IMPORT_FILE_PREFIX = "cp_import_file:" -CP_IMPORT_META_PREFIX = "cp_import_meta:" -CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:" -CP_IMPORT_REDIS_TTL = 3600 # 1 hour - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"CP import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"cp_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"CP import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{CP_IMPORT_FILE_PREFIX}{job_id}", - f"{CP_IMPORT_META_PREFIX}{job_id}", - f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"CP import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: - """Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido.""" - if not val or not str(val).strip(): - return None - v = str(val).strip().lower() - if v in ("client", "cliente", "c"): - return ClientOrProviderEnum.CLIENT - if v in ("provider", "proveedor", "p"): - return ClientOrProviderEnum.PROVIDER - if v in ("both", "ambos", "b", "cliente y proveedor"): - return ClientOrProviderEnum.BOTH - return None - - -def _parse_active(val: Optional[str]) -> bool: - """Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True.""" - if not val or not str(val).strip(): - return True - v = str(val).strip().lower() - if v in ("1", "true", "si", "sí", "yes", "s", "x"): - return True - if v in ("0", "false", "no", "n"): - return False - return True - - -def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Cliente/Proveedor. Retorna error dict o None.""" - rfc = (row.get("RFC") or "").strip() - if not rfc: - return {"line": line_num, "col": "RFC", "msg": "Requerido"} - if len(rfc) > 30: - return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"} - - tipo_raw = (row.get("TIPO") or "").strip() - if tipo_raw and _parse_client_or_provider(tipo_raw) is None: - return { - "line": line_num, - "col": "TIPO", - "msg": "Valor no válido. Use Cliente, Proveedor o Ambos.", - } - - name = (row.get("NOMBRE") or "").strip() - if len(name) > 256: - return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"} - - short_name = (row.get("SHORT_NAME") or "").strip() - if short_name and len(short_name) > 10: - return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"} - - curp = (row.get("CURP") or "").strip() - if curp and len(curp) > 19: - return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"CP import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"CP import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_client_provider(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"CP import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=CP_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"CP import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider. - Upsert por (tenant_id, company_id, rfc). - """ - logger.info(f"CP import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"CP import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - inserted_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - # Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key - existing_by_rfc: Dict[str, ClientProvider] = {} - for cp in ( - session.query(ClientProvider) - .filter( - ClientProvider.tenant_id == tenant_id, - ClientProvider.company_id == company_id, - ) - .all() - ): - key = (cp.rfc or "").strip() - existing_by_rfc[key] = cp - - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_client_provider(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - rfc = _str_or_none(row_norm.get("RFC"), 30) - if not rfc: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "RFC requerido"}) - continue - - client_or_provider = _parse_client_or_provider(row_norm.get("TIPO")) - if client_or_provider is None: - client_or_provider = ClientOrProviderEnum.BOTH - - is_active = _parse_active(row_norm.get("ACTIVO")) - - existing = existing_by_rfc.get(rfc) - if existing: - existing.name = _str_or_none(row_norm.get("NOMBRE"), 256) - existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10) - existing.curp = _str_or_none(row_norm.get("CURP"), 19) - existing.client_or_provider = client_or_provider - existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80) - existing.position = _str_or_none(row_norm.get("POSICION"), 30) - existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19) - existing.is_active = is_active - session.add(existing) - inserted_count += 1 - else: - new_cp = ClientProvider( - tenant_id=tenant_id, - company_id=company_id, - rfc=rfc, - name=_str_or_none(row_norm.get("NOMBRE"), 256), - short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10), - curp=_str_or_none(row_norm.get("CURP"), 19), - client_or_provider=client_or_provider, - responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80), - position=_str_or_none(row_norm.get("POSICION"), 30), - incoterm=_str_or_none(row_norm.get("INCOTERM"), 19), - is_active=is_active, - ) - session.add(new_cp) - session.flush() - existing_by_rfc[rfc] = new_cp - inserted_count += 1 - - # Opcional: crear dirección si hay email/teléfono/dirección - email = _str_or_none(row_norm.get("EMAIL"), 100) - phone = _str_or_none(row_norm.get("TELEFONO"), 30) - address_str = _str_or_none(row_norm.get("DIRECCION"), 100) - if email or phone or address_str: - addr = ClientProviderAddress( - client_id=new_cp.id, - tenant_id=tenant_id, - company_id=company_id, - streets=address_str, - postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), - city=_str_or_none(row_norm.get("CIUDAD"), 30), - state=_str_or_none(row_norm.get("ESTADO"), 30), - country=_str_or_none(row_norm.get("PAIS"), 3), - phone=phone, - email=email, - contact=_str_or_none(row_norm.get("CONTACTO"), 50), - ) - session.add(addr) - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"CP import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"CP import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"CP import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py deleted file mode 100644 index f0968d9d..00000000 --- a/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls). -Solo se leen columnas definidas aquí; el resto se ignora. -Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls). -""" - -from typing import Dict, List, Any, Optional - -TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { - "client_providers": [ - {"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]}, - {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]}, - {"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]}, - {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]}, - {"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]}, - {"canonical": "CURP", "aliases": []}, - {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]}, - {"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]}, - {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, - {"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]}, - {"canonical": "ESTADO", "aliases": []}, - {"canonical": "PAIS", "aliases": ["COUNTRY"]}, - {"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]}, - {"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]}, - {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, - {"canonical": "INCOTERM", "aliases": []}, - {"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]}, - ], -} - - -def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: - """normalized_header -> canonical_name para plantilla client_providers.""" - cols = TEMPLATE_COLUMNS.get("client_providers") - if not cols: - return {} - lookup: Dict[str, str] = {} - for item in cols: - canonical = item["canonical"] - lookup[normalize_header_fn(canonical)] = canonical - for alias in item.get("aliases") or []: - lookup[normalize_header_fn(alias)] = canonical - return lookup - - -def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: - """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" - lookup = build_normalized_lookup(normalize_header_fn) - if not lookup: - return {normalize_header_fn(k): v for k, v in row.items()} - out: Dict[str, Any] = {} - for csv_header, value in row.items(): - key_norm = normalize_header_fn(csv_header) - if key_norm in lookup: - out[lookup[key_norm]] = value - return out diff --git a/backend/api/v1/modules/a76/clients_and_providers/routes.py b/backend/api/v1/modules/a76/clients_and_providers/routes.py index c9c67efe..bd3ad62c 100644 --- a/backend/api/v1/modules/a76/clients_and_providers/routes.py +++ b/backend/api/v1/modules/a76/clients_and_providers/routes.py @@ -20,7 +20,7 @@ from .dto import ( ) from .service import ClientProviderService from .models import ClientProvider -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.clients_and_providers.routes import router as imports_router # Create main router to add custom endpoints router = APIRouter(prefix="/clients-providers") diff --git a/backend/api/v1/modules/a76/clients_and_providers/test_client_and_provider.py b/backend/api/v1/modules/a76/clients_and_providers/test_client_and_provider.py deleted file mode 100644 index ba8afa4a..00000000 --- a/backend/api/v1/modules/a76/clients_and_providers/test_client_and_provider.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from .routes import router - -app = FastAPI() -app.include_router(router) -client = TestClient(app) - - -@pytest.mark.usefixtures("client", "access_token") -def test_list_clients_and_providers(client, access_token): - headers = {"Authorization": f"Bearer {access_token}"} - response = client.get("/client_and_provider/", headers=headers) - assert response.status_code == 200 - assert "items" in response.json() - assert "page" in response.json() - assert "page_size" in response.json() - - -@pytest.mark.usefixtures("client", "access_token") -def test_get_client_or_provider_not_found(client, access_token): - headers = {"Authorization": f"Bearer {access_token}"} - response = client.get("/client_and_provider/invalid_id", headers=headers) - assert response.status_code == 404 - - -def test_create_client_or_provider_forbidden(): - response = client.post( - "/client_and_provider/", json={"name": "Test Client/Provider"} - ) - assert response.status_code in (403, 405, 404) - - -def test_update_client_or_provider_forbidden(): - response = client.put( - "/client_and_provider/1", json={"name": "Updated Client/Provider"} - ) - assert response.status_code in (403, 405, 404) diff --git a/backend/api/v1/modules/a76/csv_templates/registry.py b/backend/api/v1/modules/a76/csv_templates/registry.py index e6486244..076363c5 100644 --- a/backend/api/v1/modules/a76/csv_templates/registry.py +++ b/backend/api/v1/modules/a76/csv_templates/registry.py @@ -7,37 +7,46 @@ import io from typing import Dict, List, Optional # Importar configs de cada módulo -from api.v1.modules.a76.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.facturas.template_config import ( TEMPLATE_COLUMNS as IMPORTS_TEMPLATE_COLUMNS, _resolve_template_columns as resolve_imports_template, ) -from api.v1.modules.a76.parts.imports.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS -from api.v1.modules.a76.boms.imports.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS -from api.v1.modules.a76.classes.imports.template_config import TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS -from api.v1.modules.a76.customs_brokers.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.parts.template_config import ( + TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS, + TEMPLATE_DOWNLOAD_HEADERS as PARTS_TEMPLATE_DOWNLOAD_HEADERS, +) +from api.v1.modules.a76.layouts_csv.boms.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS +from api.v1.modules.a76.layouts_csv.classes.template_config import ( + TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS, + TEMPLATE_DOWNLOAD_HEADERS as CLASSES_TEMPLATE_DOWNLOAD_HEADERS, +) +from api.v1.modules.a76.layouts_csv.customs_brokers.template_config import ( TEMPLATE_COLUMNS as CUSTOMS_BROKERS_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.clients_and_providers.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.clients_and_providers.template_config import ( TEMPLATE_COLUMNS as CLIENTS_PROVIDERS_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.general_catalogs.exchange_rate.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.exchange_rate.template_config import ( TEMPLATE_COLUMNS as EXCHANGE_RATE_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.us_tariff_fractions.template_config import ( TEMPLATE_COLUMNS as US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.pedmientos.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.pedmientos.template_config import ( TEMPLATE_COLUMNS as PEDIMENTOS_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.transportation.vehicles.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.vehicles.template_config import ( TEMPLATE_COLUMNS as VEHICLES_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.transportation.drivers.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.drivers.template_config import ( TEMPLATE_COLUMNS as DRIVERS_TEMPLATE_COLUMNS, ) -from api.v1.modules.a76.transportation.trailers.imports.template_config import ( +from api.v1.modules.a76.layouts_csv.trailers.template_config import ( TEMPLATE_COLUMNS as TRAILERS_TEMPLATE_COLUMNS, ) +from api.v1.modules.a76.layouts_csv.transportistas.template_config import ( + TEMPLATE_COLUMNS as TRANSPORTERS_TEMPLATE_COLUMNS, +) def _canonicals_from_columns(cols: Optional[List[Dict]]) -> List[str]: @@ -56,15 +65,26 @@ def _build_registry() -> Dict[str, List[str]]: registry[tid] = _canonicals_from_columns(cols) # part_numbers (parts); "items" usa la misma plantilla - part_cols = PARTS_TEMPLATE_COLUMNS.get("part_numbers") - registry["part_numbers"] = _canonicals_from_columns(part_cols) - registry["items"] = _canonicals_from_columns(part_cols) + registry["part_numbers"] = ( + PARTS_TEMPLATE_DOWNLOAD_HEADERS + if PARTS_TEMPLATE_DOWNLOAD_HEADERS + else _canonicals_from_columns(PARTS_TEMPLATE_COLUMNS.get("part_numbers")) + ) + registry["items"] = ( + PARTS_TEMPLATE_DOWNLOAD_HEADERS + if PARTS_TEMPLATE_DOWNLOAD_HEADERS + else _canonicals_from_columns(PARTS_TEMPLATE_COLUMNS.get("part_numbers")) + ) # boms registry["boms"] = _canonicals_from_columns(BOMS_TEMPLATE_COLUMNS.get("boms")) - # material_classes - registry["material_classes"] = _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes")) + # material_classes: cabeceras de descarga según plantilla usuario (CLAVE, CLASE, DESCRIPCION, etc.) + registry["material_classes"] = ( + CLASSES_TEMPLATE_DOWNLOAD_HEADERS + if CLASSES_TEMPLATE_DOWNLOAD_HEADERS + else _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes")) + ) # customs_brokers registry["customs_brokers"] = _canonicals_from_columns(CUSTOMS_BROKERS_TEMPLATE_COLUMNS.get("customs_brokers")) @@ -92,6 +112,9 @@ def _build_registry() -> Dict[str, List[str]]: # trailers registry["trailers"] = _canonicals_from_columns(TRAILERS_TEMPLATE_COLUMNS.get("trailers")) + # transporters (transportistas) + registry["transporters"] = _canonicals_from_columns(TRANSPORTERS_TEMPLATE_COLUMNS.get("transporters")) + return registry @@ -111,6 +134,7 @@ TEMPLATE_FILENAMES: Dict[str, str] = { "transports": "EstructuraCatTransportes.csv", "drivers": "EstructuraCatConductor.csv", "trailers": "EstructuraCatTrailers.csv", + "transporters": "EstructuraCatTransportistas.csv", "imp_temp_header": "EstructuraEncFacImpoTemp.csv", "imp_temp_details": "EstructuraParFacImpoTempAF.csv", "imp_def_header": "EstructuraEncFacImpoDef.csv", diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py deleted file mode 100644 index c1c90d12..00000000 --- a/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# CSV import flow for Agentes Aduanales (Customs Brokers). -# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py deleted file mode 100644 index ecaed98f..00000000 --- a/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py +++ /dev/null @@ -1,447 +0,0 @@ -""" -Tareas Celery para importación CSV de Agentes Aduanales. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -# Redis keys (prefijo propio para no colisionar con a76.imports) -CB_IMPORT_FILE_PREFIX = "cb_import_file:" -CB_IMPORT_META_PREFIX = "cb_import_meta:" -CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:" -CB_IMPORT_REDIS_TTL = 3600 # 1 hour - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"CB import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"cb_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"CB import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{CB_IMPORT_FILE_PREFIX}{job_id}", - f"{CB_IMPORT_META_PREFIX}{job_id}", - f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"CB import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Agente Aduanal. Retorna error dict o None.""" - clave = (row.get("CLAVE") or "").strip() - if not clave: - return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} - if len(clave) > 5: - return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"} - if not re.match(r"^[a-zA-Z0-9]+$", clave): - return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"} - - licencia = (row.get("LICENCIA") or "").strip() - if licencia and (len(licencia) > 4 or not licencia.isdigit()): - return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"CB import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"CB import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_customs_broker(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"CB import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - # Guardar números de línea con error en Redis para insert_valid_rows - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=CB_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"CB import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker. - """ - logger.info(f"CB import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"CB import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.customs_brokers.models import CustomsBroker - - inserted_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - existing_by_key: Dict[str, CustomsBroker] = {} - for b in ( - session.query(CustomsBroker) - .filter( - CustomsBroker.tenant_id == tenant_id, - CustomsBroker.company_id == company_id, - ) - .all() - ): - existing_by_key[b.broker_key] = b - - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_customs_broker(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - clave = (row_norm.get("CLAVE") or "").strip()[:5] - if not clave: - skipped_invalid += 1 - continue - - existing = existing_by_key.get(clave) - if existing: - existing.type = _str_or_none(row_norm.get("TIPO"), 9) - existing.name = _str_or_none(row_norm.get("NOMBRE"), 80) - existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500) - existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15) - existing.city = _str_or_none(row_norm.get("CIUDAD"), 30) - existing.state = _str_or_none(row_norm.get("ESTADO"), 30) - existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30) - existing.fax = _str_or_none(row_norm.get("FAX"), 30) - existing.email = _str_or_none(row_norm.get("EMAIL"), 100) - existing.country = _str_or_none(row_norm.get("PAIS"), 3) - existing.tax_id = _str_or_none(row_norm.get("RFC"), 30) - existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20) - existing.position = _str_or_none(row_norm.get("POSICION"), 30) - lic = (row_norm.get("LICENCIA") or "").strip() - existing.license = lic[:4] if lic and lic.isdigit() else None - existing.company = _str_or_none(row_norm.get("EMPRESA"), 200) - existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80) - session.add(existing) - inserted_count += 1 - else: - lic = (row_norm.get("LICENCIA") or "").strip() - license_val = lic[:4] if lic and lic.isdigit() else None - new_broker = CustomsBroker( - tenant_id=tenant_id, - company_id=company_id, - broker_key=clave, - type=_str_or_none(row_norm.get("TIPO"), 9), - name=_str_or_none(row_norm.get("NOMBRE"), 80), - address=_str_or_none(row_norm.get("DIRECCION"), 1500), - postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), - city=_str_or_none(row_norm.get("CIUDAD"), 30), - state=_str_or_none(row_norm.get("ESTADO"), 30), - phone=_str_or_none(row_norm.get("TELEFONO"), 30), - fax=_str_or_none(row_norm.get("FAX"), 30), - email=_str_or_none(row_norm.get("EMAIL"), 100), - country=_str_or_none(row_norm.get("PAIS"), 3), - tax_id=_str_or_none(row_norm.get("RFC"), 30), - personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20), - position=_str_or_none(row_norm.get("POSICION"), 30), - license=license_val, - company=_str_or_none(row_norm.get("EMPRESA"), 200), - contact=_str_or_none(row_norm.get("CONTACTO"), 80), - ) - session.add(new_broker) - existing_by_key[clave] = new_broker - inserted_count += 1 - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"CB import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"CB import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - # Limpieza - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"CB import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/customs_brokers/routes.py b/backend/api/v1/modules/a76/customs_brokers/routes.py index a94eccd1..960fb1f9 100644 --- a/backend/api/v1/modules/a76/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/customs_brokers/routes.py @@ -7,7 +7,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from . import dto, services -from .imports.routes import router as imports_router +from ..layouts_csv.customs_brokers.routes import router as imports_router router = APIRouter() diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py deleted file mode 100644 index c214b80b..00000000 --- a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py +++ /dev/null @@ -1,465 +0,0 @@ -""" -Tareas Celery para importación CSV de Tipos de Cambio. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from datetime import datetime, time -from decimal import Decimal -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -ER_IMPORT_FILE_PREFIX = "er_import_file:" -ER_IMPORT_META_PREFIX = "er_import_meta:" -ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:" -ER_IMPORT_REDIS_TTL = 3600 # 1 hour - -DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] -TEMPLATE_ID = "exchange_rates" - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"ER import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"er_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"ER import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{ER_IMPORT_FILE_PREFIX}{job_id}", - f"{ER_IMPORT_META_PREFIX}{job_id}", - f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"ER import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _parse_date(val: Optional[str]) -> Optional[datetime]: - """Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc.""" - if not val or not str(val).strip(): - return None - raw = str(val).strip() - for fmt in DATE_FORMATS: - try: - parsed = datetime.strptime(raw, fmt) - return datetime.combine(parsed.date(), time.min) - except ValueError: - continue - return None - - -def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Tipo de Cambio. Retorna error dict o None.""" - fecha_raw = (row.get("FECHA") or "").strip() - if not fecha_raw: - return {"line": line_num, "col": "FECHA", "msg": "Requerido"} - if _parse_date(fecha_raw) is None: - return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} - - valor_raw = (row.get("VALOR") or "").strip() - if not valor_raw: - return {"line": line_num, "col": "VALOR", "msg": "Requerido"} - try: - v = float(valor_raw.replace(",", ".")) - if v <= 0: - return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} - except ValueError: - return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} - - local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper() - if local_raw and len(local_raw) > 7: - return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"} - - foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper() - if foreign_raw and len(foreign_raw) > 7: - return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"ER import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"ER import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_exchange_rate(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"ER import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=ER_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"ER import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha). - """ - logger.info(f"ER import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"ER import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate - - inserted_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - existing_by_date: Dict[tuple, ExchangeRate] = {} - for er in ( - session.query(ExchangeRate) - .filter( - ExchangeRate.tenant_id == tenant_id, - ExchangeRate.company_id == company_id, - ) - .all() - ): - d = er.date.date() if hasattr(er.date, "date") else er.date - existing_by_date[(tenant_id, company_id, d)] = er - - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_exchange_rate(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - parsed_date = _parse_date(row_norm.get("FECHA")) - if not parsed_date: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "FECHA: no parseable"}) - continue - - try: - v = float((row_norm.get("VALOR") or "").strip().replace(",", ".")) - value_decimal = Decimal(str(round(v, 6))) - except (ValueError, TypeError): - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "VALOR: no numérico"}) - continue - - local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7) - if local_currency: - local_currency = local_currency.upper() - foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7) - if foreign_currency: - foreign_currency = foreign_currency.upper() - - key_date = parsed_date.date() - existing = existing_by_date.get((tenant_id, company_id, key_date)) - - if existing: - existing.value = value_decimal - existing.local_currency = local_currency or None - existing.foreign_currency = foreign_currency or None - session.add(existing) - inserted_count += 1 - else: - new_er = ExchangeRate( - tenant_id=tenant_id, - company_id=company_id, - date=parsed_date, - value=value_decimal, - local_currency=local_currency, - foreign_currency=foreign_currency, - ) - session.add(new_er) - existing_by_date[(tenant_id, company_id, key_date)] = new_er - inserted_count += 1 - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"ER import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"ER import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"ER import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py index 09086071..4fdef285 100644 --- a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py @@ -35,7 +35,7 @@ from fastapi import APIRouter custom_router = APIRouter(prefix="/exchange-rate", tags=[]) # CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.exchange_rate.routes import router as imports_router custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"]) @custom_router.get("/test-ping") diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py deleted file mode 100644 index edac5df2..00000000 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py +++ /dev/null @@ -1,474 +0,0 @@ -""" -Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions). -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from decimal import Decimal -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -FA_IMPORT_FILE_PREFIX = "fa_import_file:" -FA_IMPORT_META_PREFIX = "fa_import_meta:" -FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:" -FA_IMPORT_REDIS_TTL = 3600 # 1 hour - -TEMPLATE_ID = "us_tariff_fractions" - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"FA import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"fa_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"FA import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{FA_IMPORT_FILE_PREFIX}{job_id}", - f"{FA_IMPORT_META_PREFIX}{job_id}", - f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"FA import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _normalize_code(raw: Optional[str]) -> str: - """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" - if not raw: - return "" - s = str(raw).strip().replace(".", "").replace("-", "") - return s[:16] if len(s) > 16 else s - - -def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Fracción Americana. Retorna error dict o None.""" - code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip() - if not code_raw: - return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} - code_norm = _normalize_code(code_raw) - if not code_norm: - return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} - if len(code_norm) > 16: - return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"} - - prefix_raw = (row.get("PREFIJO") or "").strip() - if prefix_raw and len(prefix_raw) > 10: - return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"} - - um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip() - if um_raw and len(um_raw) > 10: - return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"} - - tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip() - if tipo_raw and len(tipo_raw) > 10: - return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"} - - adv_pct = row.get("ADVALOREM_PCT") - if adv_pct is not None and str(adv_pct).strip(): - try: - v = float(str(adv_pct).strip().replace(",", ".")) - if v < 0: - return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"} - except ValueError: - return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"} - - adv_dlls = row.get("ADVALOREM_DLLS") - if adv_dlls is not None and str(adv_dlls).strip(): - try: - v = float(str(adv_dlls).strip().replace(",", ".")) - if v < 0: - return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"} - except ValueError: - return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"FA import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"FA import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_us_tariff_fraction(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"FA import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=FA_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"FA import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _parse_float(val: Any) -> Optional[float]: - if val is None or str(val).strip() == "": - return None - try: - return float(str(val).strip().replace(",", ".")) - except (ValueError, TypeError): - return None - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code). - """ - logger.info(f"FA import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"FA import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction - - inserted_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_us_tariff_fraction(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA")) - if not code: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"}) - continue - - prefix = _str_or_none(row_norm.get("PREFIJO"), 10) - unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10) - description = _str_or_none(row_norm.get("DESCRIPCION")) - type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10) - ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT")) - fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS")) - fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None - - existing = ( - session.query(USTariffFraction) - .filter( - USTariffFraction.tenant_id == tenant_id, - USTariffFraction.company_id == company_id, - USTariffFraction.code == code, - ) - .first() - ) - - if existing: - existing.prefix = prefix - existing.type_code = type_code - existing.ad_valorem = ad_valorem - existing.fixed_cost = fixed_cost - existing.unit_of_measure = unit_of_measure - existing.description = description - session.add(existing) - inserted_count += 1 - else: - new_row = USTariffFraction( - tenant_id=tenant_id, - company_id=company_id, - code=code, - prefix=prefix, - type_code=type_code, - ad_valorem=ad_valorem, - fixed_cost=fixed_cost, - unit_of_measure=unit_of_measure, - description=description, - ) - session.add(new_row) - inserted_count += 1 - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"FA import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"FA import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path_clean = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path_clean): - os.remove(meta_path_clean) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"FA import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py index 61907ffa..d228d085 100644 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py @@ -31,7 +31,7 @@ crud_router = TenantCRUDRoutes( ) # Master router with prefix so all routes live under /us-tariff-fractions -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.us_tariff_fractions.routes import router as imports_router main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"]) main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"]) main_router.include_router(crud_router.router) diff --git a/backend/api/v1/modules/a76/layouts_csv/__init__.py b/backend/api/v1/modules/a76/layouts_csv/__init__.py new file mode 100644 index 00000000..29379c6d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/__init__.py @@ -0,0 +1 @@ +# Lógica centralizada de cargas por CSV (routes, tasks, validaciones, template_config por proceso) diff --git a/backend/api/v1/modules/a76/boms/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/boms/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/boms/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/boms/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py new file mode 100644 index 00000000..2d458d94 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py @@ -0,0 +1 @@ +# common validators, mappers, fk_loader for boms CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py new file mode 100644 index 00000000..7710d9b4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py @@ -0,0 +1,79 @@ +""" +Helpers reutilizables para validación de filas CSV (BOMs). +""" +import re +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional + + +def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_decimal_required_min( + row: Dict[str, Any], col: str, line_num: int, min_val: Decimal +) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or val == "": + return {"line": line_num, "col": col, "msg": "Requerido"} + try: + v = Decimal(str(val)) + if v < min_val: + return {"line": line_num, "col": col, "msg": "Debe ser mayor o igual a cero"} + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser número"} + return None + + +def _optional_number(val: Any) -> bool: + if val is None: + return True + s = re.sub(r"\s+", "", str(val).strip()) + if not s: + return True + try: + float(s.replace(",", ".")) + return True + except (ValueError, TypeError): + return False + + +def check_optional_number(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + if not _optional_number(row.get(col)): + return {"line": line_num, "col": col, "msg": "Debe ser número"} + return None + + +def check_in_set( + row: Dict[str, Any], + col: str, + line_num: int, + allowed: Optional[set], + msg: str, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val or allowed is None or len(allowed) == 0: + return None + if val not in allowed: + return {"line": line_num, "col": col, "msg": msg} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py new file mode 100644 index 00000000..1ee2b81c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py @@ -0,0 +1,27 @@ +""" +Carga de conjuntos FK para validación de import CSV de BOMs. +""" +from typing import Set + +from core.database import CoreSessionLocal + + +def load_boms_fk_sets(tenant_id: int, company_id: int) -> Set[str]: + """Carga valid_part_numbers (Part.part_number por tenant/company).""" + valid_part_numbers: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.parts.models import Part + for p in ( + session.query(Part.part_number) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + valid_part_numbers.add(p[0]) + except Exception as e: + import logging + logging.getLogger(__name__).warning("BOMs import: could not load parts: %s", e) + return valid_part_numbers diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py new file mode 100644 index 00000000..997723a8 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py @@ -0,0 +1,49 @@ +""" +Mapeo fila CSV → datos para BOM (para cuando exista tabla BOM). +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, Set + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val).replace(",", ".")) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_bom_data( + row_norm: Dict[str, Any], + valid_part_numbers: Set[str], +) -> Dict[str, Any]: + """ + Mapea una fila normalizada del CSV a un diccionario de datos para BOM. + Para uso futuro cuando exista tabla BOM. + """ + parent = _str_or_none(row_norm.get("NUMPARTE_PADRE"), 70) + component = _str_or_none(row_norm.get("NUMPARTE_COMPONENTE"), 70) + quantity = _decimal_or_none(row_norm.get("CANTIDAD")) + uom = _str_or_none(row_norm.get("UNIMED"), 10) + version_bom = _decimal_or_none(row_norm.get("VERSION_BOM")) + version_bill = _decimal_or_none(row_norm.get("VERSION_BILL")) + return { + "parent_part_number": parent, + "component_part_number": component, + "quantity": quantity, + "uom": uom, + "version_bom": version_bom, + "version_bill": version_bill, + } diff --git a/backend/api/v1/modules/a76/boms/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/boms/routes.py similarity index 98% rename from backend/api/v1/modules/a76/boms/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/boms/routes.py index 4a87ed0e..730878bb 100644 --- a/backend/api/v1/modules/a76/boms/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/boms/routes.py @@ -14,6 +14,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -78,7 +79,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/boms/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/boms/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/boms/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/boms/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py b/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py new file mode 100644 index 00000000..5e585be5 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py @@ -0,0 +1,168 @@ +""" +Tareas Celery para importación CSV de BOMs. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Sin tabla BOM dedicada aún: insert_valid_rows solo valida y cuenta filas válidas. +Usa layouts_csv.common y common.fk_loader, validators. +""" +import json +import logging +import os +from typing import Dict, Any, List + +from core.celery_app import celery_app + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import csv_reader as common_csv +from ..common import meta as common_meta +from ..common import responses as common_responses +from .template_config import row_from_template +from .validators import validate_row_bom +from .common.fk_loader import load_boms_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "bom" + +# Para routes.py +BOM_IMPORT_FILE_PREFIX = "bom_import_file:" +BOM_IMPORT_META_PREFIX = "bom_import_meta:" +BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:" +BOM_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("BOMs import: starting scan for job %s", job_id) + + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + valid_part_numbers = load_boms_fk_sets(tenant_id, company_id) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv.iter_csv_rows(file_path): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("BOMs import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail + ) + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("BOMs import: starting commit for job %s", job_id) + + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + valid_part_numbers = load_boms_fk_sets(tenant_id, company_id) + + inserted_count = 0 + skipped_invalid = 0 + valid_count = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + meta_path = common_meta.get_meta_path(file_path) + + try: + for i, row in common_csv.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + valid_count += 1 + # Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas. + + response = common_responses.commit_result( + "finished", inserted_count, skipped_invalid, 0, 0, skipped_details, + ) + if valid_count > 0 and inserted_count == 0: + response["message"] = ( + f"WIP: {valid_count} filas válidas. " + "La tabla BOM aún no existe en el sistema; no se insertó nada." + ) + except Exception as e: + logger.exception("BOMs import task failed") + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e), + ) + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if response is None: + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado", + ) + return response diff --git a/backend/api/v1/modules/a76/boms/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/boms/template_config.py similarity index 100% rename from backend/api/v1/modules/a76/boms/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/boms/template_config.py diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py new file mode 100644 index 00000000..43a40a2d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_bom + +__all__ = ["validate_row_bom"] diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py new file mode 100644 index 00000000..739690ac --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py @@ -0,0 +1,54 @@ +""" +Validaciones comunes de fila para import CSV de BOMs. +""" +from decimal import Decimal +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + check_max_length, + check_decimal_required_min, + check_optional_number, + check_in_set, +) + + +def validate_row_required_parent(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "NUMPARTE_PADRE", 70, line_num, required=True) + + +def validate_row_required_component(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "NUMPARTE_COMPONENTE", 70, line_num, required=True) + + +def validate_row_quantity(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_decimal_required_min(row, "CANTIDAD", line_num, Decimal("0")) + + +def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "UNIMED", 10, line_num) + + +def validate_row_optional_numbers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + err = check_optional_number(row, "VERSION_BOM", line_num) + if err: + return err + return check_optional_number(row, "VERSION_BILL", line_num) + + +def validate_row_fks( + row: Dict[str, Any], + line_num: int, + valid_part_numbers: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + if not valid_part_numbers or len(valid_part_numbers) == 0: + return None + err = check_in_set( + row, "NUMPARTE_PADRE", line_num, + valid_part_numbers, "Parte padre no existe en catálogo", + ) + if err: + return err + return check_in_set( + row, "NUMPARTE_COMPONENTE", line_num, + valid_part_numbers, "Parte componente no existe en catálogo", + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py new file mode 100644 index 00000000..cad75aa0 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py @@ -0,0 +1,43 @@ +""" +Punto de entrada de validación para import de una fila BOM. +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_required_parent, + validate_row_required_component, + validate_row_quantity, + validate_row_lengths, + validate_row_optional_numbers, + validate_row_fks, +) + + +def validate_row_bom( + row: Dict[str, Any], + line_num: int, + valid_part_numbers: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de BOMs. + Encadena: requeridos (padre, componente, cantidad) → longitudes → opcionales numéricos → FKs. + """ + err = validate_row_required_parent(row, line_num) + if err: + return err + err = validate_row_required_component(row, line_num) + if err: + return err + err = validate_row_quantity(row, line_num) + if err: + return err + err = validate_row_lengths(row, line_num) + if err: + return err + err = validate_row_optional_numbers(row, line_num) + if err: + return err + err = validate_row_fks(row, line_num, valid_part_numbers) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/__init__.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/__init__.py new file mode 100644 index 00000000..df6c436e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/__init__.py @@ -0,0 +1 @@ +# layouts_csv.cambio_regimen_regularizacion — carga CSV Cambio de régimen y Regularización (encabezado y partidas) diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py new file mode 100644 index 00000000..0bba9dce --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py @@ -0,0 +1,157 @@ +""" +Rutas de importación CSV para Cambio de régimen y Regularización (encabezado y partidas). +Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, UploadFile, Depends, Form, Query +from sqlalchemy.orm import Session +from typing import Literal, Optional, Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.paths import layout_path +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse, CommitRequest +from .tasks import scan_file, insert_valid_rows, JOB_TYPE, CRREG_IMPORT_REDIS_TTL +from ..common import storage as common_storage + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _default_template_id(model_target: str, document_type: Optional[str]) -> str: + if document_type == "regulariz": + return "regulariz_header" if model_target == "invoice_header" else "regulariz_details" + return "cam_reg_header" if model_target == "invoice_header" else "cam_reg_details" + + +@router.post("/upload/{model_target}", response_model=ImportJobResponse) +async def upload_import_file( + model_target: Literal["invoice_header", "invoice_details"], + file: UploadFile = File(...), + footer_config: Optional[str] = Form(None), + template_id: Optional[str] = Form(None), + document_type: Optional[str] = Query(None, description="cam_reg | regulariz"), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """Subir CSV, guardar en Redis, encolar scan. template_id/document_type distinguen Cambio de régimen vs Regularización.""" + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error("Cambio régimen/Regularización import: access validation failed: %s", e) + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id) + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "footer_config": footer_config, + "document_type": document_type or "cam_reg", + "template_id": template_id or _default_template_id(model_target, document_type), + } + + try: + r = _get_redis() + r.set(file_key, base64.b64encode(contents), ex=CRREG_IMPORT_REDIS_TTL) + r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=CRREG_IMPORT_REDIS_TTL) + except Exception as e: + logger.error("Cambio régimen/Regularización import: Redis store error: %s", e) + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = layout_path("imports", "temp") + os.makedirs(upload_dir, exist_ok=True) + csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + with open(csv_path, "wb") as f: + f.write(contents) + meta_path = csv_path.replace(".csv", ".meta.json") + with open(meta_path, "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning("Cambio régimen/Regularización import: local file save failed: %s", e) + + scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """Polling: estado del escaneo o del commit.""" + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"): + return task_result.result + + logger.warning("Cambio régimen/Regularización import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg: + result = getattr(task_result, "result", None) + if result is not None and not isinstance(result, dict): + err_msg = str(result) + elif isinstance(result, dict) and (result.get("error") or result.get("message")): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str, body: CommitRequest): + """Usuario confirma; se encola la tarea de commit (por ahora sin inserción real).""" + task = insert_valid_rows.delay(job_id, body.model_target) + return { + "status": "committing", + "message": "Proceso de commit iniciado.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/schemas.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/schemas.py new file mode 100644 index 00000000..2a043f30 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/schemas.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel +from typing import Optional, Literal + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + model_target: Literal["invoice_header", "invoice_details"] + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + error_file: Optional[str] = None diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py new file mode 100644 index 00000000..c06ce2b9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py @@ -0,0 +1,146 @@ +""" +Tareas Celery para importación CSV de Cambio de régimen y Regularización (encabezado y partidas). +Flujo: scan_file (sin validaciones) → insert_valid_rows (sin inserción en BD). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +Validaciones independientes por document_type (cam_reg vs regulariz) se añadirán después. +""" +import logging +import os +from typing import Dict, Any, Optional + +from core.celery_app import celery_app + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +JOB_TYPE = "crreg" +CRREG_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _ensure_file(job_id: str) -> Optional[str]: + return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Cambio régimen/Regularización import") + + +def _ensure_meta(job_id: str, file_path: str) -> bool: + return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Cambio régimen/Regularización import") + + +def _norm_row(row: Dict[str, Any], template_id: str) -> Dict[str, Any]: + return row_from_template(row, template_id, common_normalize.normalize_header) + + +@celery_app.task(bind=True, name="api.v1.modules.a76.layouts_csv.cambio_regimen_regularizacion.tasks.scan_file") +def scan_file(self, job_id: str, model_target: str, config: str = None): + """ + Scan CSV sin validaciones: leer, normalizar con plantilla, devolver total_rows y 0 errores. + """ + logger.info("Cambio régimen/Regularización import: starting scan for job %s target %s", job_id, model_target) + + file_path = _ensure_file(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + if os.path.getsize(file_path) == 0: + return {"status": "failed", "error": "El archivo está vacío."} + _ensure_meta(job_id, file_path) + + try: + common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + document_type = meta.get("document_type") or "cam_reg" + template_id = meta.get("template_id") or ( + "cam_reg_header" if model_target == "invoice_header" else "cam_reg_details" + ) + if document_type == "regulariz" and not meta.get("template_id"): + template_id = "regulariz_header" if model_target == "invoice_header" else "regulariz_details" + + total_rows = 0 + processed_rows = 0 + + try: + total_rows = common_csv_reader.count_csv_rows(file_path, has_header=True) + except Exception as e: + return {"status": "failed", "error": str(e)} + + def on_progress(current: int, total: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total}) + + try: + for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None): + if i % 500 == 0: + on_progress(i, total_rows) + _norm_row(row, template_id) + processed_rows += 1 + except Exception as e: + logger.error("Cambio régimen/Regularización import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, 0, []) + + +@celery_app.task(bind=True, name="api.v1.modules.a76.layouts_csv.cambio_regimen_regularizacion.tasks.insert_valid_rows") +def insert_valid_rows(self, job_id: str, model_target: str): + """ + Commit sin inserción en BD: leer CSV, omitir líneas de error (vacío por ahora), cleanup, devolver finished con inserted=0. + """ + logger.info("Cambio régimen/Regularización import: starting commit for job %s target %s", job_id, model_target) + + file_path = _ensure_file(job_id) + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + _ensure_meta(job_id, file_path) + + try: + common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + meta_path = common_meta.get_meta_path(file_path) + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + document_type = meta.get("document_type") or "cam_reg" + template_id = meta.get("template_id") or ( + "cam_reg_header" if model_target == "invoice_header" else "cam_reg_details" + ) + if document_type == "regulariz" and not meta.get("template_id"): + template_id = "regulariz_header" if model_target == "invoice_header" else "regulariz_details" + + try: + for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None): + if i in error_lines: + continue + _norm_row(row, template_id) + except Exception as e: + logger.error("Cambio régimen/Regularización import commit read failed: %s", e) + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + return { + "status": "finished", + "inserted": 0, + "skipped_invalid": 0, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": [], + "message": "Proceso base listo; validaciones e inserción pendientes.", + } diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/template_config.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/template_config.py new file mode 100644 index 00000000..642e413e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/template_config.py @@ -0,0 +1,138 @@ +""" +Plantillas CSV para Cambio de régimen y Regularización (encabezado y partidas). +Por ahora misma estructura que encabezado/partidas de exportación; luego se ajustan columnas si difieren. +""" + +from typing import Dict, List, Any, Optional + +# Cambio de régimen: cam_reg_header, cam_reg_details +# Regularización: regulariz_header, regulariz_details +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "cam_reg_header": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, + {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, + {"canonical": "FECHA EMISION"}, + {"canonical": "CLAVE PROVEEDOR"}, + {"canonical": "CLAVE VENDIDO A"}, + {"canonical": "CLAVE ENVIADO A"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, + {"canonical": "ADUANA DE CRUCE"}, + {"canonical": "CLAVE MONEDA"}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "TIPO PESO"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "REMESA"}, + {"canonical": "AGENTE ADUANAL"}, + {"canonical": "FLETES"}, + {"canonical": "VALOR SEGUROS"}, + {"canonical": "SEGUROS"}, + {"canonical": "EMBALAJES"}, + {"canonical": "OTROS INCREMENTABLES"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "OBSERVACIONES E"}, + {"canonical": "OBSERVACIONES I"}, + {"canonical": "E DOCUMENT"}, + {"canonical": "NUM OPERACION"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "PRECINTO"}, + ], + "cam_reg_details": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]}, + {"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]}, + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]}, + {"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]}, + {"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]}, + {"canonical": "CANTIDAD"}, + {"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]}, + {"canonical": "FRACCION"}, + {"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]}, + ], + "regulariz_header": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, + {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, + {"canonical": "FECHA EMISION"}, + {"canonical": "CLAVE PROVEEDOR"}, + {"canonical": "CLAVE VENDIDO A"}, + {"canonical": "CLAVE ENVIADO A"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, + {"canonical": "ADUANA DE CRUCE"}, + {"canonical": "CLAVE MONEDA"}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "TIPO PESO"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "REMESA"}, + {"canonical": "AGENTE ADUANAL"}, + {"canonical": "FLETES"}, + {"canonical": "VALOR SEGUROS"}, + {"canonical": "SEGUROS"}, + {"canonical": "EMBALAJES"}, + {"canonical": "OTROS INCREMENTABLES"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "OBSERVACIONES E"}, + {"canonical": "OBSERVACIONES I"}, + {"canonical": "E DOCUMENT"}, + {"canonical": "NUM OPERACION"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "PRECINTO"}, + ], + "regulariz_details": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]}, + {"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]}, + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]}, + {"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]}, + {"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]}, + {"canonical": "CANTIDAD"}, + {"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]}, + {"canonical": "FRACCION"}, + {"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]}, + ], +} + + +def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]: + return TEMPLATE_COLUMNS.get(template_id) + + +def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name.""" + cols = _resolve_template_columns(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]: + """Fila CSV -> dict con nombres canónicos de la plantilla.""" + lookup = build_normalized_lookup(template_id, normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/classes/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/classes/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/classes/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/classes/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py new file mode 100644 index 00000000..e7e4b5e4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py @@ -0,0 +1 @@ +# common validators, mappers, fk_loader for classes CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py new file mode 100644 index 00000000..c08ace58 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py @@ -0,0 +1,108 @@ +""" +Helpers reutilizables para validación de filas CSV (clases de materiales). +""" +from typing import Dict, Any, Optional + + +def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_min_length( + row: Dict[str, Any], + col: str, + min_len: int, + line_num: int, + msg: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Solo valida si hay valor; error si longitud menor que min_len.""" + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) < min_len: + return { + "line": line_num, + "col": col, + "msg": msg or f"Mínimo {min_len} caracteres", + } + return None + + +def check_int_range( + row: Dict[str, Any], + col: str, + line_num: int, + min_val: int, + max_val: int, +) -> Optional[Dict[str, Any]]: + """Solo valida si hay valor; devuelve error si no es int o está fuera de rango.""" + val = row.get(col) + if val is None or val == "": + return None + try: + v = int(val) + if v < min_val or v > max_val: + return {"line": line_num, "col": col, "msg": "Valor fuera de rango"} + except (ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser número entero"} + return None + + +def check_in_set( + row: Dict[str, Any], + col: str, + line_num: int, + allowed: Optional[set], + msg: str = "No existe en el catálogo", +) -> Optional[Dict[str, Any]]: + """Solo valida si hay valor y allowed no es None.""" + val = (row.get(col) or "").strip() + if not val or allowed is None: + return None + if val not in allowed: + return {"line": line_num, "col": col, "msg": msg} + return None + + +def check_decimal_max( + row: Dict[str, Any], + col: str, + line_num: int, + max_val: float, + msg: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Solo valida si hay valor; error si no es numérico o si es mayor que max_val (Clarion Col H).""" + val = row.get(col) + if val is None or val == "": + return None + try: + v = float(val) + if v > max_val: + return { + "line": line_num, + "col": col, + "msg": msg or f"El valor no puede ser mayor a {max_val}.", + } + except (ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser un número."} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py new file mode 100644 index 00000000..95f331d6 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py @@ -0,0 +1,86 @@ +""" +Carga de conjuntos FK para validación/mapeo de import CSV de clases de materiales. +Clarion: Tipo Activo Fijo, U.M., Fracción Mex (GFracGenSifra + histórico), Fracción Ame (GFracAme), Código Producto CP (si existe). +""" +from typing import Set, Tuple + +from core.database import CoreSessionLocal + + +def load_classes_fk_sets( + tenant_id: int, + company_id: int, +) -> Tuple[Set[str], Set[str], Set[str], Set[str], Set[str]]: + """ + Carga todos los conjuntos necesarios para validación CSV de clases (paridad Clarion). + Devuelve (valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp). + - valid_fraction_mex_8: códigos de 8 caracteres válidos (TariffFraction + HistoricalTariffFraction). + - valid_fraction_ame: códigos de fracción americana (USTariffFraction por tenant/company). + - valid_product_codes_cp: códigos de producto/servicio CP (vacío si no existe catálogo). + """ + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_fraction_mex_8: Set[str] = set() + valid_fraction_ame: Set[str] = set() + valid_product_codes_cp: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.a76.general_catalogs.fractions.tariff_fractions.models import TariffFraction + from api.v1.modules.a76.general_catalogs.fractions.historical_tariff_fractions.models import HistoricalTariffFraction + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + + for m in session.query(MaterialType.key).all(): + if m[0]: + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + if u[0]: + valid_uom_codes.add(u[0]) + + for row in session.query(TariffFraction.code).all(): + if row[0]: + code = row[0].strip() + valid_fraction_mex_8.add(code[:8]) + + for row in ( + session.query(HistoricalTariffFraction.historical_fraction) + .filter( + HistoricalTariffFraction.tenant_id == tenant_id, + HistoricalTariffFraction.company_id == company_id, + HistoricalTariffFraction.historical_fraction.isnot(None), + ) + .distinct() + .all() + ): + if row[0] and row[0].strip(): + valid_fraction_mex_8.add(row[0].strip()[:8]) + + for row in ( + session.query(USTariffFraction.code) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + ) + .all() + ): + if row[0]: + valid_fraction_ame.add(row[0].strip()) + + except Exception as e: + import logging + logging.getLogger(__name__).warning("Classes import: could not load FK sets: %s", e) + return ( + valid_material_keys, + valid_uom_codes, + valid_fraction_mex_8, + valid_fraction_ame, + valid_product_codes_cp, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py new file mode 100644 index 00000000..d7380170 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py @@ -0,0 +1,82 @@ +""" +Mapeo fila CSV → datos para Class (clases de materiales). +""" +from typing import Dict, Any, Optional, Set + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +def row_to_class_data( + row_norm: Dict[str, Any], + valid_material_keys: Set[str], + valid_uom_codes: Set[str], +) -> Dict[str, Any]: + """ + Mapea una fila normalizada del CSV a un diccionario de datos para Class. + class_code se normaliza a mayúsculas (Clip(Upper) Clarion). + """ + raw_clase = _str_or_none(row_norm.get("CLASE"), 8) + class_code = raw_clase.upper() if raw_clase else None + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) + if material_key and material_key not in valid_material_keys: + material_key = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + fraction = _str_or_none(row_norm.get("FRACCION"), 20) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) + physical_review = _int_or_none(row_norm.get("REVFISICA")) + iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) + + return { + "class_code": class_code, + "description_es": desc_es, + "description_en": desc_en, + "material_key": material_key, + "unit_of_measure": unit_of_measure, + "fraction": fraction, + "us_fraction": us_fraction, + "sub_key": sub_key, + "physical_review": physical_review, + "iva_exempt_fraction": iva_exempt_fraction, + } + + +def row_to_class_data_merge_existing( + row_norm: Dict[str, Any], + existing_data: Dict[str, Any], + valid_material_keys: Set[str], + valid_uom_codes: Set[str], +) -> Dict[str, Any]: + """ + Para modo actualizar (parcial): valores del CSV si no vacíos, sino los de la clase existente (Clarion VALIDA_PARCIAL_CLASE). + """ + data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes) + if not data["class_code"]: + return data + for key in ("description_es", "description_en", "material_key", "unit_of_measure", + "fraction", "us_fraction", "sub_key", "physical_review", "iva_exempt_fraction"): + if data.get(key) is None or (isinstance(data[key], str) and not data[key].strip()): + data[key] = existing_data.get(key) + return data diff --git a/backend/api/v1/modules/a76/classes/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/classes/routes.py similarity index 79% rename from backend/api/v1/modules/a76/classes/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/classes/routes.py index 5c9c869f..338c97ea 100644 --- a/backend/api/v1/modules/a76/classes/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/classes/routes.py @@ -14,6 +14,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -39,6 +40,8 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo actualizar (ACT): validación parcial si la clase existe"), + siempre_toda: bool = Query(False, description="Forzar siempre validación completa"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -59,6 +62,8 @@ async def upload_import_file( "company_id": company_id, "user_id": current_user.get("id"), "template_id": "material_classes", + "actualizar": actualizar, + "siempre_toda": siempre_toda, } try: @@ -78,7 +83,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f: f.write(contents) @@ -119,6 +124,27 @@ async def get_import_status(job_id: str): if isinstance(result, dict) and result.get("status") in ("finished", "warning"): return result + # Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica + if isinstance(result, str): + try: + parsed = json.loads(result) + if isinstance(parsed, dict) and ( + parsed.get("status") == "waiting_confirmation" + or (parsed.get("job_id") and "total_rows" in parsed) + ): + return parsed + if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"): + return parsed + except (json.JSONDecodeError, TypeError): + pass + + # Si el resultado tiene forma de escaneo (waiting_confirmation), devolverlo para que el front muestre el modal + if isinstance(result, dict) and ( + result.get("status") == "waiting_confirmation" + or (result.get("job_id") and "total_rows" in result) + ): + return result + logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state) err_msg = None tb = getattr(task_result, "traceback", None) diff --git a/backend/api/v1/modules/a76/classes/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/classes/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/classes/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/classes/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py b/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py new file mode 100644 index 00000000..eb765a83 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py @@ -0,0 +1,309 @@ +""" +Tareas Celery para importación CSV de Clases de Materiales. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers. +""" +import json +import logging +import os +from typing import Dict, Any, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import csv_reader as common_csv +from ..common import meta as common_meta +from ..common import responses as common_responses +from .template_config import row_from_template, detect_headers_or_data +from .validators import validate_row_class, validate_row_class_partial +from .common.mappers import row_to_class_data, row_to_class_data_merge_existing +from .common.fk_loader import load_classes_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "cls" + +# Para routes.py +CLS_IMPORT_FILE_PREFIX = "cls_import_file:" +CLS_IMPORT_META_PREFIX = "cls_import_meta:" +CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" +CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Classes import: starting scan for job %s", job_id) + + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header) + try: + total_rows = common_csv.count_csv_rows(file_path, has_header=has_header) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + actualizar = meta.get("actualizar", False) + siempre_toda = meta.get("siempre_toda", False) + + valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets( + tenant_id, company_id + ) + + from api.v1.modules.a76.classes.models import Class + existing_class_codes = set() + try: + with CoreSessionLocal() as session: + for c in session.query(Class.class_code).filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ).all(): + if c[0]: + existing_class_codes.add(c[0].strip().upper()) + except Exception as e: + logger.warning("Classes import: could not load existing class codes: %s", e) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_class( + row_norm, + i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + actualizar=actualizar, + siempre_toda=siempre_toda, + existing_class_codes=existing_class_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_fraction_ame=valid_fraction_ame, + valid_product_codes_cp=valid_product_codes_cp, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Classes import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail + ) + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Classes import: starting commit for job %s", job_id) + + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + actualizar = meta.get("actualizar", False) + siempre_toda = meta.get("siempre_toda", False) + + from api.v1.modules.a76.classes.models import Class + + valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets( + tenant_id, company_id + ) + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + meta_path = common_meta.get_meta_path(file_path) + + fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header) + + try: + with CoreSessionLocal() as session: + existing_by_code = {} + for c in session.query(Class).filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ).all(): + key = (c.class_code or "").strip().upper() + if key: + existing_by_code[key] = c + + for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + class_code_raw = (row_norm.get("CLASE") or "").strip().upper()[:8] + use_partial = actualizar and class_code_raw and class_code_raw in existing_by_code and not siempre_toda + + if use_partial: + err = validate_row_class_partial( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_fraction_ame=valid_fraction_ame, + valid_product_codes_cp=valid_product_codes_cp, + ) + else: + err = validate_row_class( + row_norm, + i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + actualizar=actualizar, + siempre_toda=siempre_toda, + existing_class_codes=set(existing_by_code.keys()), + valid_fraction_mex_8=valid_fraction_mex_8, + valid_fraction_ame=valid_fraction_ame, + valid_product_codes_cp=valid_product_codes_cp, + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + if use_partial: + existing = existing_by_code.get(class_code_raw) + existing_data = { + "description_es": existing.description_es, + "description_en": existing.description_en, + "material_key": existing.material_key, + "unit_of_measure": existing.unit_of_measure, + "fraction": existing.fraction, + "us_fraction": existing.us_fraction, + "sub_key": existing.sub_key, + "physical_review": existing.physical_review, + "iva_exempt_fraction": existing.iva_exempt_fraction, + } + data = row_to_class_data_merge_existing( + row_norm, existing_data, valid_material_keys, valid_uom_codes, + ) + else: + data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes) + + class_code = data.get("class_code") + if not class_code: + skipped_invalid += 1 + continue + + existing = existing_by_code.get(class_code) + if existing: + existing.description_es = data["description_es"] + existing.description_en = data["description_en"] + existing.material_key = data["material_key"] + existing.unit_of_measure = data["unit_of_measure"] + existing.fraction = data["fraction"] + existing.us_fraction = data["us_fraction"] + existing.sub_key = data["sub_key"] + existing.physical_review = data["physical_review"] + existing.iva_exempt_fraction = data["iva_exempt_fraction"] + session.add(existing) + inserted_count += 1 + else: + new_class = Class( + tenant_id=tenant_id, + company_id=company_id, + **data, + ) + session.add(new_class) + existing_by_code[class_code] = new_class + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("Classes import DB error: %s", db_err) + return common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, error=str(db_err), + ) + + if inserted_count == 0 and skipped_invalid > 0: + response = common_responses.commit_result( + "warning", 0, skipped_invalid, 0, 0, + skipped_details, + message=f"No se insertaron registros. {skipped_invalid} rechazados.", + ) + elif inserted_count == 0: + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, + error="No hay registros válidos en el archivo CSV", + ) + else: + response = common_responses.commit_result( + "finished", inserted_count, skipped_invalid, 0, 0, + skipped_details, + ) + + except Exception as e: + logger.exception("Classes import task failed") + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, error=str(e), + ) + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if response is None: + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, error="Error inesperado", + ) + return response diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/template_config.py b/backend/api/v1/modules/a76/layouts_csv/classes/template_config.py new file mode 100644 index 00000000..b4c9623e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/template_config.py @@ -0,0 +1,107 @@ +""" +Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls). +Cabeceras de descarga = Clarion: CLAVE CLASE, DESCRIPCION ESPAÑOL, DESCRIPCION INGLES, etc. +""" +import csv +import io +from typing import Dict, List, Any, Optional, Tuple + + +# Valores que indican que la primera fila es cabecera (primera columna normalizada) +FIRST_COLUMN_HEADER_VALUES = ("CLAVE CLASE", "CLASE") + + +def detect_headers_or_data( + file_path: str, + normalize_header_fn, + encoding: str = "utf-8-sig", +) -> Tuple[Optional[List[str]], bool]: + """ + Lee la primera línea del CSV y decide si es cabecera o dato. + Devuelve (fieldnames, has_header). + - Si la primera celda normalizada está en FIRST_COLUMN_HEADER_VALUES -> has_header=True, fieldnames=None + (la primera fila es cabecera; iter_csv_rows sin fieldnames). + - Si no -> has_header=False, fieldnames=TEMPLATE_DOWNLOAD_HEADERS (la primera fila es dato). + """ + try: + with open(file_path, "r", encoding=encoding) as f: + sample = f.read(2048) + except Exception: + return None, True + lines = sample.splitlines() + if not lines: + return None, True + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = csv.excel + reader = csv.reader(io.StringIO(lines[0]), dialect=dialect) + first_row = next(reader, None) + if not first_row: + return None, True + first_cell = (first_row[0] or "").strip() + first_cell_norm = normalize_header_fn(first_cell) + if first_cell_norm in FIRST_COLUMN_HEADER_VALUES: + return None, True + return list(TEMPLATE_DOWNLOAD_HEADERS), False + +# Cabeceras que se escriben al descargar la plantilla CSV (igual que Clarion EstructuraCatClasesAF) +TEMPLATE_DOWNLOAD_HEADERS: List[str] = [ + "CLAVE CLASE", + "DESCRIPCION ESPAÑOL", + "DESCRIPCION INGLES", + "TIPO DE MATERIAL", + "U.M. COMERCIAL", + "FRACCION ARANCELARIA", + "FRACCION AMERICANA", + "TASA DE DEPRECIACION", + "REVISION FISICA (1/0)", + "CODIGO DE PRODUCTO/SERVICIO CP", +] + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "material_classes": [ + {"canonical": "CLASE", "aliases": ["CLAVE CLASE", "CLASS", "CODIGO", "CLASE CODIGO"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION ESPAÑOL", "DESCRIPCION", "DESCRIPCION ES", "DESCRIPCION ESPAÑOL"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION INGLES", "DESCRIPCION EN", "DESCRIPTION", "DESCRIPCION INGLES"]}, + {"canonical": "CLAVEMAT", "aliases": ["TIPO DE MATERIAL", "MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]}, + {"canonical": "UNIMED", "aliases": ["U.M. COMERCIAL", "UNIDAD MEDIDA", "UNIT", "UOM", "TIPO DE MU.M.", "U.M. COMERCIAL"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION ARANCELARIA", "FRACCION MEX", "COM FRACCION"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION AMERICANA", "FRACCION USA", "US FRACTION", "FRACCION"]}, + {"canonical": "TASADEPRECIA", "aliases": ["TASA DE DEPRECIACION", "TASA DEPRECIACIÓN", "TASA DEPRECIACION"]}, + {"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]}, + {"canonical": "REVFISICA", "aliases": ["REVISION FISICA (1/0)", "REV FISICA", "PHYSICAL REVIEW", "TASA DE REVISION", "REVISION FISICA"]}, + {"canonical": "FRACCIONEXENTAIVA", "aliases": ["CODIGO DE PRODUCTO/SERVICIO CP", "EXENTA IVA", "FRACCION EXENTA IVA", "CODIGO PRODUCTO SERVICIO CP"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("material_classes") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + elif key_norm.startswith("CLAVE CLASE"): + # CSV leído con delimitador incorrecto: primera columna es "CLAVE CLASE,..." -> usar primer valor como CLASE + if "CLASE" not in out and value: + first_val = (value.split(",")[0] if "," in str(value) else value).strip() + if first_val: + out["CLASE"] = first_val + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py new file mode 100644 index 00000000..9d108067 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_class, validate_row_class_partial + +__all__ = ["validate_row_class", "validate_row_class_partial"] diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py new file mode 100644 index 00000000..5f4bd02b --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py @@ -0,0 +1,239 @@ +""" +Validaciones comunes de fila para import CSV de clases de materiales. +Paridad con Clarion: VALIDACIONES_CLASE (Col A len, Col D/E/F/G en catálogo, Col F len≥8, Col H ≤100, Col J en catálogo CP). +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + check_max_length, + check_int_range, + check_decimal_max, +) + +MSG_CLASE_VACIO = "Error: (Col. A) La columna de Clase esta vacio y no se pueden hacer las validaciones." +MSG_CLASE_VACIO_SOLUCION = "Capturar en la Columna A una Clase nueva o una ya existente a la cual desee actualizar campos" + + +def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """CLASE obligatorio; mensaje Clarion si está vacío. Acepta clave 'CLASE' o 'CLAVE CLASE' (por si el CSV no se normalizó).""" + val = (row.get("CLASE") or row.get("CLAVE CLASE") or "").strip() + if not val: + return {"line": line_num, "col": "CLASE", "msg": f"{MSG_CLASE_VACIO} {MSG_CLASE_VACIO_SOLUCION}"} + if len(val) > 8: + return { + "line": line_num, + "col": "CLASE", + "msg": f"Error: (Col. A) La Clase: {val} supera la longitud de caracteres. Capturar en la columna A una Clase de 8 caracteres como máximo.", + } + return None + + +def validate_row_required_full(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Obligatorios en validación completa: B, D, E, F (Clarion VALIDA_TODA_CLASE).""" + cols_labels = [ + ("DESCRIPCIONE", "(Col.B) Descripción Español"), + ("CLAVEMAT", "(Col.D) Tipo de Material"), + ("UNIMED", "(Col.E) U.M. Comercial"), + ("FRACCION", "(Col.F) Fraccion Arancelaria Mex."), + ] + missing = [(col, label) for col, label in cols_labels if not (row.get(col) or "").strip()] + if not missing: + return None + campos = ", ".join(label for _, label in missing) + first_col = missing[0][0] + return { + "line": line_num, + "col": first_col, + "msg": f"Existen campos vacios que son obligatorios, es la {campos}. Revisar la línea del archivo y capturar los campos con la información correcta.", + } + + +def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("DESCRIPCIONE", 500), + ("DESCRIPCIONI", 500), + ("CLAVEMAT", 10), + ("UNIMED", 5), + ("FRACCION", 20), + ("FRACCIONAME", 16), + ("CLAVESUB", 5), + ("FRACCIONEXENTAIVA", 4), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_fraction_min(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """FRACCION (Col F): si no vacío, mínimo 8 caracteres (Clarion VALIDACIONES_CLASE).""" + val = (row.get("FRACCION") or "").strip() + if not val: + return None + if len(val) < 8: + return { + "line": line_num, + "col": "FRACCION", + "msg": f"La Fraccion {val} no alcanza la longitud de 8 caracteres. Capturar en la columna F una Fracción de 8 caracteres.", + } + return None + + +def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_int_range(row, "REVFISICA", line_num, -32768, 32767) + + +def validate_row_fks( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]], + valid_uom_codes: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + val_d = (row.get("CLAVEMAT") or "").strip() + if val_d and valid_material_keys is not None and val_d not in valid_material_keys: + return { + "line": line_num, + "col": "CLAVEMAT", + "msg": f"Error: (Col. D) El Tipo de Activo Fijo: {val_d} no existe en el Catálogo de Tipos de Activo Fijo. Revisar este Tipo de Activo Fijo en el archivo, en caso de ser correcto actualice los catálogos fijos.", + } + val_e = (row.get("UNIMED") or "").strip() + if val_e and valid_uom_codes is not None and val_e not in valid_uom_codes: + return { + "line": line_num, + "col": "UNIMED", + "msg": f"Error: (Col. E) La Unidad de Medida Comercial: {val_e} no existe en el Catálogo de U.M. Revisar esta Unidad de Medida en el archivo, en caso de ser correcta actualice los catálogos.", + } + return None + + +def _normalize_fraction_mex_8(value: str) -> str: + """Primeros 8 caracteres si len>=10, sino hasta 8 (Clarion SUB(ColumnaF, 1, 8)).""" + if not value: + return "" + v = value.strip() + if len(v) >= 10: + return v[:8] + return v[:8] if len(v) > 8 else v + + +def validate_row_fraction_mex_catalog( + row: Dict[str, Any], + line_num: int, + valid_fraction_mex_8: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col F: si no vacía, debe existir en catálogo Mex (GFracGenSifra) o Histórico (Clarion).""" + val = (row.get("FRACCION") or "").strip() + if not val or valid_fraction_mex_8 is None: + return None + code_8 = _normalize_fraction_mex_8(val) + if not code_8: + return None + if code_8 in valid_fraction_mex_8: + return None + return { + "line": line_num, + "col": "FRACCION", + "msg": ( + f"Error: (Col. F) La Fraccion Mexicana: {val} no existe en el Catálogo de Fracciones Arancelarias Sifr@ ni en el Historico. " + "Revisar esta Fracción Arancelaria en el archivo, en caso de ser correcta Actualizar las Fracciones Arancelarias." + ), + } + + +def validate_row_fraction_ame_catalog( + row: Dict[str, Any], + line_num: int, + valid_fraction_ame: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col G: si no vacía, debe existir en catálogo Fracciones Americanas (Clarion GFracAme).""" + val = (row.get("FRACCIONAME") or "").strip() + if not val or valid_fraction_ame is None: + return None + if val in valid_fraction_ame: + return None + return { + "line": line_num, + "col": "FRACCIONAME", + "msg": ( + f"Error: (Col. G) La Fraccion Americana: {val} no existe en el Catálogo de Fracciones Americanas. " + "Dar de alta la Fracción Americana en el Catálogo de Fracciones Americanas." + ), + } + + +def validate_row_tasa_depreciacion(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col H: si viene informada, no puede ser mayor al 100 % (Clarion).""" + val = row.get("TASADEPRECIA") + if val is None or val == "": + return None + err = check_decimal_max(row, "TASADEPRECIA", line_num, 100.0, msg=None) + if err and "100" in (err.get("msg") or ""): + v = (row.get("TASADEPRECIA") or "").strip() + err["msg"] = f"Error: (Col. H) La Tasa de Depreciación: {v} no puede ser mayor al 100 %. Ajustar la Tasa de Depreciacion." + return err + + +def validate_row_codigo_producto_cp( + row: Dict[str, Any], + line_num: int, + valid_product_codes_cp: Optional[Set[str]], + class_code: str = "", +) -> Optional[Dict[str, Any]]: + """Col J: si no vacía y existe catálogo CP, debe existir en GCodigosProductoCP (Clarion).""" + val = (row.get("FRACCIONEXENTAIVA") or "").strip() + if not val: + return None + if valid_product_codes_cp is None or len(valid_product_codes_cp) == 0: + return None + if val in valid_product_codes_cp: + return None + cl = class_code or (row.get("CLASE") or "").strip() + return { + "line": line_num, + "col": "FRACCIONEXENTAIVA", + "msg": ( + f"Error: (Col. J) La clase: {cl} tiene asignado un código de producto inexistente. " + "Capturar un código de producto correcto." + ), + } + + +def validaciones_clase( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]], + valid_uom_codes: Optional[Set[str]], + valid_fraction_mex_8: Optional[Set[str]] = None, + valid_fraction_ame: Optional[Set[str]] = None, + valid_product_codes_cp: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Reglas compartidas Clarion (VALIDACIONES_CLASE): longitudes, tipos, FKs, fracciones, tasa, código CP.""" + err = validate_row_lengths(row, line_num) + if err: + return err + err = validate_row_fraction_min(row, line_num) + if err: + return err + err = validate_row_types(row, line_num) + if err: + return err + err = validate_row_fks(row, line_num, valid_material_keys, valid_uom_codes) + if err: + return err + err = validate_row_fraction_mex_catalog(row, line_num, valid_fraction_mex_8) + if err: + return err + err = validate_row_fraction_ame_catalog(row, line_num, valid_fraction_ame) + if err: + return err + err = validate_row_tasa_depreciacion(row, line_num) + if err: + return err + class_code = (row.get("CLASE") or "").strip() + err = validate_row_codigo_producto_cp( + row, line_num, valid_product_codes_cp, class_code + ) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py new file mode 100644 index 00000000..10ea806d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py @@ -0,0 +1,81 @@ +""" +Punto de entrada de validación para import de una fila de clase de material. +Flujo Clarion: no ACT → siempre VALIDA_TODA_CLASE; ACT y clase existe → VALIDA_PARCIAL_CLASE; +ACT y clase no existe → VALIDA_TODA_CLASE (validación completa) y si pasa se crea en el insert (ADD). +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_required, + validate_row_required_full, + validaciones_clase, +) + + +def validate_row_class( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + actualizar: bool = False, + siempre_toda: bool = False, + existing_class_codes: Optional[Set[str]] = None, + valid_fraction_mex_8: Optional[Set[str]] = None, + valid_fraction_ame: Optional[Set[str]] = None, + valid_product_codes_cp: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Igual que Clarion: + - No ACT (actualizar=False): siempre TODA (B,D,E,F obligatorios + validaciones_clase). + - ACT y clase existe: PARCIAL (solo CLASE + validaciones_clase); en insert se actualiza (PUT). + - ACT y clase no existe: TODA (validación completa); si pasa, en insert se crea (ADD). + siempre_toda fuerza TODA en todos los casos. + """ + err = validate_row_required(row, line_num) + if err: + return err + + class_code = (row.get("CLASE") or "").strip().upper()[:8] + use_full = siempre_toda or not actualizar + if actualizar and existing_class_codes is not None and class_code in existing_class_codes: + use_full = False + + if use_full: + err = validate_row_required_full(row, line_num) + if err: + return err + # En Actualizar, si la clase no existe se valida completa y si pasa se crea en insert (como Clarion ADD). + + return validaciones_clase( + row, + line_num, + valid_material_keys, + valid_uom_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_fraction_ame=valid_fraction_ame, + valid_product_codes_cp=valid_product_codes_cp, + ) + + +def validate_row_class_partial( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_fraction_mex_8: Optional[Set[str]] = None, + valid_fraction_ame: Optional[Set[str]] = None, + valid_product_codes_cp: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Validación parcial (modo Act, clase existente): solo CLASE + validaciones_clase.""" + err = validate_row_required(row, line_num) + if err: + return err + return validaciones_clase( + row, + line_num, + valid_material_keys, + valid_uom_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_fraction_ame=valid_fraction_ame, + valid_product_codes_cp=valid_product_codes_cp, + ) diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/clients_and_providers/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py new file mode 100644 index 00000000..36641126 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for clients_and_providers) diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py new file mode 100644 index 00000000..42f41ae9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py @@ -0,0 +1,284 @@ +""" +Validadores reutilizables para import CSV de clientes y proveedores. +Paridad Clarion: procedencia E/N, tipo C/P/A, clave máx 8, SECON, Prosec, Vinculación, +Es Empresa Certificada, Transformador/SubMaq, desfase. +""" +from typing import Dict, Any, Optional + +from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum + +RFC_MAX = 30 +NAME_MAX = 256 +SHORT_NAME_MAX = 10 +# Clarion: Col C máx 8 caracteres +SHORT_NAME_MAX_CLARION = 8 +CURP_MAX = 19 + +# Valores permitidos Col Q (Tipo programa SECON) +TIPO_PROGRAMA_SECON_VALIDOS = frozenset( + {"IMMEX", "Maquila", "Pitex", "Ecex", "RECIME", "Pronex", "Ninguno"} +) + + +def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: + if not val or not str(val).strip(): + return None + s = str(val).strip() + v = s.lower() + # Una sola letra: C, P, A o B (Clarion: A=Ambos; B también usado como Ambos) + if len(v) == 1: + if v == "c": + return ClientOrProviderEnum.CLIENT + if v == "p": + return ClientOrProviderEnum.PROVIDER + if v in ("a", "b"): + return ClientOrProviderEnum.BOTH + if v in ("client", "cliente"): + return ClientOrProviderEnum.CLIENT + if v in ("provider", "proveedor"): + return ClientOrProviderEnum.PROVIDER + if v in ("both", "ambos"): + return ClientOrProviderEnum.BOTH + return None + + +def check_tipo_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + tipo_raw = (row.get("TIPO") or "").strip() + if not tipo_raw: + return None + if parse_client_or_provider(tipo_raw) is None: + return { + "line": line_num, + "col": "TIPO", + "msg": "Valor no válido. Use Cliente (C), Proveedor (P), Ambos (A) o dejar vacío.", + "solution": "Capturar una opción valida: C para Cliente, P para Proveedor, A para Ambos o dejar el campo vacio.", + } + return None + + +def check_procedencia( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Col A: E o N. Si PAIS (Col L) tiene valor: N → MEX, E → no MEX.""" + val = (row.get("PROCEDENCIA") or "").strip().upper() + if not val: + return None + if val not in ("E", "N"): + return { + "line": line_num, + "col": "PROCEDENCIA", + "msg": f"Error: (Col. A) El tipo de cliente: {val} es incorrecto.", + "solution": "Capturar en la columna A el Tipo de cliente correcto: E para Extranjero y N para Nacional.", + } + pais = (row.get("PAIS") or "").strip().upper() + if not pais: + return None + if val == "N" and pais != "MEX": + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. L) El tipo de cliente es Nacional y tiene el país {pais}, es incorrecto.", + "solution": "Capturar en la columna L el país con clave MEX, ya que es un cliente Nacional.", + } + if val == "E" and pais == "MEX": + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. L) El tipo de cliente es Extranjero y tiene el país MEX, es incorrecto.", + "solution": "Capturar en la columna L un país con clave diferente de MEX, ya que es un cliente Extranjero.", + } + return None + + +def check_short_name_max_clarion( + row: Dict[str, Any], line_num: int, max_len: int = SHORT_NAME_MAX_CLARION +) -> Optional[Dict[str, Any]]: + """Col C: Clave cliente/proveedor máx 8 caracteres (Clarion).""" + val = (row.get("SHORT_NAME") or "").strip() + if not val: + return None + if len(val) > max_len: + return { + "line": line_num, + "col": "SHORT_NAME", + "msg": f"Error: (Col. C) La Clave de Cliente/Proveedor supera la longitud de caracteres (máx {max_len}).", + "solution": f"Capturar en la columna C una Clave de Cliente/Proveedor de {max_len} caracteres como máximo.", + } + return None + + +def check_tipo_programa_secon( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Col Q: IMMEX, Maquila, Pitex, Ecex, RECIME, Pronex, Ninguno. Si no Ninguno → R y S obligatorios; si Ninguno → R y S vacíos.""" + q = (row.get("TIPO_PROGRAMA_SECON") or "").strip() + r = (row.get("NUM_PROGRAMA_SECON") or "").strip() + s = (row.get("FECHA_AUT_SECON") or "").strip() + if q and q not in TIPO_PROGRAMA_SECON_VALIDOS: + return { + "line": line_num, + "col": "TIPO_PROGRAMA_SECON", + "msg": f"Error: (Col. Q) El Tipo de Programa SECON: {q} es incorrecto.", + "solution": "Capturar los Tipos de Programa correctos: IMMEX, Maquila, Pitex, Ecex, RECIME, Pronex o Ninguno.", + } + if not q or q == "Ninguno": + if r: + return { + "line": line_num, + "col": "NUM_PROGRAMA_SECON", + "msg": "Error: (Col. R) El Tipo de Programa es Ninguno y está capturado el número de programa.", + "solution": "Borrar la información en la columna R del archivo o asignar un Programa en la columna Q.", + } + if s: + return { + "line": line_num, + "col": "FECHA_AUT_SECON", + "msg": "Error: (Col. S) El Tipo de Programa es Ninguno y está capturada la Fecha de autorización.", + "solution": "Borrar la información en la columna S del archivo o asignar un Programa en la columna Q.", + } + return None + # No es Ninguno: R y S obligatorios + if not r: + return { + "line": line_num, + "col": "NUM_PROGRAMA_SECON", + "msg": f"Error: (Col. R) El Tipo de Programa es: {q} y no está capturado el número de programa.", + "solution": "Capturarlo en la columna R del archivo.", + } + if not s: + return { + "line": line_num, + "col": "FECHA_AUT_SECON", + "msg": f"Error: (Col. S) El Tipo de Programa es: {q} y no está capturada la Fecha de autorización.", + "solution": "Capturarla en la columna S del archivo.", + } + return None + + +def check_es_prosec_num_aut(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col T (SI/NO): si SI → Col U obligatoria; si no SI → Col U vacía.""" + t = (row.get("ES_PROSEC") or "").strip().upper() + u = (row.get("NUM_AUT_PROSEC") or "").strip() + if t and t not in ("SI", "NO"): + return { + "line": line_num, + "col": "ES_PROSEC", + "msg": f"Error: (Col. T) La opción de si Es Prosec? {t} no es valida.", + "solution": "Capturar una opción valida: SI, NO, o dejar el campo vacio (se asigna NO).", + } + if t == "SI" and not u: + return { + "line": line_num, + "col": "NUM_AUT_PROSEC", + "msg": "Error: (Col. U) Es Prosec? es SI y no está capturado el número de permiso.", + "solution": "Capturar en la columna U el número de Permiso PROSEC o cambiar la opcion a NO en la columna T.", + } + if t != "SI" and u: + return { + "line": line_num, + "col": "NUM_AUT_PROSEC", + "msg": "Error: (Col. U) Es Prosec? no es SI y está capturado el número de permiso.", + "solution": "Borrar la información de la columna U o cambiar la opcion a SI en la columna T.", + } + return None + + +def check_vinculacion(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col V: solo 0, 1 o 2 (o vacío → 0).""" + val = (row.get("VINCULACION") or "").strip() + if not val: + return None + if val not in ("0", "1", "2"): + return { + "line": line_num, + "col": "VINCULACION", + "msg": f"Error: (Col. V) La opción de Vinculación: {val} no es valida.", + "solution": "Capturar una opción valida: 0, 1, 2 o dejar el campo vacio (se asigna 0).", + } + return None + + +def check_es_empresa_certificada_registro(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col W (SI/NO): si SI → Col X obligatoria; si no SI → Col X vacía.""" + w = (row.get("ES_EMPRESA_CERTIFICADA") or "").strip().upper() + x = (row.get("REGISTRO_EMPRESA_CERT") or "").strip() + if w and w not in ("SI", "NO"): + return { + "line": line_num, + "col": "ES_EMPRESA_CERTIFICADA", + "msg": f"Error: (Col. W) La opción de si Es Empresa Certificada? {w} no es valida.", + "solution": "Capturar una opción valida: SI, NO, o dejar el campo vacio (se asigna NO).", + } + if w == "SI" and not x: + return { + "line": line_num, + "col": "REGISTRO_EMPRESA_CERT", + "msg": "Error: (Col. X) Es Empresa Certificada? es SI y no está capturado el número de empresa certificada.", + "solution": "Capturar en la columna X el número de Empresa Certificada o cambiar la opción a NO en la columna W.", + } + if w != "SI" and x: + return { + "line": line_num, + "col": "REGISTRO_EMPRESA_CERT", + "msg": "Error: (Col. X) Es Empresa Certificada? no es SI y está capturado el número de empresa certificada.", + "solution": "Borrar la información de la columna X o cambiar la opción a SI en la columna W.", + } + return None + + +def check_transformador_submaq(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col AF: primera letra T, S o N (Transformador, SubMaquila, Ninguno) o vacío → Ninguno.""" + val = (row.get("TRANSFORMA_SUBMAQ") or "").strip().upper() + if not val: + return None + first = val[:1] if val else "" + if first not in ("T", "S", "N"): + return { + "line": line_num, + "col": "TRANSFORMA_SUBMAQ", + "msg": f"Error: (Col. AF) La opción (SUBMAQUILA/TRANSFORMADOR/NINGUNO): {val} no es valida.", + "solution": "Capturar una opción valida: Transformador, SubMaquila, Ninguno o dejar el campo vacio (se asigna Ninguno).", + } + return None + + +def check_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA (Col AH) tiene valor → advertencia de desfase.""" + val = (row.get("COL_EXTRA") or "").strip() + if not val: + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": "Advertencia: Podría existir un desfase en esta línea.", + "solution": "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta.", + } + + +def parse_active(val: Optional[str]) -> bool: + if not val or not str(val).strip(): + return True + v = str(val).strip().lower() + if v in ("1", "true", "si", "sí", "yes", "s", "x"): + return True + if v in ("0", "false", "no", "n"): + return False + return True diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py new file mode 100644 index 00000000..c281f814 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py @@ -0,0 +1,200 @@ +""" +Mapeo fila CSV → datos para ClientProvider, ClientProviderAddress y ClientProviderPrograms. +Paridad Clarion: columnas A–AG a modelos. + +El CSV se alimenta en base a las tablas/modelos: +- cp_data: atributos de ClientProvider (clients_and_providers). Claves = nombres de columna del modelo. +- address_data: atributos de ClientProviderAddress (clients_and_providers_address). Se asignan en tasks. +- programs_data: atributos de ClientProviderPrograms (clients_and_providers_programs). Se asignan en tasks. + +Las validaciones (validators) aplican reglas de negocio Clarion y respetan longitudes máximas de los modelos. +""" +from datetime import datetime +from decimal import Decimal +from typing import Dict, Any, Optional, Tuple + +from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum + +from .common_validators import ( + parse_client_or_provider, + parse_active, +) + +MAX_LEN = { + "rfc": 30, + "name": 256, + "short_name": 10, + "curp": 19, + "responsible": 80, + "position": 30, + "incoterm": 19, + "email": 100, + "phone": 30, + "address": 100, + "postal_code": 15, + "city": 30, + "state": 30, + "country": 3, + "contact": 50, + "extra_information": 399, + "web_key": 40, + "program": 7, + "program_number": 40, + "prosec_authorization": 20, + "secon_authorization": 20, + "manufacturer_id": 25, + "tax_id_programs": 30, + "broker": 6, + "import_broker": 6, + "transfer_key": 8, + "certified_company_registry": 40, + "neighborhood": 40, + "exterior_number": 20, + "fax": 30, +} + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _parse_date_to_yyyymmdd(val: Any) -> Optional[int]: + """Convierte fecha dd/mm/yyyy o similar a entero YYYYMMDD para secon_auth_date.""" + if not val or not str(val).strip(): + return None + s = str(val).strip() + for fmt in ("%d/%m/%Y", "%Y-%m-%d", "%d-%m-%Y", "%Y/%m/%d"): + try: + d = datetime.strptime(s[:10], fmt) + return d.year * 10000 + d.month * 100 + d.day + except ValueError: + continue + return None + + +def row_to_client_provider_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int +) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: + """ + Mapea fila normalizada a datos para ClientProvider, ClientProviderAddress y ClientProviderPrograms. + Devuelve (cp_data, address_data_or_none, programs_data_or_none). + Para compatibilidad con Clarion: se requiere RFC o SHORT_NAME para considerar la fila válida. + """ + rfc = _str_or_none(row_norm.get("RFC"), MAX_LEN["rfc"]) + short_name = _str_or_none(row_norm.get("SHORT_NAME"), MAX_LEN["short_name"]) + if not rfc and not short_name: + return ({}, None, None) + + client_or_provider = parse_client_or_provider(row_norm.get("TIPO")) or ClientOrProviderEnum.BOTH + procedencia = _str_or_none(row_norm.get("PROCEDENCIA"), 1) + if procedencia: + procedencia = procedencia.upper()[:1] + + # Vinculación 0/1/2 → string + vinc = (row_norm.get("VINCULACION") or "").strip() + linking = None + if vinc in ("0", "1", "2"): + linking = vinc + + # Transformador/SubMaquila: primera letra T/S/N + trans = (row_norm.get("TRANSFORMA_SUBMAQ") or "").strip().upper()[:1] + transform_subassembly = trans if trans in ("T", "S", "N") else None + + cp_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "rfc": rfc or None, + "name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]), + "short_name": short_name, + "curp": _str_or_none(row_norm.get("CURP"), MAX_LEN["curp"]), + "client_or_provider": client_or_provider, + "type_nat_foreign": procedencia, + "linking": linking, + "transform_subassembly": transform_subassembly, + "extra_information": _str_or_none(row_norm.get("INFORMACION_EXTRA"), MAX_LEN["extra_information"]), + "web_key": _str_or_none(row_norm.get("CLAVE_WEB"), MAX_LEN["web_key"]), + "responsible": _str_or_none(row_norm.get("RESPONSABLE"), MAX_LEN["responsible"]), + "position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]), + "incoterm": _str_or_none(row_norm.get("INCOTERM"), MAX_LEN["incoterm"]), + "is_active": parse_active(row_norm.get("ACTIVO")), + "is_national_provider": True if procedencia == "N" else (False if procedencia == "E" else None), + } + + # Address + email = _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]) + phone = _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]) + address_str = _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]) + address_data = None + if email or phone or address_str or _str_or_none(row_norm.get("CODIGO POSTAL")) or _str_or_none(row_norm.get("COLONIA")) or _str_or_none(row_norm.get("NUM_EXT")): + address_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "streets": address_str, + "exterior_number": _str_or_none(row_norm.get("NUM_EXT"), MAX_LEN["exterior_number"]), + "neighborhood": _str_or_none(row_norm.get("COLONIA"), MAX_LEN["neighborhood"]), + "postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "phone": phone, + "fax_number": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]), + "email": email, + "contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]), + } + + # Programs (SECON, PROSEC, empresa certificada, etc.) + tipo_prog = _str_or_none(row_norm.get("TIPO_PROGRAMA_SECON"), MAX_LEN["program"]) + if tipo_prog and tipo_prog.lower() == "ninguno": + tipo_prog = None + num_prog = _str_or_none(row_norm.get("NUM_PROGRAMA_SECON"), MAX_LEN["program_number"]) + fecha_secon = _parse_date_to_yyyymmdd(row_norm.get("FECHA_AUT_SECON")) + es_prosec = (row_norm.get("ES_PROSEC") or "").strip().upper() + prosec_val = "1" if es_prosec == "SI" else ("0" if es_prosec else None) + num_aut_prosec = _str_or_none(row_norm.get("NUM_AUT_PROSEC"), MAX_LEN["prosec_authorization"]) + es_cert = (row_norm.get("ES_EMPRESA_CERTIFICADA") or "").strip().upper() + is_certified = "S" if es_cert == "SI" else ("N" if es_cert else None) + reg_cert = _str_or_none(row_norm.get("REGISTRO_EMPRESA_CERT"), MAX_LEN["certified_company_registry"]) + vinc_prop = row_norm.get("VINCULACION") + applied_proportion = None + if vinc_prop is not None and str(vinc_prop).strip() in ("0", "1", "2"): + try: + applied_proportion = Decimal(str(vinc_prop).strip()) + except Exception: + pass + + programs_data = None + if ( + tipo_prog or num_prog or fecha_secon is not None or prosec_val or num_aut_prosec + or is_certified or reg_cert + or _str_or_none(row_norm.get("MANUFACTURER_ID")) + or _str_or_none(row_norm.get("TAX_ID_PROGRAMS")) + or _str_or_none(row_norm.get("BROKER_EXPO")) + or _str_or_none(row_norm.get("BROKER_IMPO")) + or _str_or_none(row_norm.get("CLAVE_TRANSFER")) + or applied_proportion is not None + ): + programs_data = { + "program": tipo_prog[:7] if tipo_prog else None, + "program_number": num_prog, + "secon_authorization": num_prog, + "secon_auth_date": fecha_secon, + "prosec": prosec_val, + "prosec_authorization": num_aut_prosec, + "is_certified_company": is_certified, + "certified_company_registry": reg_cert, + "manufacturer_id": _str_or_none(row_norm.get("MANUFACTURER_ID"), MAX_LEN["manufacturer_id"]), + "tax_id": _str_or_none(row_norm.get("TAX_ID_PROGRAMS"), MAX_LEN["tax_id_programs"]), + "broker": _str_or_none(row_norm.get("BROKER_EXPO"), MAX_LEN["broker"]), + "import_broker": _str_or_none(row_norm.get("BROKER_IMPO"), MAX_LEN["import_broker"]), + "transfer_key": _str_or_none(row_norm.get("CLAVE_TRANSFER"), MAX_LEN["transfer_key"]), + "applied_proportion": applied_proportion, + } + + return (cp_data, address_data, programs_data) diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py similarity index 98% rename from backend/api/v1/modules/a76/clients_and_providers/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py index 68029417..6795787f 100644 --- a/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py @@ -14,6 +14,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -81,7 +82,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/clients_and_providers/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py new file mode 100644 index 00000000..603924e7 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py @@ -0,0 +1,423 @@ +""" +Tareas Celery para importación CSV de Clientes y Proveedores. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import json +import logging +import os +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template +from .validators import validate_row_client_provider +from .common.mappers import row_to_client_provider_data + +logger = logging.getLogger(__name__) + +JOB_TYPE = "cp" + +# Para routes.py +CP_IMPORT_FILE_PREFIX = "cp_import_file:" +CP_IMPORT_META_PREFIX = "cp_import_meta:" +CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:" +CP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv_reader.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_short_names: Set[str] = set() + if actualizar: + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + if (cp.short_name or "").strip(): + existing_short_names.add((cp.short_name or "").strip()) + except Exception as e: + logger.warning("CP import: could not load existing short_names for ACT: %s", e) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_client_provider( + row_norm, i, + actualizar=actualizar, + existing_short_names=existing_short_names if actualizar else None, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("CP import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("CP import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_short_names: Set[str] = set() + if actualizar: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + try: + with CoreSessionLocal() as session: + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + if (cp.short_name or "").strip(): + existing_short_names.add((cp.short_name or "").strip()) + except Exception as e: + logger.warning("CP commit: could not load existing short_names for ACT: %s", e) + + from api.v1.modules.a76.clients_and_providers.models import ( + ClientProvider, + ClientProviderAddress, + ClientProviderPrograms, + ) + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + existing_by_rfc: Dict[str, ClientProvider] = {} + existing_by_short_name: Dict[str, ClientProvider] = {} + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + rfc_key = (cp.rfc or "").strip() + if rfc_key: + existing_by_rfc[rfc_key] = cp + sn_key = (cp.short_name or "").strip() + if sn_key: + existing_by_short_name[sn_key] = cp + + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_client_provider( + row_norm, i, + actualizar=actualizar, + existing_short_names=existing_short_names if actualizar else None, + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + cp_data, address_data, programs_data = row_to_client_provider_data(row_norm, tenant_id, company_id) + if not cp_data: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "Fila sin RFC ni Clave"}) + continue + if actualizar: + short_name_key = (cp_data.get("short_name") or "").strip() + if not short_name_key: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "Clave (SHORT_NAME) requerida en modo Actualizar"}) + continue + existing = existing_by_short_name.get(short_name_key) + if not existing: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "Clave no existe en catálogo"}) + continue + # Merge: fill from existing when csv value is empty + for k, v in cp_data.items(): + if k in ("tenant_id", "company_id"): + continue + if v is None or (isinstance(v, str) and not v.strip()): + existing_val = getattr(existing, k, None) + if existing_val is not None: + cp_data[k] = existing_val + for k, v in cp_data.items(): + if k not in ("tenant_id", "company_id"): + setattr(existing, k, v) + session.add(existing) + updated_count += 1 + # Update address if present + if address_data and existing.address: + addr = existing.address + for k, v in address_data.items(): + if k not in ("tenant_id", "company_id") and v is not None: + setattr(addr, k, v) + session.add(addr) + elif address_data: + addr = ClientProviderAddress( + client_id=existing.id, + tenant_id=tenant_id, + company_id=company_id, + streets=address_data.get("streets"), + postal_code=address_data.get("postal_code"), + city=address_data.get("city"), + state=address_data.get("state"), + country=address_data.get("country"), + phone=address_data.get("phone"), + email=address_data.get("email"), + contact=address_data.get("contact"), + exterior_number=address_data.get("exterior_number"), + neighborhood=address_data.get("neighborhood"), + fax_number=address_data.get("fax_number"), + ) + session.add(addr) + # Update or create programs + if programs_data: + prog = session.query(ClientProviderPrograms).filter( + ClientProviderPrograms.client_id == existing.id, + ).first() + if prog: + for k, v in programs_data.items(): + if v is not None: + setattr(prog, k, v) + session.add(prog) + else: + prog = ClientProviderPrograms( + client_id=existing.id, + tenant_id=tenant_id, + company_id=company_id, + **{k: v for k, v in programs_data.items() if v is not None}, + ) + session.add(prog) + else: + # Alta / Reemplazar: key por RFC + if not cp_data.get("rfc"): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "RFC requerido"}) + continue + rfc = cp_data["rfc"] + existing = existing_by_rfc.get(rfc) + if existing: + for k, v in cp_data.items(): + if k not in ("tenant_id", "company_id", "rfc"): + setattr(existing, k, v) + session.add(existing) + updated_count += 1 + if address_data and existing.address: + addr = existing.address + for k, v in address_data.items(): + if k not in ("tenant_id", "company_id") and v is not None: + setattr(addr, k, v) + session.add(addr) + elif address_data: + addr = ClientProviderAddress( + client_id=existing.id, + tenant_id=tenant_id, + company_id=company_id, + streets=address_data.get("streets"), + postal_code=address_data.get("postal_code"), + city=address_data.get("city"), + state=address_data.get("state"), + country=address_data.get("country"), + phone=address_data.get("phone"), + email=address_data.get("email"), + contact=address_data.get("contact"), + exterior_number=address_data.get("exterior_number"), + neighborhood=address_data.get("neighborhood"), + fax_number=address_data.get("fax_number"), + ) + session.add(addr) + if programs_data: + prog = session.query(ClientProviderPrograms).filter( + ClientProviderPrograms.client_id == existing.id, + ).first() + if prog: + for k, v in programs_data.items(): + if v is not None: + setattr(prog, k, v) + session.add(prog) + else: + prog = ClientProviderPrograms( + client_id=existing.id, + tenant_id=tenant_id, + company_id=company_id, + **{k: v for k, v in programs_data.items() if v is not None}, + ) + session.add(prog) + else: + new_cp = ClientProvider(**cp_data) + session.add(new_cp) + session.flush() + existing_by_rfc[rfc] = new_cp + if (new_cp.short_name or "").strip(): + existing_by_short_name[(new_cp.short_name or "").strip()] = new_cp + inserted_count += 1 + if address_data: + addr = ClientProviderAddress( + client_id=new_cp.id, + tenant_id=tenant_id, + company_id=company_id, + streets=address_data.get("streets"), + postal_code=address_data.get("postal_code"), + city=address_data.get("city"), + state=address_data.get("state"), + country=address_data.get("country"), + phone=address_data.get("phone"), + email=address_data.get("email"), + contact=address_data.get("contact"), + exterior_number=address_data.get("exterior_number"), + neighborhood=address_data.get("neighborhood"), + fax_number=address_data.get("fax_number"), + ) + session.add(addr) + if programs_data: + prog = ClientProviderPrograms( + client_id=new_cp.id, + tenant_id=tenant_id, + company_id=company_id, + **{k: v for k, v in programs_data.items() if v is not None}, + ) + session.add(prog) + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("CP import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("CP import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if inserted_count == 0 and updated_count == 0 and skipped_invalid > 0: + return { + "status": "warning", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron ni actualizaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0 and updated_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("CP import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/template_config.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/template_config.py new file mode 100644 index 00000000..be387efc --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/template_config.py @@ -0,0 +1,118 @@ +""" +Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls). +Layout Clarion: Col A = PROCEDENCIA (E/N), B = TIPO (C/P/A), C = CLAVE (máx 8), D = NOMBRE, E = RFC, F–AG. +Solo se leen columnas definidas aquí; el resto se ignora. +Correspondencia Clarion → canonical: A=PROCEDENCIA, B=TIPO, C=SHORT_NAME, D=NOMBRE, E=RFC, F=CALLES(DIRECCION), +G=NUM_EXT, H=CODIGO POSTAL, I=COLONIA, J=CIUDAD, K=ESTADO, L=PAIS, M=TELEFONO, N=FAX, O=EMAIL, P=CURP, +Q=TIPO_PROGRAMA_SECON, R=NUM_PROGRAMA_SECON, S=FECHA_AUT_SECON, T=ES_PROSEC, U=NUM_AUT_PROSEC, V=VINCULACION, +W=ES_EMPRESA_CERTIFICADA, X=REGISTRO_EMPRESA_CERT, Y=INFORMACION_EXTRA, Z=CONTACTO, AA=MANUFACTURER_ID, +AB=TAX_ID_PROGRAMS, AC=BROKER_EXPO, AD=BROKER_IMPO, AE=CLAVE_TRANSFER, AF=TRANSFORMA_SUBMAQ, AG=CLAVE_WEB, +AH=COL_EXTRA (desfase). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "client_providers": [ + # Col A - Procedencia (E=Extranjero, N=Nacional) + {"canonical": "PROCEDENCIA", "aliases": ["TIPO PROCEDENCIA", "EXTranjero/Nacional", "E/N"]}, + # Col B - Tipo Cliente (C/P/A) + {"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]}, + # Col C - Clave cliente/proveedor (máx 8 Clarion) + {"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]}, + # Col D - Nombre + {"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]}, + # Col E - RFC + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]}, + # Col F - Calles + {"canonical": "DIRECCION", "aliases": ["CALLES", "DOMICILIO", "DIRECCION FISCAL", "CALLE"]}, + # Col G - Número exterior + {"canonical": "NUM_EXT", "aliases": ["NUM EXTERIOR", "NUMERO EXTERIOR", "NO EXT"]}, + # Col H - Código postal + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + # Col I - Colonia + {"canonical": "COLONIA", "aliases": []}, + # Col J - Ciudad + {"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]}, + # Col K - Estado + {"canonical": "ESTADO", "aliases": []}, + # Col L - País M3 + {"canonical": "PAIS", "aliases": ["COUNTRY", "PAIS M3"]}, + # Col M - Teléfono + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]}, + # Col N - Fax + {"canonical": "FAX", "aliases": ["NUMERO DE FAX", "FAX NUMBER"]}, + # Col O - Email + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]}, + # Col P - CURP + {"canonical": "CURP", "aliases": []}, + # Col Q - Tipo programa SECON + {"canonical": "TIPO_PROGRAMA_SECON", "aliases": ["PROGRAMA", "TIPO PROGRAMA SECON", "TIPO PROGRAMA"]}, + # Col R - Número programa SECON + {"canonical": "NUM_PROGRAMA_SECON", "aliases": ["NUM PROGRAMA SECON", "NUMERO PROGRAMA"]}, + # Col S - Fecha autorización SECON + {"canonical": "FECHA_AUT_SECON", "aliases": ["FECHA AUT SECON", "FECHA AUTORIZACION SECON"]}, + # Col T - Es Prosec? + {"canonical": "ES_PROSEC", "aliases": ["ES PROSEC", "PROSEC"]}, + # Col U - Número autorización PROSEC + {"canonical": "NUM_AUT_PROSEC", "aliases": ["NUM AUT PROSEC", "NUMERO AUTORIZACION PROSEC"]}, + # Col V - Vinculación (0/1/2) + {"canonical": "VINCULACION", "aliases": []}, + # Col W - Es Empresa Certificada? + {"canonical": "ES_EMPRESA_CERTIFICADA", "aliases": ["ES EMPRESA CERTIFICADA", "EMPRESA CERTIFICADA"]}, + # Col X - Registro empresa certificada + {"canonical": "REGISTRO_EMPRESA_CERT", "aliases": ["REGISTRO EMPRESA CERTIFICADA", "REGISTRO EMP CERT"]}, + # Col Y - Información extra + {"canonical": "INFORMACION_EXTRA", "aliases": ["INFORMACION EXTRA", "INFO EXTRA"]}, + # Col Z - Contacto + {"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]}, + # Col AA - Manufacturer ID + {"canonical": "MANUFACTURER_ID", "aliases": ["MANUFACTURERID", "MANUFACTURER ID"]}, + # Col AB - Tax ID (programas) + {"canonical": "TAX_ID_PROGRAMS", "aliases": ["TAX ID", "TAXID PROGRAMS"]}, + # Col AC - Broker exportación + {"canonical": "BROKER_EXPO", "aliases": ["BROKER EXPO", "BROKER EXPORTACION"]}, + # Col AD - Broker importación + {"canonical": "BROKER_IMPO", "aliases": ["BROKER IMPO", "BROKER IMPORTACION"]}, + # Col AE - Clave transfer + {"canonical": "CLAVE_TRANSFER", "aliases": ["CLAVE TRANSFER", "TRANSFER KEY"]}, + # Col AF - Transformador/SubMaquila/Ninguno (T/S/N) + {"canonical": "TRANSFORMA_SUBMAQ", "aliases": ["TRANSFORMADOR SUBMAQUILA", "TRASFORMA SUBMAQ"]}, + # Col AG - Clave interface web + {"canonical": "CLAVE_WEB", "aliases": ["CLAVE WEB", "CLAVE INTERFACE WEB", "WEB KEY"]}, + # Col AH - Desfase (si tiene valor → advertencia) + {"canonical": "COL_EXTRA", "aliases": ["DESFASE", "COLUMNA EXTRA"]}, + # Legacy / otros + {"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "INCOTERM", "aliases": []}, + {"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla client_providers.""" + cols = TEMPLATE_COLUMNS.get("client_providers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py new file mode 100644 index 00000000..895b3327 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_client_provider + +__all__ = ["validate_row_client_provider"] diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py new file mode 100644 index 00000000..12b311a6 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py @@ -0,0 +1,177 @@ +""" +Validaciones comunes de fila para import CSV de clientes y proveedores. +Paridad Clarion: VALIDA_TODA_CLIENTE_O_PROV, VALIDA_PARCIAL_CLIENTE_O_PROV, VALIDACIONES_CLIENTE_O_PROV. +Origen de reglas: código legacy Clarion (EstructuraCatClienteProv). +Mapa columnas: A=PROCEDENCIA, B=TIPO, C=SHORT_NAME, D=NOMBRE, E=RFC, F–AG (ver template_config). +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + RFC_MAX, + NAME_MAX, + SHORT_NAME_MAX_CLARION, + CURP_MAX, + check_required_max, + check_max_length, + check_tipo_client_provider, + check_procedencia, + check_short_name_max_clarion, + check_tipo_programa_secon, + check_es_prosec_num_aut, + check_vinculacion, + check_es_empresa_certificada_registro, + check_transformador_submaq, + check_desfase, +) + + +# --- Mensajes obligatorios (VALIDA_TODA) --- +MSG_CAMPOS_OBLIGATORIOS = "Existen campos vacíos que son obligatorios: {campos}." +MSG_CAMPOS_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta." +MSG_CLAVE_VACIA = ( + "Error: La Clave de Cliente/Proveedor está vacía y no se pueden hacer las validaciones." +) +MSG_CLAVE_VACIA_SOLUCION = ( + "Capturar una Clave de Cliente/Proveedor nueva o existente a la cual desee agregar, remplazar o actualizar campos." +) +MSG_CLAVE_NO_EXISTE = "Error: (Col.C) Clave de Proveedor/Cliente no existe." +MSG_CLAVE_NO_EXISTE_SOLUCION = "La clave debe existir en el catálogo cuando el modo es Actualizar." + + +def validate_row_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA tiene valor → advertencia de desfase.""" + return check_desfase(row, line_num) + + +def validate_row_clave_vacia(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Clave (SHORT_NAME) vacía → error inmediato.""" + val = (row.get("SHORT_NAME") or "").strip() + if not val: + return { + "line": line_num, + "col": "SHORT_NAME", + "msg": MSG_CLAVE_VACIA, + "solution": MSG_CLAVE_VACIA_SOLUCION, + } + return None + + +def validaciones_cliente_o_prov(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + VALIDACIONES_CLIENTE_O_PROV: reglas de dominio compartidas. + Col A (E/N + coherencia con L), B (C/P/A), C (máx 8), Q/R/S, T/U, V, W/X, AF. + """ + err = check_procedencia(row, line_num) + if err: + return err + err = check_tipo_client_provider(row, line_num) + if err: + return err + err = check_short_name_max_clarion(row, line_num) + if err: + return err + err = check_tipo_programa_secon(row, line_num) + if err: + return err + err = check_es_prosec_num_aut(row, line_num) + if err: + return err + err = check_vinculacion(row, line_num) + if err: + return err + err = check_es_empresa_certificada_registro(row, line_num) + if err: + return err + err = check_transformador_submaq(row, line_num) + if err: + return err + return None + + +def valida_toda_cliente_o_prov( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, +) -> Optional[Dict[str, Any]]: + """ + VALIDA_TODA: obligatorios Col A (Procedencia), Col D (Nombre) cuando no es ACT. + Si modo ACT y clave no existe → error se devuelve antes (en validate_row_client_provider). + Luego ejecuta VALIDACIONES_CLIENTE_O_PROV. + """ + campos_oblig = [] + # Col A - Procedencia obligatoria + if not (row.get("PROCEDENCIA") or "").strip(): + campos_oblig.append("(Col.A) Tipo Cliente Procedencia (E/N)") + # Col D - Nombre obligatorio solo cuando no es actualizar + if not actualizar and not (row.get("NOMBRE") or "").strip(): + campos_oblig.append("(Col.D) Nombre") + if campos_oblig: + return { + "line": line_num, + "col": "PROCEDENCIA" if not (row.get("PROCEDENCIA") or "").strip() else "NOMBRE", + "msg": MSG_CAMPOS_OBLIGATORIOS.format(campos=", ".join(campos_oblig)), + "solution": MSG_CAMPOS_OBLIGATORIOS_SOLUCION, + } + return validaciones_cliente_o_prov(row, line_num) + + +def valida_parcial_cliente_o_prov(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """VALIDA_PARCIAL: solo VALIDACIONES_CLIENTE_O_PROV (no exige A ni D).""" + return validaciones_cliente_o_prov(row, line_num) + + +def validate_row_client_provider( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, + existing_short_names: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de clientes y proveedores. + - Desfase (COL_EXTRA) primero. + - Clave vacía → error. + - Si actualizar y clave no existe en existing_short_names → error "Clave no existe". + - Si actualizar y clave existe → VALIDA_PARCIAL (solo validaciones comunes). + - Si no actualizar o clave no existe → VALIDA_TODA (A y D obligatorios cuando aplique, luego comunes). + Además se validan RFC (requerido max 30), longitudes NOMBRE/SHORT_NAME/CURP para compatibilidad. + """ + err = validate_row_desfase(row, line_num) + if err: + return err + err = validate_row_clave_vacia(row, line_num) + if err: + return err + + short_name = (row.get("SHORT_NAME") or "").strip() + existing = existing_short_names or set() + use_partial = actualizar and short_name in existing + + if actualizar and short_name and short_name not in existing: + return { + "line": line_num, + "col": "SHORT_NAME", + "msg": MSG_CLAVE_NO_EXISTE, + "solution": MSG_CLAVE_NO_EXISTE_SOLUCION, + } + + if use_partial: + err = valida_parcial_cliente_o_prov(row, line_num) + else: + err = valida_toda_cliente_o_prov(row, line_num, actualizar=actualizar) + if err: + return err + + # Compatibilidad: RFC requerido y longitudes (como antes) + err = check_required_max(row, "RFC", RFC_MAX, line_num) + if err: + return err + err = check_max_length(row, "NOMBRE", NAME_MAX, line_num) + if err: + return err + err = check_max_length(row, "SHORT_NAME", 10, line_num) # modelo permite 10 + if err: + return err + err = check_max_length(row, "CURP", CURP_MAX, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py new file mode 100644 index 00000000..734e7f8b --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py @@ -0,0 +1,8 @@ +""" +Punto de entrada de validación para import de una fila cliente/proveedor. +""" +from typing import Dict, Any, Optional + +from .common import validate_row_client_provider + +__all__ = ["validate_row_client_provider"] diff --git a/backend/api/v1/modules/a76/layouts_csv/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/common/__init__.py new file mode 100644 index 00000000..d5f02bc1 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/__init__.py @@ -0,0 +1 @@ +# Shared utilities for layouts_csv imports (storage, normalize, csv, meta, responses) diff --git a/backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py b/backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py new file mode 100644 index 00000000..24359188 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py @@ -0,0 +1,96 @@ +""" +Lectura de CSV con detección de delimitador (compartida por layouts_csv). +Si se pasa fieldnames, no se usa la primera fila como cabecera y se toma como dato (CSV sin cabeceras). +Si se pasa headerless_first_cell_values, se detecta si la primera fila es cabecera o dato por el valor de la primera celda. +""" +import csv +import io +from typing import Iterator, Tuple, Dict, Any, Optional, List, Set + + +def _normalize_empty_headers(headers: List[str]) -> List[str]: + """Sustituye cabeceras vacías por _COL_0_, _COL_1_, ... para que DictReader no colapse columnas.""" + result: List[str] = [] + empty_idx = 0 + for h in headers: + if (h or "").strip() == "": + result.append(f"_COL_{empty_idx}_") + empty_idx += 1 + else: + result.append(h) + return result + + +def iter_csv_rows( + file_path: str, + fieldnames: Optional[List[str]] = None, + headerless_first_cell_values: Optional[Set[str]] = None, + headerless_second_cell_key_pattern: Optional[str] = None, +) -> Iterator[Tuple[int, Dict[str, Any]]]: + """ + Abre el CSV, detecta dialecto y devuelve (line_num, row_dict) por cada fila. + line_num empieza en 1 (primera fila de datos). + Si fieldnames y headerless_first_cell_values se pasan: si la primera celda de la primera fila + (quitando BOM, strip, upper) está en headerless_first_cell_values, se trata como dato y se usan fieldnames. + headerless_second_cell_key_pattern se ignora si no se usa (reservado para otros layouts). + """ + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + if fieldnames and headerless_first_cell_values is not None: + first_line = f.readline() + if not first_line: + return + row_reader = csv.reader(io.StringIO(first_line), dialect=dialect) + first_cells = next(row_reader, None) + if not first_cells: + return + first_cell_clean = (first_cells[0] or "").lstrip("\ufeff").strip().upper() + use_headerless = first_cell_clean in headerless_first_cell_values + if use_headerless: + pad = len(fieldnames) - len(first_cells) + cells = first_cells[: len(fieldnames)] + ([""] * pad if pad > 0 else []) + yield 1, dict(zip(fieldnames, cells)) + reader = csv.DictReader(f, fieldnames=fieldnames, dialect=dialect, restval="") + for i, row in enumerate(reader, start=2): + yield i, dict(row) + return + f.seek(0) + first_line = f.readline() + if not first_line: + return + row_reader = csv.reader(io.StringIO(first_line), dialect=dialect) + raw_headers = next(row_reader, None) + if not raw_headers: + return + normalized = _normalize_empty_headers(raw_headers) + reader = csv.DictReader(f, fieldnames=normalized, dialect=dialect, restval="") + for i, row in enumerate(reader, start=1): + yield i, row + elif fieldnames: + reader = csv.DictReader(f, fieldnames=fieldnames, dialect=dialect) + for i, row in enumerate(reader, start=1): + yield i, dict(row) + else: + first_line = f.readline() + if not first_line: + return + row_reader = csv.reader(io.StringIO(first_line), dialect=dialect) + raw_headers = next(row_reader, None) + if not raw_headers: + return + normalized = _normalize_empty_headers(raw_headers) + reader = csv.DictReader(f, fieldnames=normalized, dialect=dialect, restval="") + for i, row in enumerate(reader, start=1): + yield i, row + + +def count_csv_rows(file_path: str, has_header: bool = True) -> int: + """Cuenta filas del CSV. Si has_header=True (por defecto), no cuenta la cabecera.""" + with open(file_path, "r", encoding="utf-8-sig") as f: + total_lines = sum(1 for _ in f) + return total_lines if not has_header else max(0, total_lines - 1) diff --git a/backend/api/v1/modules/a76/layouts_csv/common/meta.py b/backend/api/v1/modules/a76/layouts_csv/common/meta.py new file mode 100644 index 00000000..fc68d96f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/meta.py @@ -0,0 +1,35 @@ +""" +Carga y guardado de meta (tenant_id, company_id) para imports CSV. +""" +import json +import os +from typing import Dict, Any, Tuple + + +def load_meta(file_path: str) -> Dict[str, Any]: + """Carga meta desde archivo .meta.json asociado al CSV. Devuelve dict vacío si no existe o falla.""" + meta_path = file_path.replace(".csv", ".meta.json") + if not os.path.exists(meta_path): + return {} + try: + with open(meta_path, "r", encoding="utf-8") as f: + return json.load(f) or {} + except Exception: + return {} + + +def require_tenant_context(file_path: str) -> Tuple[int, int]: + """ + Obtiene tenant_id y company_id del meta. Lanza ValueError si faltan. + """ + meta = load_meta(file_path) + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + raise ValueError("Falta contexto (tenant/company)") + return int(tenant_id), int(company_id) + + +def get_meta_path(file_path: str) -> str: + """Ruta del archivo .meta.json para un CSV.""" + return file_path.replace(".csv", ".meta.json") diff --git a/backend/api/v1/modules/a76/layouts_csv/common/normalize.py b/backend/api/v1/modules/a76/layouts_csv/common/normalize.py new file mode 100644 index 00000000..6af81dbb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/normalize.py @@ -0,0 +1,16 @@ +""" +Normalización de cabeceras CSV (compartida por todos los módulos layouts_csv). +""" +import re +import unicodedata +from typing import Optional + + +def normalize_header(name: Optional[str]) -> str: + """Normaliza nombre de columna: NFKD, mayúsculas, sin acentos, espacios colapsados.""" + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() diff --git a/backend/api/v1/modules/a76/layouts_csv/common/responses.py b/backend/api/v1/modules/a76/layouts_csv/common/responses.py new file mode 100644 index 00000000..af33f9fb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/responses.py @@ -0,0 +1,55 @@ +""" +Helpers para construir respuestas de scan y commit (formato unificado). +""" +from typing import Dict, Any, List, Optional + + +def scan_result( + job_id: str, + processed_rows: int, + error_count: int, + errors_detail: List[Dict[str, Any]], + total_rows_in_file: Optional[int] = None, + message: Optional[str] = None, +) -> Dict[str, Any]: + """Respuesta de scan_file (waiting_confirmation). + Si total_rows_in_file no se pasa, se usa processed_rows como total (comportamiento anterior). + """ + total = total_rows_in_file if total_rows_in_file is not None else processed_rows + out: Dict[str, Any] = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": total, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + if message: + out["message"] = message + return out + + +def commit_result( + status: str, + inserted: int, + skipped_invalid: int, + skipped_missing_fk: int, + skipped_duplicate: int, + skipped_details: List[Dict[str, Any]], + message: Optional[str] = None, + error: Optional[str] = None, +) -> Dict[str, Any]: + """Respuesta de insert_valid_rows (finished / warning / failed).""" + out = { + "status": status, + "inserted": inserted, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + if message: + out["message"] = message + if error: + out["error"] = error + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/common/storage.py b/backend/api/v1/modules/a76/layouts_csv/common/storage.py new file mode 100644 index 00000000..f2875d5c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/storage.py @@ -0,0 +1,171 @@ +""" +Redis y rutas de archivos para imports CSV (compartido por módulos layouts_csv). +Cada módulo usa un job_type (ej. "part", "cls", "bom") para prefijos y nombres de archivo. +""" +import os +import base64 +import json +import logging +from typing import Optional, Set, List + +from core.paths import layout_path + +logger = logging.getLogger(__name__) + +IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def upload_dir() -> str: + """Directorio temporal para CSV en el worker.""" + return layout_path("imports", "temp") + + +def error_dir() -> str: + """Directorio de archivos JSONL de errores.""" + return layout_path("imports", "errors") + + +def storage_keys(job_type: str, job_id: str) -> tuple: + """Prefijos Redis para un job_type y job_id. Devuelve (file_key, meta_key, error_lines_key).""" + # Facturas usa prefijo "import_" sin tipo (compatibilidad con rutas existentes) + if job_type == "" or job_type == "invoice": + prefix = "import_" + else: + prefix = f"{job_type}_import_" + return ( + f"{prefix}file:{job_id}", + f"{prefix}meta:{job_id}", + f"{prefix}error_lines:{job_id}", + ) + + +def file_path_for_job(job_type: str, job_id: str) -> str: + """Ruta local del archivo CSV para un job.""" + if job_type == "" or job_type == "invoice": + return os.path.join(upload_dir(), f"{job_id}.csv") + return os.path.join(upload_dir(), f"{job_type}_{job_id}.csv") + + +def error_path_for_job(job_type: str, job_id: str) -> str: + """Ruta del archivo JSONL de errores para un job.""" + os.makedirs(error_dir(), exist_ok=True) + if job_type == "" or job_type == "invoice": + return os.path.join(error_dir(), f"{job_id}.jsonl") + return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl") + + +def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]: + """ + Descarga contenido del CSV desde Redis y lo escribe en disco. + Devuelve la ruta del archivo o None si no hay datos o falla. + """ + file_key, _, _ = storage_keys(job_type, job_id) + r = _get_redis() + data = r.get(file_key) + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e) + return None + path = file_path_for_job(job_type, job_id) + os.makedirs(upload_dir(), exist_ok=True) + with open(path, "wb") as f: + f.write(raw) + return path + + +def ensure_meta_from_redis(job_type: str, job_id: str, file_path: str, log_prefix: str = "") -> bool: + """Descarga meta desde Redis y la escribe en .meta.json. Devuelve True si hubo datos.""" + _, meta_key, _ = storage_keys(job_type, job_id) + r = _get_redis() + data = r.get(meta_key) + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning("%s failed to decode meta from Redis: %s", log_prefix or job_type, e) + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def store_error_lines(job_type: str, job_id: str, line_numbers: List[int]) -> None: + """Guarda la lista de números de línea con error en Redis.""" + _, _, error_key = storage_keys(job_type, job_id) + try: + r = _get_redis() + r.set(error_key, json.dumps(line_numbers).encode("utf-8"), ex=IMPORT_REDIS_TTL) + except Exception as e: + logger.warning("Failed to store error lines in Redis: %s", e) + + +def get_error_lines(job_type: str, job_id: str, error_path: str) -> Set[int]: + """ + Obtiene el conjunto de líneas con error: primero desde Redis, si está vacío desde el JSONL. + """ + _, _, error_key = storage_keys(job_type, job_id) + lines = set() + try: + r = _get_redis() + raw = r.get(error_key) + if raw: + lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug("Could not load error lines from Redis: %s", e) + if not lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + lines.add(err["line"]) + except Exception: + pass + return lines + + +def delete_import_from_redis(job_type: str, job_id: str) -> None: + """Borra claves Redis del import (file, meta, error_lines).""" + file_key, meta_key, error_key = storage_keys(job_type, job_id) + try: + r = _get_redis() + r.delete(file_key, meta_key, error_key) + except Exception as e: + logger.warning("Failed to delete import keys from Redis: %s", e) + + +def cleanup_import_job( + job_type: str, + job_id: str, + file_path: Optional[str] = None, + error_path: Optional[str] = None, + meta_path: Optional[str] = None, +) -> None: + """Elimina archivos locales y claves Redis del job.""" + if file_path and os.path.exists(file_path): + try: + os.remove(file_path) + except Exception as e: + logger.warning("Cleanup: failed to remove file %s: %s", file_path, e) + if error_path and os.path.exists(error_path): + try: + os.remove(error_path) + except Exception as e: + logger.warning("Cleanup: failed to remove error file %s: %s", error_path, e) + if meta_path and os.path.exists(meta_path): + try: + os.remove(meta_path) + except Exception as e: + logger.warning("Cleanup: failed to remove meta %s: %s", meta_path, e) + delete_import_from_redis(job_type, job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/__init__.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/__init__.py new file mode 100644 index 00000000..2632bd0c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/__init__.py @@ -0,0 +1 @@ +# layouts_csv.customs_brokers diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py new file mode 100644 index 00000000..47ef2168 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers, fk_loader diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py new file mode 100644 index 00000000..65292796 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py @@ -0,0 +1,194 @@ +""" +Validadores reutilizables para import CSV de agentes aduanales (customs brokers). +Paridad Clarion: VALIDACIONES_AGENTE_ADUANAL, tipo MEX/AME, patente obligatoria si MEX, +país en catálogo, RFC/CURP máx, desfase. +""" +import re +from typing import Dict, Any, Optional, Set + +BROKER_KEY_MAX = 5 +LICENSE_MAX = 4 +RFC_MAX = 30 +CURP_MAX = 19 + + +def check_required_broker_key(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col B: Clave de Agente Aduanal obligatoria, máx 5 caracteres.""" + clave = (row.get("CLAVE") or "").strip() + if not clave: + return { + "line": line_num, + "col": "CLAVE", + "msg": f"Error: (Celda B{line_num}) La Clave de Agente Aduanal está vacía y no se pueden hacer las validaciones.", + "solution": f"Capturar en la Celda B{line_num} una Clave de Agente Aduanal nueva o existente a la cual desee agregar, remplazar o actualizar campos", + } + if len(clave) > BROKER_KEY_MAX: + return { + "line": line_num, + "col": "CLAVE", + "msg": f"Error: (Col. B) La Clave de Agente Aduanal: {clave} supera la longitud de caracteres.", + "solution": f"Capturar en la columna B una Clave de Agente Aduanal de {BROKER_KEY_MAX} caracteres como máximo.", + } + if not re.match(r"^[a-zA-Z0-9]+$", clave): + return { + "line": line_num, + "col": "CLAVE", + "msg": "Error: (Col. B) La Clave de Agente Aduanal solo puede contener letras y números.", + "solution": "Capturar en la columna B una Clave alfanumérica.", + } + return None + + +def check_optional_license(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col C: Patente/Licencia opcional; si tiene valor, máx 4 dígitos.""" + licencia = (row.get("LICENCIA") or "").strip() + if not licencia: + return None + if len(licencia) > LICENSE_MAX or not licencia.isdigit(): + return { + "line": line_num, + "col": "LICENCIA", + "msg": f"Error: (Col. C) La Patente debe ser de hasta {LICENSE_MAX} dígitos numéricos.", + "solution": f"Capturar en la columna C una Patente de hasta {LICENSE_MAX} dígitos.", + } + return None + + +def check_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA (Col O) tiene valor → advertencia de desfase.""" + val = (row.get("COL_EXTRA") or "").strip() + if not val: + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": "Advertencia: Podría existir un desfase en esta línea.", + "solution": "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta.", + } + + +def parse_tipo_agente_aduanal(val: Optional[str]) -> Optional[str]: + """ + Parsea TIPO (Col A): letra (M/A) o palabra (MEX/Mexicano, AME/Americano). + Devuelve 'M' o 'A'; si no reconoce, None. + """ + if not val or not str(val).strip(): + return None + s = str(val).strip() + v = s.upper() + if len(v) == 1: + if v == "M": + return "M" + if v == "A": + return "A" + v_lower = s.lower() + if v_lower in ("mex", "mexicano"): + return "M" + if v_lower in ("ame", "americano"): + return "A" + return None + + +def check_tipo_mex_ame(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Col A: TIPO debe ser MEX o AME (flexible: M/MEX/Mexicano, A/AME/Americano). + Si TIPO=M entonces PAIS (Col J) debe ser MEX; si TIPO=A entonces PAIS no debe ser MEX. + """ + tipo_raw = (row.get("TIPO") or "").strip() + if not tipo_raw: + return None + tipo = parse_tipo_agente_aduanal(tipo_raw) + if tipo is None: + return { + "line": line_num, + "col": "TIPO", + "msg": f"Error: (Col. A) El tipo de agente aduanal: {tipo_raw} es incorrecto.", + "solution": "Capturar en columna A el Tipo de agente aduanal correcto, MEX para Mexicano y AME para Americano.", + } + pais = (row.get("PAIS") or "").strip().upper() + if not pais: + return None + if tipo == "M" and pais != "MEX": + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. J) El tipo de cliente: {tipo_raw} tiene el país {pais}, es incorrecto.", + "solution": "Capturar en la columna J el país con clave MEX, ya que es un Agente Aduanal Mexicano", + } + if tipo == "A" and pais == "MEX": + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. J) El tipo de cliente: {tipo_raw} tiene el país {pais}, es incorrecto.", + "solution": "Capturar en la columna J un país con clave diferente de MEX, ya que es un Agente Aduanal Americano.", + } + return None + + +def check_patente_obligatoria_si_mex(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col C: Patente obligatoria si TIPO es MEX.""" + tipo = parse_tipo_agente_aduanal((row.get("TIPO") or "").strip()) + if tipo != "M": + return None + licencia = (row.get("LICENCIA") or "").strip() + if not licencia: + clave = (row.get("CLAVE") or "").strip() + return { + "line": line_num, + "col": "LICENCIA", + "msg": f"Error: (Col. C) La Patente para el Agente Aduanal con Clave: {clave} no está capturada.", + "solution": "Capturar en la columna C una Patente de Agente Aduanal.", + } + return None + + +def check_rfc_max(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col E: RFC opcional; si tiene valor, máx 30 caracteres.""" + val = (row.get("RFC") or "").strip() + if not val: + return None + if len(val) > RFC_MAX: + clave = (row.get("CLAVE") or "").strip() + return { + "line": line_num, + "col": "RFC", + "msg": f"Error: (Col. E) El RFC de Agente Aduanal: {clave} supera la longitud de caracteres.", + "solution": f"Capturar en la columna E un RFC de {RFC_MAX} caracteres como máximo.", + } + return None + + +def check_curp_max(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col N: CURP/PERSONAL_ID opcional; si tiene valor, máx 19 caracteres.""" + val = (row.get("PERSONAL_ID") or "").strip() + if not val: + return None + if len(val) > CURP_MAX: + clave = (row.get("CLAVE") or "").strip() + return { + "line": line_num, + "col": "PERSONAL_ID", + "msg": f"Error: (Col. N) El CURP de Agente Aduanal: {clave} supera la longitud de caracteres.", + "solution": f"Capturar en la columna N un CURP de Agente Aduanal de {CURP_MAX} caracteres como máximo.", + } + return None + + +def check_pais_catalogo( + row: Dict[str, Any], + line_num: int, + valid_country_m3: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col J: Si PAIS tiene valor, debe existir en catálogo (clave M3).""" + val = (row.get("PAIS") or "").strip() + if not val or valid_country_m3 is None: + return None + if val.upper() in valid_country_m3: + return None + clave = (row.get("CLAVE") or "").strip() + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. J) El Pais del Agente Aduanal: {clave} no está en el Catálogo de Países.", + "solution": "Capturar en la columna J un País válido.", + } diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/fk_loader.py new file mode 100644 index 00000000..255f2603 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/fk_loader.py @@ -0,0 +1,26 @@ +""" +Carga de conjuntos FK para validación de import CSV de agentes aduanales. +Clarion: catálogo de países (GPaises / m3_key). +""" +from typing import Set +import logging + +from core.database import CoreSessionLocal + +logger = logging.getLogger(__name__) + + +def load_customs_brokers_fk_sets() -> Set[str]: + """ + Carga valid_country_m3 (códigos país m3_key) para validar Col J PAIS. + """ + valid_country_m3: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.countries.models import Country + for row in session.query(Country.m3_key).all(): + if row[0]: + valid_country_m3.add((row[0] or "").strip().upper()) + except Exception as e: + logger.warning("Customs brokers import: could not load country m3 keys: %s", e) + return valid_country_m3 diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py new file mode 100644 index 00000000..4fbd6eae --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py @@ -0,0 +1,77 @@ +""" +Mapeo fila CSV → datos para CustomsBroker. +Clarion: TIPO normalizado a M/A con parse_tipo_agente_aduanal; CURP/personal_id máx 19. +""" +from typing import Dict, Any, Optional + +from .common_validators import parse_tipo_agente_aduanal, CURP_MAX + +MAX_LEN = { + "broker_key": 5, + "type": 9, + "name": 80, + "address": 1500, + "postal_code": 15, + "city": 30, + "state": 30, + "phone": 30, + "fax": 30, + "email": 100, + "country": 3, + "tax_id": 30, + "personal_id": 20, + "position": 30, + "license": 4, + "company": 200, + "contact": 80, +} + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _license_value(row_norm: Dict[str, Any]) -> Optional[str]: + lic = (row_norm.get("LICENCIA") or "").strip() + if not lic or not lic.isdigit(): + return None + return lic[:4] + + +def row_to_customs_broker_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int +) -> Dict[str, Any]: + """Build dict for CustomsBroker model (create or update).""" + clave = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["broker_key"]) + if not clave: + return {} + tipo_raw = (row_norm.get("TIPO") or "").strip() + tipo_normalized = parse_tipo_agente_aduanal(tipo_raw) if tipo_raw else None + return { + "tenant_id": tenant_id, + "company_id": company_id, + "broker_key": clave, + "type": tipo_normalized or _str_or_none(row_norm.get("TIPO"), MAX_LEN["type"]), + "name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]), + "address": _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]), + "postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "phone": _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]), + "fax": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]), + "email": _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "tax_id": _str_or_none(row_norm.get("RFC"), MAX_LEN["tax_id"]), + "personal_id": _str_or_none(row_norm.get("PERSONAL_ID"), CURP_MAX), + "position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]), + "license": _license_value(row_norm), + "company": _str_or_none(row_norm.get("EMPRESA"), MAX_LEN["company"]), + "contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]), + } diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py similarity index 98% rename from backend/api/v1/modules/a76/customs_brokers/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py index 99f7c609..369208f0 100644 --- a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py @@ -14,6 +14,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -81,7 +82,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/customs_brokers/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/customs_brokers/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py new file mode 100644 index 00000000..2f58f124 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py @@ -0,0 +1,273 @@ +""" +Tareas Celery para importación CSV de Agentes Aduanales. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import json +import logging +import os +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import ( + row_from_template, + CUSTOMS_BROKERS_FIELDNAMES_ORDER, + CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL, +) +from .validators import validate_row_customs_broker +from .common.mappers import row_to_customs_broker_data +from .common.fk_loader import load_customs_brokers_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "cb" + +# Para routes.py +CB_IMPORT_FILE_PREFIX = "cb_import_file:" +CB_IMPORT_META_PREFIX = "cb_import_meta:" +CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:" +CB_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv_reader.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_broker_keys: Set[str] = set() + if actualizar: + try: + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + with CoreSessionLocal() as session: + for b in ( + session.query(CustomsBroker) + .filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ) + .all() + ): + if (b.broker_key or "").strip(): + existing_broker_keys.add((b.broker_key or "").strip()) + except Exception as e: + logger.warning("CB import: could not load existing broker_keys for ACT: %s", e) + + valid_country_m3 = load_customs_brokers_fk_sets() + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows( + file_path, + fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER, + headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL, + ): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_customs_broker( + row_norm, + i, + actualizar=actualizar, + existing_broker_keys=existing_broker_keys, + valid_country_m3=valid_country_m3, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("CB import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("CB import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + valid_country_m3 = load_customs_brokers_fk_sets() + + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + existing_by_key: Dict[str, CustomsBroker] = {} + for b in ( + session.query(CustomsBroker) + .filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ) + .all() + ): + existing_by_key[b.broker_key] = b + + existing_broker_keys = set(existing_by_key.keys()) + + for i, row in common_csv_reader.iter_csv_rows( + file_path, + fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER, + headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL, + ): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_customs_broker( + row_norm, + i, + actualizar=actualizar, + existing_broker_keys=existing_broker_keys, + valid_country_m3=valid_country_m3, + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + data = row_to_customs_broker_data(row_norm, tenant_id, company_id) + if not data or not data.get("broker_key"): + skipped_invalid += 1 + continue + + clave = data["broker_key"] + existing = existing_by_key.get(clave) + if existing: + for k, v in data.items(): + if k not in ("tenant_id", "company_id", "broker_key"): + setattr(existing, k, v) + session.add(existing) + else: + new_broker = CustomsBroker(**data) + session.add(new_broker) + existing_by_key[clave] = new_broker + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("CB import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("CB import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if inserted_count == 0 and skipped_invalid > 0: + return { + "status": "warning", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("CB import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/template_config.py similarity index 53% rename from backend/api/v1/modules/a76/customs_brokers/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/customs_brokers/template_config.py index f4801729..b656b59a 100644 --- a/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/template_config.py @@ -1,5 +1,8 @@ """ Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls). +Layout Clarion: A=TIPO, B=CLAVE AADUANAL, C=PATENTE, D=NOMBRE, E=RFC, F=DIRECCION, G=CODIGO POSTAL, +H=CIUDAD, I=ESTADO, J=PAIS, K=TELEFONO, L=NUMERO FAX, M=CORREO ELECTRONICO, N=CURP, O=COL_EXTRA (desfase). +Encabezado ejemplo: "TIPO(MEX=Mexicano,AME=AMERICANO)",CLAVE AADUANAL,PATENTE,NOMBRE,RFC,... Solo se leen columnas definidas aquí; el resto se ignora. """ @@ -7,26 +10,66 @@ from typing import Dict, List, Any, Optional TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { "customs_brokers": [ - {"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]}, - {"canonical": "TIPO"}, - {"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]}, - {"canonical": "DIRECCION", "aliases": ["DOMICILIO"]}, - {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, - {"canonical": "CIUDAD"}, - {"canonical": "ESTADO"}, - {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]}, - {"canonical": "FAX"}, - {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]}, - {"canonical": "PAIS", "aliases": ["COUNTRY"]}, - {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]}, - {"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]}, - {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + # Col A - TIPO (MEX/Mexicano, AME/Americano) + {"canonical": "TIPO", "aliases": ["TIPO(MEX=Mexicano,AME=AMERICANO)"]}, + # Col B - Clave agente aduanal (máx 5) + {"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "CLAVE AADUANAL", "CLAVE ADUANAL", "ID"]}, + # Col C - Patente / Licencia (obligatoria si TIPO=MEX) {"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]}, + # Col D - Nombre + {"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]}, + # Col E - RFC (máx 30) + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]}, + # Col F - Dirección + {"canonical": "DIRECCION", "aliases": ["DOMICILIO"]}, + # Col G - Código postal + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + # Col H - Ciudad + {"canonical": "CIUDAD"}, + # Col I - Estado + {"canonical": "ESTADO"}, + # Col J - País (clave M3) + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + # Col K - Teléfono + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]}, + # Col L - Fax + {"canonical": "FAX", "aliases": ["NUMERO FAX"]}, + # Col M - Email + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]}, + # Col N - CURP / Personal ID (máx 19) + {"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL", "CURP"]}, + # Col O - Desfase (si tiene valor → advertencia) + {"canonical": "COL_EXTRA", "aliases": ["COLUMNA EXTRA", "DESFASE"]}, + # Otros opcionales (no en encabezado oficial) + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, {"canonical": "EMPRESA", "aliases": ["COMPANY"]}, {"canonical": "CONTACTO", "aliases": ["CONTACT"]}, ], } +# Orden oficial de columnas (para CSV sin encabezado o detección) +CUSTOMS_BROKERS_FIELDNAMES_ORDER = [ + "TIPO", + "CLAVE", + "LICENCIA", + "NOMBRE", + "RFC", + "DIRECCION", + "CODIGO POSTAL", + "CIUDAD", + "ESTADO", + "PAIS", + "TELEFONO", + "FAX", + "EMAIL", + "PERSONAL_ID", +] + +# Si la primera celda de la primera fila está en este set, se trata como CSV sin encabezado +CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL = frozenset( + {"MEX", "AME", "M", "A", "MEXICANO", "AMERICANO"} +) + def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: """normalized_header -> canonical_name para plantilla customs_brokers.""" @@ -39,6 +82,8 @@ def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: lookup[normalize_header_fn(canonical)] = canonical for alias in item.get("aliases") or []: lookup[normalize_header_fn(alias)] = canonical + for idx, name in enumerate(CUSTOMS_BROKERS_FIELDNAMES_ORDER): + lookup[normalize_header_fn(f"_COL_{idx}_")] = name return lookup diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py new file mode 100644 index 00000000..ecfaa325 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_customs_broker + +__all__ = ["validate_row_customs_broker"] diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py new file mode 100644 index 00000000..867bbe70 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py @@ -0,0 +1,126 @@ +""" +Validaciones comunes de fila para import CSV de agentes aduanales. +Paridad Clarion: VALIDA_TODA_AGENTE_ADUANAL, VALIDA_PARCIAL_AGENTE_ADUANAL, VALIDACIONES_AGENTE_ADUANAL. +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + check_required_broker_key, + check_optional_license, + check_desfase, + check_tipo_mex_ame, + check_patente_obligatoria_si_mex, + check_rfc_max, + check_curp_max, + check_pais_catalogo, +) + +MSG_CLAVE_NO_EXISTE = "Error: (Col. B) Clave de A. Aduanal No Existe en el Catalogo." +MSG_CLAVE_NO_EXISTE_SOLUCION = "La clave debe existir en el catálogo cuando el modo es Actualizar." +MSG_CAMPOS_OBLIGATORIOS = "Existen campos vacíos que son obligatorios, es la {campos}." +MSG_CAMPOS_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta." + + +def validaciones_agente_aduanal( + row: Dict[str, Any], + line_num: int, + valid_country_m3: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + VALIDACIONES_AGENTE_ADUANAL: reglas de dominio compartidas. + Tipo MEX/AME + coherencia País, patente si MEX, RFC/CURP máx, país en catálogo, licencia formato. + """ + err = check_tipo_mex_ame(row, line_num) + if err: + return err + err = check_patente_obligatoria_si_mex(row, line_num) + if err: + return err + err = check_optional_license(row, line_num) + if err: + return err + err = check_rfc_max(row, line_num) + if err: + return err + err = check_curp_max(row, line_num) + if err: + return err + err = check_pais_catalogo(row, line_num, valid_country_m3) + if err: + return err + return None + + +def valida_toda_agente_aduanal( + row: Dict[str, Any], + line_num: int, + valid_country_m3: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + VALIDA_TODA: TIPO (Col A) y NOMBRE (Col D) obligatorios; luego VALIDACIONES_AGENTE_ADUANAL. + """ + campos_oblig = [] + if not (row.get("TIPO") or "").strip(): + campos_oblig.append("(Col.A) Tipo") + if not (row.get("NOMBRE") or "").strip(): + campos_oblig.append("(Col.D) Nombre") + if campos_oblig: + return { + "line": line_num, + "col": "TIPO" if not (row.get("TIPO") or "").strip() else "NOMBRE", + "msg": MSG_CAMPOS_OBLIGATORIOS.format(campos=", ".join(campos_oblig)), + "solution": MSG_CAMPOS_OBLIGATORIOS_SOLUCION, + } + return validaciones_agente_aduanal(row, line_num, valid_country_m3) + + +def valida_parcial_agente_aduanal( + row: Dict[str, Any], + line_num: int, + valid_country_m3: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_PARCIAL: solo VALIDACIONES_AGENTE_ADUANAL (no exige TIPO ni NOMBRE).""" + return validaciones_agente_aduanal(row, line_num, valid_country_m3) + + +def validate_row_customs_broker( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, + existing_broker_keys: Optional[Set[str]] = None, + valid_country_m3: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de agentes aduanales. + 1. Desfase (COL_EXTRA) primero. + 2. Clave vacía → error. + 3. Si actualizar y clave no existe en existing_broker_keys → error. + 4. Si actualizar y clave existe → valida_parcial_agente_aduanal. + 5. Si no actualizar o clave no existe → valida_toda_agente_aduanal. + """ + err = check_desfase(row, line_num) + if err: + return err + err = check_required_broker_key(row, line_num) + if err: + return err + + clave = (row.get("CLAVE") or "").strip() + existing = existing_broker_keys or set() + use_partial = actualizar and clave in existing + + if actualizar and clave and clave not in existing: + return { + "line": line_num, + "col": "CLAVE", + "msg": MSG_CLAVE_NO_EXISTE, + "solution": MSG_CLAVE_NO_EXISTE_SOLUCION, + } + + if use_partial: + err = valida_parcial_agente_aduanal(row, line_num, valid_country_m3) + else: + err = valida_toda_agente_aduanal(row, line_num, valid_country_m3) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py new file mode 100644 index 00000000..6336bbb2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py @@ -0,0 +1,6 @@ +""" +Punto de entrada de validación para import de una fila agente aduanal. +""" +from .common import validate_row_customs_broker + +__all__ = ["validate_row_customs_broker"] diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/drivers/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/drivers/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py new file mode 100644 index 00000000..695ad37a --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py @@ -0,0 +1,4 @@ +# common validators, mappers, fk_loader for drivers CSV import +from .fk_loader import load_drivers_fk_sets + +__all__ = ["load_drivers_fk_sets"] diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py new file mode 100644 index 00000000..743780f4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py @@ -0,0 +1,261 @@ +""" +Helpers reutilizables para validación de filas CSV (conductores). +Paridad Clarion: VALIDACIONES_CONDUCTOR, obligatorios A/C, catálogos transportista/países, Sexo M/F, Si/No, tipo identificación, desfase. +""" +import re +from typing import Dict, Any, Optional, Set + + +MAX_LEN = { + "transporter_key": 5, + "driver_name": 80, + "license_number": 29, + "express_line_id": 17, + "ace_id": 20, + "gender": 1, + "birth_country": 3, + "hazardous_material_auth": 2, + "hazardous_material_state": 30, + "first_name": 20, + "last_name": 20, + "id_key1": 40, + "id_number1": 20, + "id_state1": 30, + "id_country1": 3, + "id_key2": 40, + "id_number2": 20, + "id_state2": 30, + "id_country2": 3, +} + +# Clarion: Col H Sexo M o F +SEXO_VALIDOS = {"M", "F"} + +# Clarion: Col J Si o No (comparar en mayúsculas) +MATERIAL_PELIGROSO_VALIDOS = {"SI", "NO"} + +# Clarion: Col N y Col R tipo identificación +FORMA_IDENTIFICACION_CLAVES = frozenset( + {"ACW", "ALR", "BCP", "BCN", "CDN", "CON", "OTD", "REP", "RTP", "5J", "5K", "30"} +) + +# Mapeo clave CSV → valor guardado en BD (Clarion QueCSV:ColumnaN/R) +FORMA_IDENTIFICACION_MAP = { + "ACW": "ACW-Pasaporte", + "ALR": "ALR-Residencia", + "BCP": "BCP-Permiso Cruce", + "BCN": "BCN-Acta Nacimiento", + "CDN": "CDN-Ciudadania", + "CON": "CON-CertificadoNaturalizacion", + "OTD": "OTD-Otra Identificación", + "REP": "REP-Permiso Reentrada", + "RTP": "RTP-Permiso de Viaje", + "5J": "5J - Licencia", + "5K": "5K -Licencia", + "30": "30 -Visa de EU", +} + + +def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"} + return None + + +def parse_int(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d+$", s): + return int(s) + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + return None + + +def check_int_positive(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + v = parse_int(row.get(col)) + if v is None: + return {"line": line_num, "col": col, "msg": "Debe ser numerico"} + if v <= 0: + return {"line": line_num, "col": col, "msg": "Debe ser mayor a 0"} + return None + + +def parse_birth_date(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + return None + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + return None + break + return None + + +def check_optional_birth_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or not str(val).strip(): + return None + if parse_birth_date(val) is None: + return { + "line": line_num, + "col": col, + "msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)", + } + return None + + +def check_transportista_catalog( + row: Dict[str, Any], + line_num: int, + valid_transporter_keys: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col A: Si TRANSPORTISTA no vacío, debe existir en catálogo (GTransportista).""" + val = (row.get("TRANSPORTISTA") or "").strip() + if not val or valid_transporter_keys is None: + return None + if val.upper() in valid_transporter_keys: + return None + return { + "line": line_num, + "col": "TRANSPORTISTA", + "msg": f"Error: (Col. A) La Clave de Transportista: {val} es incorrecto.", + "solution": "Capturar en columna A un Transportista existente en catalogo.", + } + + +def check_sexo_m_f(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col H: Si SEXO no vacío, debe ser M o F (Clarion).""" + val = (row.get("SEXO") or "").strip() + if not val: + return None + if val.upper() in SEXO_VALIDOS: + return None + conductor = (row.get("CLAVE CONDUCTOR") or "").strip() or "(Conductor)" + return { + "line": line_num, + "col": "SEXO", + "msg": f"Error: (Col. H) El Sexo: {val} del Conductor: {conductor} es incorrecto.", + "solution": "Capturar en columna H el sexo correcto (M o F).", + } + + +def check_pais_catalog_drivers( + row: Dict[str, Any], + col: str, + line_num: int, + valid_country_ame: Optional[Set[str]] = None, + col_letter: str = "", +) -> Optional[Dict[str, Any]]: + """Si col (PAIS NACIMIENTO, PAIS, PAIS 2) no vacío, debe ser clave americana en catálogo (GPaises.Pais_Ame).""" + val = (row.get(col) or "").strip() + if not val or valid_country_ame is None: + return None + val_upper = val.upper() + if len(val) > 3: + return { + "line": line_num, + "col": col, + "msg": f"Error: ({col_letter}) El Pais: {val} es incorrecto.", + "solution": "Capturar un Pais en clave americana (US, MX, etc.).", + } + if val_upper in valid_country_ame: + return None + return { + "line": line_num, + "col": col, + "msg": f"Error: ({col_letter}) El Pais: {val} es incorrecto.", + "solution": "Capturar en columna un Pais en clave americana.", + } + + +def check_material_peligroso_si_no(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col J: TRANSPORTA MAT. PELIGROSO? debe ser Si o No (Clarion; comparar en mayúsculas).""" + val = (row.get("TRANSPORTA MAT. PELIGROSO?") or "").strip() + if not val: + return None + if val.upper() in MATERIAL_PELIGROSO_VALIDOS: + return None + return { + "line": line_num, + "col": "TRANSPORTA MAT. PELIGROSO?", + "msg": "Error: (Col. J) La Autorizacion para el Manejo de Material Peligroso es incorrecta.", + "solution": "Capturar en columna J Si o No la autorizacion.", + } + + +def check_tipo_identificacion( + row: Dict[str, Any], + col: str, + line_num: int, + col_letter: str = "", + primera_o_segunda: str = "Primera", +) -> Optional[Dict[str, Any]]: + """Col N o R: Si FORMA IDENTIFICACION no vacío, debe ser clave válida (ACW, ALR, ...).""" + val = (row.get(col) or "").strip() + if not val: + return None + val_upper = val.upper() + if val_upper in FORMA_IDENTIFICACION_CLAVES: + return None + return { + "line": line_num, + "col": col, + "msg": f"Error: ({col_letter}) El Tipo de Identificacion: {val} de la {primera_o_segunda} Identificacion es incorrecto.", + "solution": f"Capturar en columna {col_letter} una clave de Identificacion valida (ACW, ALR, BCP, BCN, CDN, CON, OTD, REP, RTP, 5J, 5K, 30).", + } + + +def check_desfase_drivers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA (Col V) tiene valor -> advertencia de desfase (Clarion, no bloqueante).""" + val = (row.get("COL_EXTRA") or "").strip() + if not val: + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": "Advertencia: Podria existir un desfase en esta linea.", + "solution": "Revisar esta linea del archivo CSV y verificar cada campo este en la posicion correcta.", + "warning": True, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/fk_loader.py new file mode 100644 index 00000000..7584fec2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/fk_loader.py @@ -0,0 +1,58 @@ +""" +Carga de conjuntos FK para validación de import CSV de conductores. +Clarion: GTransportista (ClaveTrans), GPaises (Pais_Ame). +""" +from typing import Set, Tuple, Optional, Dict +import logging + +from core.database import CoreSessionLocal + +logger = logging.getLogger(__name__) + + +def load_drivers_fk_sets( + tenant_id: Optional[int] = None, + company_id: Optional[int] = None, +) -> Tuple[Set[str], Set[str], Dict[str, str]]: + """ + Carga conjuntos para validación CSV de conductores (paridad Clarion). + Devuelve: + - valid_transporter_keys: todas las claves de transportistas en mayúsculas (a76.transporter) + - valid_country_ame: claves americana de países (GPaises.Pais_Ame / Country.ame_key), mayúsculas + - transporter_key_actual: dict clave_upper -> clave real en BD (para insert con mismo caso que en transporter) + """ + valid_transporter_keys: Set[str] = set() + valid_country_ame: Set[str] = set() + transporter_key_actual: Dict[str, str] = {} + + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.transportation.transporters.models import Transporter + from api.v1.modules.public.reference_data.countries.models import Country + + # Solo transportistas del tenant/company del upload (paridad con Driver.tenant_id/company_id) + q = session.query(Transporter.transporter_key).filter( + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) + for row in q.all(): + if row[0]: + raw = (row[0] or "").strip() + upper = raw.upper() + valid_transporter_keys.add(upper) + transporter_key_actual[upper] = raw + + for row in session.query(Country.ame_key).all(): + if row[0]: + valid_country_ame.add((row[0] or "").strip().upper()) + + except Exception as e: + logger.warning("Drivers import: could not load FK sets: %s", e) + + logger.info( + "Drivers import FK: %d transportistas (claves: %s), %d paises", + len(valid_transporter_keys), + sorted(valid_transporter_keys)[:20] if len(valid_transporter_keys) <= 20 else sorted(valid_transporter_keys)[:10] + ["..."], + len(valid_country_ame), + ) + return (valid_transporter_keys, valid_country_ame, transporter_key_actual) diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py new file mode 100644 index 00000000..45f8ccbb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py @@ -0,0 +1,79 @@ +""" +Mapeo fila CSV → datos para Driver (conductores). +Clarion: FORMA IDENTIFICACION 1/2 se guardan expandidas (ACW → ACW-Pasaporte, etc.). +""" +from typing import Dict, Any, Optional + +from .common_validators import ( + MAX_LEN, + parse_int, + parse_birth_date, + FORMA_IDENTIFICACION_MAP, +) + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _forma_identificacion_or_raw(val: Any, max_len: int) -> Optional[str]: + """Si el valor es una clave Clarion (ACW, ALR, ...), devuelve el valor expandido; si no, el valor truncado.""" + s = _str_or_none(val, max_len) + if not s: + return None + expanded = FORMA_IDENTIFICACION_MAP.get(s.upper()) + if expanded: + return expanded[:max_len] if len(expanded) > max_len else expanded + return s + + +def row_to_driver_data( + row_norm: Dict[str, Any], + tenant_id: int, + company_id: int, +) -> Dict[str, Any]: + """Mapea una fila normalizada del CSV a un diccionario para DriverCreateDTO.""" + transporter_key = _str_or_none(row_norm.get("TRANSPORTISTA"), MAX_LEN["transporter_key"]) + line = parse_int(row_norm.get("LINEA")) + if not transporter_key or line is None: + return {} + return { + "transporter_key": transporter_key, + "line": line, + "driver_name": _str_or_none(row_norm.get("CLAVE CONDUCTOR"), MAX_LEN["driver_name"]), + "license_number": _str_or_none(row_norm.get("LICENCIA"), MAX_LEN["license_number"]), + "express_line_id": _str_or_none(row_norm.get("PERMISO LINEA EXPRESS"), MAX_LEN["express_line_id"]), + "ace_id": _str_or_none(row_norm.get("IDENTIFICACION ACE"), MAX_LEN["ace_id"]), + "birth_date": parse_birth_date(row_norm.get("FECHA NACIMIENTO")), + "gender": _str_or_none(row_norm.get("SEXO"), MAX_LEN["gender"]), + "birth_country": _str_or_none(row_norm.get("PAIS NACIMIENTO"), MAX_LEN["birth_country"]), + "hazardous_material_auth": _str_or_none( + row_norm.get("TRANSPORTA MAT. PELIGROSO?"), MAX_LEN["hazardous_material_auth"] + ), + "hazardous_material_state": _str_or_none( + row_norm.get("PERMISO MAT. PELIGROSO"), MAX_LEN["hazardous_material_state"] + ), + "first_name": _str_or_none(row_norm.get("NOMBRE(S)"), MAX_LEN["first_name"]), + "last_name": _str_or_none(row_norm.get("APELLIDO PATERNO"), MAX_LEN["last_name"]), + "id_key1": _forma_identificacion_or_raw( + row_norm.get("FORMA IDENTIFICACION 1"), MAX_LEN["id_key1"] + ), + "id_number1": _str_or_none(row_norm.get("NUM. IDENTIFICACION 1"), MAX_LEN["id_number1"]), + "id_state1": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["id_state1"]), + "id_country1": _str_or_none(row_norm.get("PAIS"), MAX_LEN["id_country1"]), + "id_key2": _forma_identificacion_or_raw( + row_norm.get("FORMA IDENTIFICACION 2"), MAX_LEN["id_key2"] + ), + "id_number2": _str_or_none(row_norm.get("NUM. IDENTIFICACION 2"), MAX_LEN["id_number2"]), + "id_state2": _str_or_none(row_norm.get("ESTADO 2"), MAX_LEN["id_state2"]), + "id_country2": _str_or_none(row_norm.get("PAIS 2"), MAX_LEN["id_country2"]), + "company_id": company_id, + "tenant_id": tenant_id, + } diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py similarity index 97% rename from backend/api/v1/modules/a76/transportation/drivers/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/drivers/routes.py index ea5d2eb4..c4b1972b 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py @@ -15,11 +15,11 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse from .tasks import ( - scan_file, run_scan_sync, run_commit_sync, DRV_IMPORT_FILE_PREFIX, @@ -81,7 +81,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f: f.write(contents) @@ -90,8 +90,6 @@ async def upload_import_file( except Exception as e: logger.warning(f"Drivers import: local file save failed: {e}") - scan_file.apply_async(args=[job_id], task_id=job_id) - def run_scan_background(): try: run_scan_sync(job_id) diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/drivers/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/drivers/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py new file mode 100644 index 00000000..e7c3f777 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py @@ -0,0 +1,497 @@ +""" +Tareas Celery para importación CSV de Conductores. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe) y clave de estado en Redis. +Paridad Clarion: actualizar, existing_driver_keys, valid_transporter_keys, valid_country_ame. +""" +import csv +import json +import logging +import os +from typing import Dict, Any, Optional, List, Set, Tuple + +from core.celery_app import celery_app +from core.database import CoreSessionLocal +from sqlalchemy import func + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from .template_config import row_from_template +from .validators import validate_row_driver, validate_row_driver_desfase +from .common.mappers import row_to_driver_data +from .common.fk_loader import load_drivers_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "drv" + +# Para routes.py +DRV_IMPORT_FILE_PREFIX = "drv_import_file:" +DRV_IMPORT_META_PREFIX = "drv_import_meta:" +DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:" +DRV_IMPORT_STATUS_PREFIX = "drv_import_status:" +DRV_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL +DRV_IMPORT_TRANSPORTER_MAP_PREFIX = "drv_import_transporter_map:" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _dedupe_headers(headers: List[str]) -> List[str]: + counts: Dict[str, int] = {} + unique: List[str] = [] + for header in headers: + name = str(header or "").strip() or "COL" + count = counts.get(name, 0) + 1 + counts[name] = count + unique.append(name if count == 1 else f"{name} {count}") + return unique + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_driver_keys: Set[Tuple[str, int]] = set() + if actualizar: + try: + from api.v1.modules.a76.transportation.drivers.models import Driver + with CoreSessionLocal() as session: + for row in ( + session.query(Driver.transporter_key, Driver.line) + .filter( + Driver.tenant_id == tenant_id, + Driver.company_id == company_id, + ) + .all() + ): + if row[0] is not None and row[1] is not None: + existing_driver_keys.add( + ((row[0] or "").strip().upper(), int(row[1])) + ) + except Exception as e: + logger.warning("Drivers import: could not load existing_driver_keys for actualizar: %s", e) + + valid_transporter_keys, valid_country_ame, transporter_key_actual = load_drivers_fk_sets(tenant_id, company_id) + + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_TRANSPORTER_MAP_PREFIX}{job_id}", + json.dumps(transporter_key_actual).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Drivers import: failed to store transporter map in Redis: %s", e) + + logger.info( + "Drivers import scan: job_id=%s tenant_id=%s company_id=%s actualizar=%s transportistas=%d", + job_id, tenant_id, company_id, actualizar, len(valid_transporter_keys), + ) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f_in, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, common_normalize.normalize_header) + _ = validate_row_driver_desfase(row_norm, i) + err = validate_row_driver( + row_norm, + i, + actualizar=actualizar, + existing_driver_keys=existing_driver_keys, + valid_transporter_keys=valid_transporter_keys, + valid_country_ame=valid_country_ame, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Drivers import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Drivers import: failed to store scan status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Drivers import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Drivers import: failed to store scan status in Redis: %s", e) + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_driver_keys: Set[Tuple[str, int]] = set() + if actualizar: + try: + from api.v1.modules.a76.transportation.drivers.models import Driver + with CoreSessionLocal() as session: + for row in ( + session.query(Driver.transporter_key, Driver.line) + .filter( + Driver.tenant_id == tenant_id, + Driver.company_id == company_id, + ) + .all() + ): + if row[0] is not None and row[1] is not None: + existing_driver_keys.add( + ((row[0] or "").strip().upper(), int(row[1])) + ) + except Exception as e: + logger.warning("Drivers import: could not load existing_driver_keys for actualizar: %s", e) + + valid_transporter_keys, valid_country_ame, transporter_key_actual = load_drivers_fk_sets(tenant_id, company_id) + + transporter_map_from_redis: Optional[Dict[str, str]] = None + try: + r = _get_redis() + map_key = f"{DRV_IMPORT_TRANSPORTER_MAP_PREFIX}{job_id}" + raw = r.get(map_key) + if raw: + transporter_map_from_redis = json.loads(raw.decode("utf-8")) + logger.info( + "Drivers import commit: using transporter map from Redis (job_id=%s, keys=%d)", + job_id, len(transporter_map_from_redis), + ) + else: + logger.warning( + "Drivers import commit: no transporter map in Redis for job_id=%s, using DB fallback", + job_id, + ) + except Exception as e: + logger.warning( + "Drivers import: could not load transporter map from Redis (job_id=%s): %s", + job_id, e, + ) + + logger.info( + "Drivers import commit: job_id=%s tenant_id=%s company_id=%s transportistas=%d", + job_id, tenant_id, company_id, len(valid_transporter_keys), + ) + + from api.v1.modules.a76.transportation.drivers.services import DriverService + from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_driver( + row_norm, + i, + actualizar=actualizar, + existing_driver_keys=existing_driver_keys, + valid_transporter_keys=valid_transporter_keys, + valid_country_ame=valid_country_ame, + ) + if err: + skipped_invalid += 1 + driver_key = (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-" + skipped_details.append({ + "line": i, + "driver_key": driver_key, + "invoice": driver_key, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + data = row_to_driver_data(row_norm, tenant_id, company_id) + if not data or not data.get("transporter_key") or data.get("line") is None: + skipped_invalid += 1 + continue + + tk = (data["transporter_key"] or "").strip() + # Usar mapa del scan (Redis) si existe; si no, resolver en la sesión del commit (fallback) + if transporter_map_from_redis is not None: + tk_upper = tk.upper() + if tk_upper not in transporter_map_from_redis: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "driver_key": f"{tk}:{data.get('line')}", + "invoice": f"{tk}:{data.get('line')}", + "reason": f"El transportista {tk} no existe en el catálogo.", + }) + continue + data["transporter_key"] = transporter_map_from_redis[tk_upper] + else: + from api.v1.modules.a76.transportation.transporters.models import Transporter + transporter_row = ( + session.query(Transporter.transporter_key) + .filter( + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + func.upper(Transporter.transporter_key) == tk.upper(), + ) + .first() + ) + if not transporter_row or not transporter_row[0]: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "driver_key": f"{tk}:{data.get('line')}", + "invoice": f"{tk}:{data.get('line')}", + "reason": f"El transportista {tk} no existe en el catálogo.", + }) + continue + data["transporter_key"] = (transporter_row[0] or "").strip() + + key = f"{data['transporter_key']}:{data['line']}" + if key in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append({ + "line": i, + "driver_key": key, + "invoice": key, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) + continue + seen_keys_in_file[key] = i + + existing = DriverService.get_driver_by_key_and_line( + session, data["transporter_key"], data["line"], str(company_id), tenant_id + ) + try: + if existing: + update_fields = { + k: v for k, v in data.items() + if k not in ("transporter_key", "line", "company_id", "tenant_id") + } + for field, value in update_fields.items(): + setattr(existing, field, value) + session.add(existing) + updated_count += 1 + else: + create_data = DriverCreateDTO(**data) + DriverService.create_driver(session, create_data) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + err_msg = str(db_err) + if "ForeignKeyViolation" in err_msg or "foreign key constraint" in err_msg.lower() or "driver_transporter_key_fkey" in err_msg: + err_msg = f"El transportista {data.get('transporter_key', '')} no existe en el catálogo." + logger.warning( + "Drivers import: FK violation linea %d transporter_key=%r (valid_transporter_keys tiene %d claves)", + i, data.get("transporter_key"), len(valid_transporter_keys), + ) + skipped_details.append({ + "line": i, "driver_key": key, "invoice": key, "reason": err_msg, + }) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("Drivers import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("Drivers import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + try: + r = _get_redis() + r.delete(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}") + r.delete(f"{DRV_IMPORT_TRANSPORTER_MAP_PREFIX}{job_id}") + except Exception as e: + logger.warning("Drivers import: failed to delete status key: %s", e) + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros validos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Drivers import: failed to store commit status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Drivers import: starting commit for job %s", job_id) + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Drivers import: failed to store commit status in Redis: %s", e) + return result diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/drivers/template_config.py similarity index 97% rename from backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/drivers/template_config.py index cd9a3194..891dccb1 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/template_config.py @@ -27,6 +27,7 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]}, {"canonical": "ESTADO 2", "aliases": ["STATE 2"]}, {"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]}, + {"canonical": "COL_EXTRA", "aliases": ["COLUMNA V", "COL V"]}, ], } diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py new file mode 100644 index 00000000..4dd127be --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_driver, validate_row_driver_desfase + +__all__ = ["validate_row_driver", "validate_row_driver_desfase"] diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py new file mode 100644 index 00000000..ab2cc764 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py @@ -0,0 +1,164 @@ +""" +Validaciones comunes de fila para import CSV de conductores. +Paridad Clarion: VALIDACIONES_CONDUCTOR, VALIDA_TODA_CONDUCTOR, VALIDA_PARCIAL_CONDUCTOR. +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + MAX_LEN, + check_max_length, + check_int_positive, + check_optional_birth_date, + check_transportista_catalog, + check_sexo_m_f, + check_pais_catalog_drivers, + check_material_peligroso_si_no, + check_tipo_identificacion, +) + + +def validate_row_driver_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Obligatorios Clarion: Col A (TRANSPORTISTA) y Col C (CLAVE CONDUCTOR). Si falta uno, no se ejecutan validaciones.""" + err = check_max_length( + row, "TRANSPORTISTA", MAX_LEN["transporter_key"], line_num, required=True + ) + if err: + return err + err = check_max_length( + row, "CLAVE CONDUCTOR", MAX_LEN["driver_name"], line_num, required=True + ) + if err: + return err + return None + + +def validate_row_driver_required_full(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """VALIDA_TODA: obligatorios A, C y LINEA (Col B) para registro nuevo.""" + err = validate_row_driver_required(row, line_num) + if err: + return err + return check_int_positive(row, "LINEA", line_num) + + +def validate_row_driver_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("CLAVE CONDUCTOR", MAX_LEN["driver_name"]), + ("LICENCIA", MAX_LEN["license_number"]), + ("PERMISO LINEA EXPRESS", MAX_LEN["express_line_id"]), + ("IDENTIFICACION ACE", MAX_LEN["ace_id"]), + ("SEXO", MAX_LEN["gender"]), + ("PAIS NACIMIENTO", MAX_LEN["birth_country"]), + ("TRANSPORTA MAT. PELIGROSO?", MAX_LEN["hazardous_material_auth"]), + ("PERMISO MAT. PELIGROSO", MAX_LEN["hazardous_material_state"]), + ("NOMBRE(S)", MAX_LEN["first_name"]), + ("APELLIDO PATERNO", MAX_LEN["last_name"]), + ("FORMA IDENTIFICACION 1", MAX_LEN["id_key1"]), + ("NUM. IDENTIFICACION 1", MAX_LEN["id_number1"]), + ("ESTADO", MAX_LEN["id_state1"]), + ("PAIS", MAX_LEN["id_country1"]), + ("FORMA IDENTIFICACION 2", MAX_LEN["id_key2"]), + ("NUM. IDENTIFICACION 2", MAX_LEN["id_number2"]), + ("ESTADO 2", MAX_LEN["id_state2"]), + ("PAIS 2", MAX_LEN["id_country2"]), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_driver_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_optional_birth_date(row, "FECHA NACIMIENTO", line_num) + + +def validaciones_conductores( + row: Dict[str, Any], + line_num: int, + valid_transporter_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + VALIDACIONES_CONDUCTOR: reglas compartidas (longitudes, fecha, transportista en catálogo, + sexo M/F, país nacimiento, material peligroso Si/No, tipo ID 1/2, país ID1/ID2). + """ + err = validate_row_driver_lengths(row, line_num) + if err: + return err + err = validate_row_driver_date(row, line_num) + if err: + return err + err = check_transportista_catalog(row, line_num, valid_transporter_keys) + if err: + return err + err = check_sexo_m_f(row, line_num) + if err: + return err + err = check_pais_catalog_drivers( + row, "PAIS NACIMIENTO", line_num, valid_country_ame, col_letter="Col. I" + ) + if err: + return err + err = check_material_peligroso_si_no(row, line_num) + if err: + return err + err = check_tipo_identificacion( + row, + "FORMA IDENTIFICACION 1", + line_num, + col_letter="Col. N", + primera_o_segunda="Primera", + ) + if err: + return err + err = check_pais_catalog_drivers( + row, "PAIS", line_num, valid_country_ame, col_letter="Col. Q" + ) + if err: + return err + err = check_tipo_identificacion( + row, + "FORMA IDENTIFICACION 2", + line_num, + col_letter="Col. R", + primera_o_segunda="Segunda", + ) + if err: + return err + err = check_pais_catalog_drivers( + row, "PAIS 2", line_num, valid_country_ame, col_letter="Col. U" + ) + if err: + return err + return None + + +def valida_toda_conductor( + row: Dict[str, Any], + line_num: int, + valid_transporter_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_TODA_CONDUCTOR: obligatorios A, C y LINEA (B) + validaciones_conductores (registro nuevo).""" + err = validate_row_driver_required_full(row, line_num) + if err: + return err + return validaciones_conductores( + row, line_num, + valid_transporter_keys=valid_transporter_keys, + valid_country_ame=valid_country_ame, + ) + + +def valida_parcial_conductor( + row: Dict[str, Any], + line_num: int, + valid_transporter_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_PARCIAL_CONDUCTOR: solo validaciones_conductores (actualizar registro existente).""" + return validaciones_conductores( + row, line_num, + valid_transporter_keys=valid_transporter_keys, + valid_country_ame=valid_country_ame, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py new file mode 100644 index 00000000..d04d14b2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py @@ -0,0 +1,64 @@ +""" +Punto de entrada de validación para import de una fila conductor. +Paridad Clarion: desfase (advertencia), obligatorios A y C, VALIDA_TODA vs VALIDA_PARCIAL según actualizar y clave existente. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from ..common.common_validators import parse_int, check_desfase_drivers +from .common import ( + validate_row_driver_required, + valida_toda_conductor, + valida_parcial_conductor, +) + + +def validate_row_driver( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, + existing_driver_keys: Optional[Set[Tuple[str, int]]] = None, + valid_transporter_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de conductores. + 1. Obligatorios A (TRANSPORTISTA) y C (CLAVE CONDUCTOR) vacíos → error. + 2. Si actualizar y (TRANSPORTISTA, LINEA) en existing_driver_keys → valida_parcial_conductor. + 3. Si no actualizar o conductor no existe → valida_toda_conductor (obligatorios A, C y LINEA + validaciones). + Desfase (COL_EXTRA) no se valida aquí; el caller puede llamar validate_row_driver_desfase para advertencias no bloqueantes. + """ + err = validate_row_driver_required(row, line_num) + if err: + return err + + existing = existing_driver_keys or set() + transporter_key = (row.get("TRANSPORTISTA") or "").strip().upper() + line = parse_int(row.get("LINEA")) + use_partial = ( + actualizar + and bool(transporter_key and line is not None) + and (transporter_key, line) in existing + ) + + if use_partial: + err = valida_parcial_conductor( + row, + line_num, + valid_transporter_keys=valid_transporter_keys, + valid_country_ame=valid_country_ame, + ) + else: + err = valida_toda_conductor( + row, + line_num, + valid_transporter_keys=valid_transporter_keys, + valid_country_ame=valid_country_ame, + ) + return err + + +def validate_row_driver_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Advertencia de desfase si COL_EXTRA (Col V) tiene valor. No bloqueante; el caller puede acumular en warnings. + """ + return check_desfase_drivers(row, line_num) diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/exchange_rate/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py new file mode 100644 index 00000000..ab0f8ea6 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for exchange_rate) diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py new file mode 100644 index 00000000..6d1bf126 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py @@ -0,0 +1,140 @@ +""" +Validadores reutilizables para import CSV de tipos de cambio. +""" +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Any, Optional, List + +DATE_FORMATS: List[str] = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] + +# Valores que envía el frontend (globalCsvParams dateFormat) -> formato strptime +# Cuando el usuario elige un formato en "Parámetros globales", solo se aceptan fechas en ese formato. +DATE_FORMAT_PREFERENCE_MAP: Dict[str, str] = { + "dd/mm/yyyy": "%d/%m/%Y", + "mm/dd/yyyy": "%m/%d/%Y", + "yyyy-mm-dd": "%Y-%m-%d", +} + +CURRENCY_MAX = 7 + + +def parse_date(val: Optional[str], date_format_preference: Optional[str] = None) -> Optional[datetime]: + """ + Parsea fecha. Si date_format_preference está definido (formato elegido en el frontend), + solo se acepta ese formato; si la cadena no coincide, se rechaza. + Si no hay preferencia, se intentan todos los formatos (retrocompatibilidad). + """ + if not val or not str(val).strip(): + return None + raw = str(val).strip() + if date_format_preference and date_format_preference in DATE_FORMAT_PREFERENCE_MAP: + fmt = DATE_FORMAT_PREFERENCE_MAP[date_format_preference] + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + return None + for fmt in DATE_FORMATS: + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + continue + return None + + +def parse_decimal_positive(val: Optional[str]) -> Optional[Decimal]: + if not val or not str(val).strip(): + return None + try: + v = float(str(val).strip().replace(",", ".")) + if v <= 0: + return None + return Decimal(str(round(v, 6))) + except (ValueError, TypeError): + return None + + +def check_required_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + fecha_raw = (row.get("FECHA") or "").strip() + if not fecha_raw: + return {"line": line_num, "col": "FECHA", "msg": "Requerido"} + if parse_date(fecha_raw) is None: + return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} + return None + + +def check_required_value_positive(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + valor_raw = (row.get("VALOR") or "").strip() + if not valor_raw: + return {"line": line_num, "col": "VALOR", "msg": "Requerido"} + try: + v = float(valor_raw.replace(",", ".")) + if v <= 0: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} + except ValueError: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} + return None + + +def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +FECHA_MAX_LEN = 10 +MSG_FECHA_LONGITUD = "Error: (Col. A) La Fecha: {fecha} supera la longitud de caracteres." +MSG_FECHA_LONGITUD_SOLUCION = "Capturar en la columna A el campo Fecha con este formato ##/##/####." +MSG_FECHA_DIA_INVALIDO = "Error: (Col. A) El día {dia} de la Fecha: {fecha} no es válido para el mes." +MSG_FECHA_DIA_SOLUCION = "Capturar correctamente en la columna A el día del campo Fecha, con este formato ##/##/####. (Día/Mes/Año)" +MSG_FECHA_MES_INVALIDO = "Error: (Col. A) El mes {mes} de la Fecha: {fecha} es mayor a 12 esto no es valido." +MSG_FECHA_MES_SOLUCION = "Capturar correctamente en la columna A el mes del campo Fecha, con este formato ##/##/####. (Día/Mes/Año)" + +# Etiquetas para mensaje cuando no coincide con el formato elegido +DATE_FORMAT_LABELS: Dict[str, str] = { + "dd/mm/yyyy": "DD/MM/YYYY (Día/Mes/Año)", + "mm/dd/yyyy": "MM/DD/YYYY (Mes/Día/Año)", + "yyyy-mm-dd": "YYYY-MM-DD (Año-Mes-Día)", +} + + +def validate_fecha_clarion( + raw_fecha: str, line_num: int, date_format_preference: Optional[str] = None +) -> Optional[Dict[str, Any]]: + """ + Valida la fecha según reglas Clarion: longitud ≤ 10, día válido para el mes, mes ≤ 12. + Sin límite de año. Si date_format_preference está definido (ej. dd/mm/yyyy), se prioriza ese formato. + """ + if not raw_fecha or not str(raw_fecha).strip(): + return None + raw = str(raw_fecha).strip() + if len(raw) > FECHA_MAX_LEN: + return { + "line": line_num, + "col": "FECHA", + "msg": f"{MSG_FECHA_LONGITUD.format(fecha=raw)} {MSG_FECHA_LONGITUD_SOLUCION}", + } + parsed = parse_date(raw, date_format_preference) + if parsed is None: + format_label = ( + DATE_FORMAT_LABELS.get(date_format_preference, "##/##/####") + if date_format_preference + else "##/##/####" + ) + return { + "line": line_num, + "col": "FECHA", + "msg": f"Error: (Col. A) La Fecha: {raw} no coincide con el formato elegido ({format_label}). {MSG_FECHA_LONGITUD_SOLUCION}", + } + d = parsed.date() + if d.month > 12: + return { + "line": line_num, + "col": "FECHA", + "msg": f"{MSG_FECHA_MES_INVALIDO.format(mes=d.month, fecha=raw)} {MSG_FECHA_MES_SOLUCION}", + } + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py new file mode 100644 index 00000000..136b1c5f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py @@ -0,0 +1,48 @@ +""" +Mapeo fila CSV → datos para ExchangeRate. +""" +from datetime import datetime +from decimal import Decimal +from typing import Dict, Any, Optional + +from .common_validators import ( + parse_date, + parse_decimal_positive, +) + +CURRENCY_MAX = 7 + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_exchange_rate_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int, date_format_preference: Optional[str] = None +) -> Dict[str, Any]: + """Build dict for ExchangeRate model. Returns {} if FECHA or VALOR invalid.""" + parsed_date = parse_date(row_norm.get("FECHA"), date_format_preference) + value_decimal = parse_decimal_positive(row_norm.get("VALOR")) + if not parsed_date or value_decimal is None: + return {} + local = _str_or_none(row_norm.get("MONEDA_LOCAL"), CURRENCY_MAX) + if local: + local = local.upper() + foreign = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), CURRENCY_MAX) + if foreign: + foreign = foreign.upper() + return { + "tenant_id": tenant_id, + "company_id": company_id, + "date": parsed_date, + "value": value_decimal, + "local_currency": local, + "foreign_currency": foreign, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py similarity index 87% rename from backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py index 75a9aef8..cb8ca60a 100644 --- a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py @@ -10,10 +10,11 @@ from uuid import uuid4 from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends from sqlalchemy.orm import Session -from typing import Dict, Any +from typing import Dict, Any, Optional from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -39,11 +40,20 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + reemplazar_sin_preguntar: bool = Query( + True, + description="Si True, reemplaza tipos de cambio existentes para la misma fecha; si False, solo agrega nuevos (omite fechas ya existentes)", + ), + date_format: Optional[str] = Query( + None, + description="Formato de fecha del CSV: dd/mm/yyyy, mm/dd/yyyy o yyyy-mm-dd. Si no se envía, se intentan todos.", + ), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): """ Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + Parámetros globales de carga: reemplazar_sin_preguntar (Modo Reemplazar vs Actualizar), date_format (Formato de Fecha). """ try: tenant_id = validate_access_to_resource(db, company_id, current_user) @@ -62,6 +72,8 @@ async def upload_import_file( "company_id": company_id, "user_id": current_user.get("id"), "template_id": "exchange_rates", + "reemplazar_sin_preguntar": reemplazar_sin_preguntar, + "date_format": date_format, } try: @@ -81,7 +93,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/exchange_rate/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py new file mode 100644 index 00000000..68610c11 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py @@ -0,0 +1,241 @@ +""" +Tareas Celery para importación CSV de Tipos de Cambio. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import json +import logging +import os +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template +from .validators import validate_row_exchange_rate +from .common.mappers import row_to_exchange_rate_data + +logger = logging.getLogger(__name__) + +JOB_TYPE = "er" +TEMPLATE_ID = "exchange_rates" + +# Para routes.py +ER_IMPORT_FILE_PREFIX = "er_import_file:" +ER_IMPORT_META_PREFIX = "er_import_meta:" +ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:" +ER_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: + return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv_reader.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + # Formato activo del selector del frontend; si no viene, mismo default que el front (dd/mm/yyyy) + date_format_preference = meta.get("date_format") or meta.get("dateFormat") or "dd/mm/yyyy" + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = _norm_row(row) + err = validate_row_exchange_rate(row_norm, i, raw_row=row, date_format_preference=date_format_preference) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("ER import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("ER import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + reemplazar_sin_preguntar = meta.get("reemplazar_sin_preguntar", True) + date_format_preference = meta.get("date_format") or meta.get("dateFormat") + + from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate + + inserted_count = 0 + updated_count = 0 + skipped_duplicate = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + existing_by_date: Dict[tuple, ExchangeRate] = {} + for er in ( + session.query(ExchangeRate) + .filter( + ExchangeRate.tenant_id == tenant_id, + ExchangeRate.company_id == company_id, + ) + .all() + ): + d = er.date.date() if hasattr(er.date, "date") else er.date + existing_by_date[(tenant_id, company_id, d)] = er + + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = _norm_row(row) + err = validate_row_exchange_rate(row_norm, i, raw_row=row, date_format_preference=date_format_preference) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + data = row_to_exchange_rate_data(row_norm, tenant_id, company_id, date_format_preference) + if not data or not data.get("date"): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FECHA o VALOR no válidos"}) + continue + + key_date = data["date"].date() if hasattr(data["date"], "date") else data["date"] + existing = existing_by_date.get((tenant_id, company_id, key_date)) + if existing: + if not reemplazar_sin_preguntar: + skipped_duplicate += 1 + continue + existing.value = data["value"] + existing.local_currency = data.get("local_currency") + existing.foreign_currency = data.get("foreign_currency") + session.add(existing) + updated_count += 1 + else: + new_er = ExchangeRate(**data) + session.add(new_er) + existing_by_date[(tenant_id, company_id, key_date)] = new_er + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("ER import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("ER import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if inserted_count == 0 and updated_count == 0 and (skipped_invalid > 0 or skipped_duplicate > 0): + return { + "status": "warning", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados, {skipped_duplicate} omitidos por fecha existente.", + } + if inserted_count == 0 and updated_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("ER import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/template_config.py similarity index 100% rename from backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/exchange_rate/template_config.py diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py new file mode 100644 index 00000000..cf3b1792 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_exchange_rate + +__all__ = ["validate_row_exchange_rate"] diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py new file mode 100644 index 00000000..6ba36f2e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py @@ -0,0 +1,90 @@ +""" +Validaciones comunes de fila para import CSV de tipos de cambio. +Paridad Clarion: desfase (Col C vacía), obligatorios (Col A Fecha, Col B Tipo de Cambio), +validación fecha (longitud, día acorde al mes, mes ≤ 12; sin límite de año). +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + check_required_value_positive, + check_optional_max_length, + validate_fecha_clarion, + CURRENCY_MAX, +) + + +MSG_DESFASE = "Error: Existe un desfase en esta línea." +MSG_DESFASE_SOLUCION = "Revisar esta línea del archivo CSV y verificar cada campo este en la posicion correcta." + +MSG_OBLIGATORIOS = "Existen campos vacios que son obligatorios, es la (Col.A) Fecha , (Col.B) Tipo de Cambio." +MSG_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta." + + +def validate_row_desfase(raw_row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Si la fila tiene 3 o más columnas y la 3ª tiene valor, error de desfase (Clarion ColumnaC <> ''). + """ + values_ordered = list(raw_row.values()) if raw_row else [] + if len(values_ordered) >= 3 and (values_ordered[2] or "").strip(): + return { + "line": line_num, + "col": "", + "msg": f"{MSG_DESFASE} {MSG_DESFASE_SOLUCION}", + } + return None + + +def validate_row_required_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """FECHA y VALOR obligatorios con mensaje Clarion.""" + fecha = (row.get("FECHA") or "").strip() + valor = (row.get("VALOR") or "").strip() + if not fecha or not valor: + return { + "line": line_num, + "col": "FECHA" if not fecha else "VALOR", + "msg": f"{MSG_OBLIGATORIOS} {MSG_OBLIGATORIOS_SOLUCION}", + } + return None + + +def validate_row_fecha_clarion( + row: Dict[str, Any], line_num: int, date_format_preference: Optional[str] = None +) -> Optional[Dict[str, Any]]: + """Longitud ≤ 10 y fecha válida (día acorde al mes, mes ≤ 12; sin límite de año).""" + raw_fecha = (row.get("FECHA") or "").strip() + if not raw_fecha: + return None + return validate_fecha_clarion(raw_fecha, line_num, date_format_preference) + + +def validate_row_exchange_rate( + row: Dict[str, Any], + line_num: int, + raw_row: Optional[Dict[str, Any]] = None, + date_format_preference: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de tipos de cambio. + Orden: desfase (si raw_row) → obligatorios → fecha Clarion → VALOR > 0 → MONEDA opc (max 7). + date_format_preference: valor del parámetro global (ej. dd/mm/yyyy, mm/dd/yyyy, yyyy-mm-dd). + """ + if raw_row is not None: + err = validate_row_desfase(raw_row, line_num) + if err: + return err + err = validate_row_required_exchange_rate(row, line_num) + if err: + return err + err = validate_row_fecha_clarion(row, line_num, date_format_preference) + if err: + return err + err = check_required_value_positive(row, line_num) + if err: + return err + err = check_optional_max_length(row, "MONEDA_LOCAL", CURRENCY_MAX, line_num) + if err: + return err + err = check_optional_max_length(row, "MONEDA_EXTRANJERA", CURRENCY_MAX, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py new file mode 100644 index 00000000..ef5d3d47 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py @@ -0,0 +1,6 @@ +""" +Punto de entrada de validación para import de una fila tipo de cambio. +""" +from .common import validate_row_exchange_rate + +__all__ = ["validate_row_exchange_rate"] diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/__init__.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/__init__.py new file mode 100644 index 00000000..dadf0cb2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/__init__.py @@ -0,0 +1 @@ +# layouts_csv.exportacion — carga CSV exportación (encabezado y partidas) diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py new file mode 100644 index 00000000..a357f1f0 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para Exportación (encabezado y partidas). +Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, UploadFile, Depends, Form, Query +from sqlalchemy.orm import Session +from typing import Literal, Optional, Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.paths import layout_path +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse, CommitRequest +from .tasks import scan_file, insert_valid_rows, JOB_TYPE, EXP_IMPORT_REDIS_TTL +from ..common import storage as common_storage + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload/{model_target}", response_model=ImportJobResponse) +async def upload_import_file( + model_target: Literal["invoice_header", "invoice_details"], + file: UploadFile = File(...), + footer_config: Optional[str] = Form(None), + template_id: Optional[str] = Form(None), + company_id: int = Query(..., description="Company ID"), + operation_type: Optional[str] = Query("exp"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """Subir CSV, guardar en Redis, encolar scan. operation_type=exp para exportación.""" + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error("Exportación import: access validation failed: %s", e) + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id) + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "footer_config": footer_config, + "operation_type": operation_type or "exp", + "template_id": template_id or ("exp_def_header" if model_target == "invoice_header" else "exp_def_details"), + } + + try: + r = _get_redis() + r.set(file_key, base64.b64encode(contents), ex=EXP_IMPORT_REDIS_TTL) + r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=EXP_IMPORT_REDIS_TTL) + except Exception as e: + logger.error("Exportación import: Redis store error: %s", e) + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = layout_path("imports", "temp") + os.makedirs(upload_dir, exist_ok=True) + csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + with open(csv_path, "wb") as f: + f.write(contents) + meta_path = csv_path.replace(".csv", ".meta.json") + with open(meta_path, "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning("Exportación import: local file save failed: %s", e) + + scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """Polling: estado del escaneo o del commit.""" + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"): + return task_result.result + + logger.warning("Exportación import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg: + result = getattr(task_result, "result", None) + if result is not None and not isinstance(result, dict): + err_msg = str(result) + elif isinstance(result, dict) and (result.get("error") or result.get("message")): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str, body: CommitRequest): + """Usuario confirma; se encola la tarea de commit (por ahora sin inserción real).""" + task = insert_valid_rows.delay(job_id, body.model_target) + return { + "status": "committing", + "message": "Proceso de commit iniciado.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/schemas.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/schemas.py new file mode 100644 index 00000000..2a043f30 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/schemas.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel +from typing import Optional, Literal + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + model_target: Literal["invoice_header", "invoice_details"] + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + error_file: Optional[str] = None diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py new file mode 100644 index 00000000..0d43f5b1 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py @@ -0,0 +1,139 @@ +""" +Tareas Celery para importación CSV de Exportación (encabezado y partidas). +Flujo: scan_file (sin validaciones) → insert_valid_rows (sin inserción en BD). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import logging +import os +from typing import Dict, Any, Optional + +from core.celery_app import celery_app + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +JOB_TYPE = "exp" +EXP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _ensure_file(job_id: str) -> Optional[str]: + return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Exportación import") + + +def _ensure_meta(job_id: str, file_path: str) -> bool: + return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Exportación import") + + +def _norm_row(row: Dict[str, Any], template_id: str) -> Dict[str, Any]: + return row_from_template(row, template_id, common_normalize.normalize_header) + + +@celery_app.task(bind=True, name="api.v1.modules.a76.layouts_csv.exportacion.tasks.scan_file") +def scan_file(self, job_id: str, model_target: str, config: str = None): + """ + Scan CSV sin validaciones: leer, normalizar con plantilla, devolver total_rows y 0 errores. + """ + logger.info("Exportación import: starting scan for job %s target %s", job_id, model_target) + + file_path = _ensure_file(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + if os.path.getsize(file_path) == 0: + return {"status": "failed", "error": "El archivo está vacío."} + _ensure_meta(job_id, file_path) + + try: + common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + template_id = meta.get("template_id") or ( + "exp_def_header" if model_target == "invoice_header" else "exp_def_details" + ) + + total_rows = 0 + processed_rows = 0 + + try: + total_rows = common_csv_reader.count_csv_rows(file_path, has_header=True) + except Exception as e: + return {"status": "failed", "error": str(e)} + + def on_progress(current: int, total: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total}) + + try: + for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None): + if i % 500 == 0: + on_progress(i, total_rows) + _norm_row(row, template_id) + processed_rows += 1 + except Exception as e: + logger.error("Exportación import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, 0, []) + + +@celery_app.task(bind=True, name="api.v1.modules.a76.layouts_csv.exportacion.tasks.insert_valid_rows") +def insert_valid_rows(self, job_id: str, model_target: str): + """ + Commit sin inserción en BD: leer CSV, omitir líneas de error (vacío por ahora), cleanup, devolver finished con inserted=0. + """ + logger.info("Exportación import: starting commit for job %s target %s", job_id, model_target) + + file_path = _ensure_file(job_id) + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + _ensure_meta(job_id, file_path) + + try: + common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + meta_path = common_meta.get_meta_path(file_path) + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + template_id = meta.get("template_id") or ( + "exp_def_header" if model_target == "invoice_header" else "exp_def_details" + ) + + try: + for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None): + if i in error_lines: + continue + _norm_row(row, template_id) + except Exception as e: + logger.error("Exportación import commit read failed: %s", e) + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + return { + "status": "finished", + "inserted": 0, + "skipped_invalid": 0, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": [], + "message": "Proceso base listo; validaciones e inserción pendientes.", + } diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/template_config.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/template_config.py new file mode 100644 index 00000000..d6d598fc --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/template_config.py @@ -0,0 +1,89 @@ +""" +Plantillas CSV para Exportación (encabezado y partidas). +Misma estructura que facturas exp_def_header / exp_def_details; módulo autocontenido. +""" + +from typing import Dict, List, Any, Optional + +# Columnas para encabezado y partidas de exportación (EstructuraEncFacExpoCamReg / EstructuraParExpoCamReg) +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "exp_def_header": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, + {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, + {"canonical": "FECHA EMISION"}, + {"canonical": "CLAVE PROVEEDOR"}, + {"canonical": "CLAVE VENDIDO A"}, + {"canonical": "CLAVE ENVIADO A"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, + {"canonical": "ADUANA DE CRUCE"}, + {"canonical": "CLAVE MONEDA"}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "TIPO PESO"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "REMESA"}, + {"canonical": "AGENTE ADUANAL"}, + {"canonical": "FLETES"}, + {"canonical": "VALOR SEGUROS"}, + {"canonical": "SEGUROS"}, + {"canonical": "EMBALAJES"}, + {"canonical": "OTROS INCREMENTABLES"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "OBSERVACIONES E"}, + {"canonical": "OBSERVACIONES I"}, + {"canonical": "E DOCUMENT"}, + {"canonical": "NUM OPERACION"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "PRECINTO"}, + ], + "exp_def_details": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]}, + {"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]}, + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]}, + {"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]}, + {"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]}, + {"canonical": "CANTIDAD"}, + {"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]}, + {"canonical": "FRACCION"}, + {"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]}, + ], +} + + +def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]: + return TEMPLATE_COLUMNS.get(template_id) + + +def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name.""" + cols = _resolve_template_columns(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]: + """Fila CSV -> dict con nombres canónicos de la plantilla.""" + lookup = build_normalized_lookup(template_id, normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/facturas/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/facturas/__init__.py diff --git a/backend/api/v1/modules/a76/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py similarity index 96% rename from backend/api/v1/modules/a76/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/facturas/routes.py index f3c3d9ee..c7e9c570 100644 --- a/backend/api/v1/modules/a76/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py @@ -11,6 +11,7 @@ from typing import Optional, Literal, Dict, Any from core.celery_app import celery_app from core.config import settings from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .tasks import ( @@ -88,7 +89,7 @@ async def upload_import_file( # Optional: also write to local disk (e.g. for same-machine worker or debugging) try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) file_path = os.path.join(upload_dir, f"{job_id}.csv") meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") diff --git a/backend/api/v1/modules/a76/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/facturas/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/facturas/schemas.py diff --git a/backend/api/v1/modules/a76/imports/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py similarity index 88% rename from backend/api/v1/modules/a76/imports/tasks.py rename to backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index b6e8e8d9..a3fe37be 100644 --- a/backend/api/v1/modules/a76/imports/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -1,5 +1,4 @@ import os -import base64 from datetime import datetime from decimal import Decimal import csv @@ -11,87 +10,37 @@ from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal +from core.paths import layout_path +from ..common import storage as common_storage +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process -# We'll need schemas for validation -# from api.v1.modules.a76.invoices.schemas import InvoiceHeaderCreate -# But for Phase 1 we use a lighter check - logger = logging.getLogger(__name__) -# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem) +# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py) +JOB_TYPE = "" + +# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="") IMPORT_FILE_KEY_PREFIX = "import_file:" IMPORT_META_KEY_PREFIX = "import_meta:" IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" -IMPORT_REDIS_TTL = 3600 # 1 hour - - -def _get_redis(): - """Redis client using same URL as Celery broker (worker and API can share data).""" - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - """Directory on the worker for temp CSV and meta (same structure as API, but local to worker).""" - return os.path.join(os.getcwd(), "uploads", "temp") +IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - """ - Load file content from Redis and write to worker's upload dir. - Returns local file_path if successful, None otherwise. - """ - redis_client = _get_redis() - key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}" - data = redis_client.get(key) - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Failed to decode import file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path + """Usa common storage con job_type vacío (prefijo import_).""" + return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import") def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - """Load meta from Redis and write to worker's meta file. Returns True if meta was found and written.""" - redis_client = _get_redis() - key = f"{IMPORT_META_KEY_PREFIX}{job_id}" - data = redis_client.get(key) - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Failed to decode import meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True + return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import") def _delete_import_from_redis(job_id: str) -> None: - """Remove file, meta and error lines from Redis after commit (cleanup).""" - try: - r = _get_redis() - r.delete( - f"{IMPORT_FILE_KEY_PREFIX}{job_id}", - f"{IMPORT_META_KEY_PREFIX}{job_id}", - f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Failed to delete import keys from Redis: {e}") + common_storage.delete_import_from_redis(JOB_TYPE, job_id) class ForeignKeyValidator: def __init__(self, session, tenant_id, company_id): @@ -188,9 +137,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} _ensure_worker_has_meta_from_redis(job_id, file_path) - # 2. Setup Error Log - error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") - os.makedirs(os.path.dirname(error_path), exist_ok=True) + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) total_rows = 0 error_count = 0 @@ -211,22 +158,12 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - tenant_id = None - company_id = None - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f_meta: - meta = json.load(f_meta) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception as e: - logger.warning(f"Failed to read meta for job {job_id}: {e}") - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Missing context (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + meta = common_meta.load_meta(file_path) template_id = meta.get("template_id") or ( "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" ) @@ -335,24 +272,11 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): except Exception: pass if error_lines_list: - r = _get_redis() - r.set( - f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=IMPORT_REDIS_TTL, - ) + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.warning(f"Failed to store error lines in Redis: {e}") - # 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV) - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def validate_row_phase_1( row: Dict[str, Any], @@ -770,13 +694,23 @@ def insert_valid_rows(self, job_id: str, model_target: str): # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) file_path = _ensure_worker_has_file_from_redis(job_id) if not file_path: - upload_dir = _worker_upload_dir() - file_path = os.path.join(upload_dir, f"{job_id}.csv") - if not os.path.exists(file_path): + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} + file_path = alt_path else: _ensure_worker_has_meta_from_redis(job_id, file_path) + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + meta_path = common_meta.get_meta_path(file_path) + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + try: from api.v1.modules.a76.invoices.models import ( InvoiceHeader, @@ -798,66 +732,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm - from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.line_financials.models import LineFinancial from api.v1.modules.a76.items.line_quantities.models import LineQuantity from api.v1.modules.a76.items.line_customs.models import LineCustom from api.v1.modules.a76.items.line_descriptions.models import LineDescription from api.v1.modules.a76.parts.models import Part - error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") + footer_config = parse_footer_config(meta.get("footer_config")) - # 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file) - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, 'r', encoding='utf-8') as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err['line']) - except Exception: - pass - - # Load Metadata (Context) - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - footer_config = {} - meta = {} - - if os.path.exists(meta_path): - try: - with open(meta_path, 'r') as f: - meta = json.load(f) or {} - tenant_id = meta.get('tenant_id') - company_id = meta.get('company_id') - operation_type_raw = meta.get('operation_type', 'imp') - footer_config = parse_footer_config(meta.get('footer_config')) - except: pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Missing context (tenant/company)"} - - # 2. Re-read and Map - # Initialize counters outside the session block so they're accessible later - headers_to_insert = [] - details_to_insert = [] - skipped_invalid = 0 - skipped_missing_invoice = 0 - skipped_missing_fk = 0 - skipped_fk_details = [] - inserted_count = 0 - response = None # Will be set inside the session block - date_format = footer_config.get("dateFormat") - # Validate and set default date_format if not provided if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format @@ -871,6 +755,15 @@ def insert_valid_rows(self, job_id: str, model_target: str): logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") + headers_to_insert = [] + details_to_insert = [] + skipped_invalid = 0 + skipped_missing_invoice = 0 + skipped_missing_fk = 0 + skipped_fk_details = [] + inserted_count = 0 + response = None + with CoreSessionLocal() as session: invoice_id_cache = {} cleared_invoices = set() # Track invoices where we've already cleared items in this job @@ -1459,11 +1352,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - _delete_import_from_redis(job_id) + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) except Exception as cleanup_err: logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) diff --git a/backend/api/v1/modules/a76/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py similarity index 100% rename from backend/api/v1/modules/a76/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py diff --git a/backend/api/v1/modules/a76/parts/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/parts/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/parts/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/parts/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py new file mode 100644 index 00000000..1efc271d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py @@ -0,0 +1 @@ +# common validators for parts CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py new file mode 100644 index 00000000..9e77983c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py @@ -0,0 +1,49 @@ +""" +Helpers reutilizables para validación de filas CSV (partes). +Cada función devuelve Optional[Dict] con keys line, col, msg. +Referencia: a76/invoices (common_validators, validators/common). +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional + + +def check_required( + row: Dict[str, Any], col: str, line_num: int +) -> Optional[Dict[str, Any]]: + """Devuelve error si el campo está vacío.""" + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + """Devuelve error si el campo excede max_len. Si required=True, también exige que haya valor.""" + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_decimal( + row: Dict[str, Any], col: str, line_num: int +) -> Optional[Dict[str, Any]]: + """Devuelve error si el valor no es un número decimal válido (solo cuando hay valor).""" + val = row.get(col) + if val is None or val == "": + return None + try: + Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser número"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py new file mode 100644 index 00000000..f173a0eb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py @@ -0,0 +1,141 @@ +""" +Carga de conjuntos FK para validación/mapeo de import CSV de partes. +Clarion: Clases, UOM, monedas, fracción Mex (+ histórico), países, sectores autorizados, excepción RFC. +""" +from typing import Set, Tuple, Optional +import logging + +from core.database import CoreSessionLocal + +logger = logging.getLogger(__name__) + +# RFCs con excepción: E (UOM) obligatorio solo cuando D (Clase) está vacía; Preferencia solo cuando D no vacía +RFC_EXCEPTION_SET = {"CLA940831AZ5", "CTE980130518"} + + +def load_parts_fk_sets( + tenant_id: int, + company_id: int, +) -> Tuple[ + Set[str], + Set[str], + Set[str], + Set[str], + Set[str], + Set[str], + bool, + bool, +]: + """ + Carga conjuntos para validación CSV de partes (paridad Clarion). + Devuelve: + - valid_class_codes + - valid_uom_codes + - valid_currency_codes (claves moneda para G=MC y genéricas) + - valid_fraction_mex_8 (fracción Mex 8 chars: TariffFraction + HistoricalTariffFraction) + - valid_country_m3 (códigos país m3_key) + - authorized_sector_keys (sectores con authorized=True) + - company_has_prosec (Company.prosec) + - is_rfc_exception (Company.rfc en RFC_EXCEPTION_SET) + """ + valid_class_codes: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_currency_codes: Set[str] = set() + valid_fraction_mex_8: Set[str] = set() + valid_country_m3: Set[str] = set() + authorized_sector_keys: Set[str] = set() + company_has_prosec = False + is_rfc_exception = False + + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.a76.general_catalogs.company.models import Company + from api.v1.modules.public.reference_data.countries.models import Country + from api.v1.modules.public.reference_data.sectors.models import Sector + from api.v1.modules.a76.general_catalogs.fractions.tariff_fractions.models import TariffFraction + from api.v1.modules.a76.general_catalogs.fractions.historical_tariff_fractions.models import ( + HistoricalTariffFraction, + ) + + for c in ( + session.query(Class.class_code) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + if c[0]: + valid_class_codes.add((c[0] or "").strip().upper()) + + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + if u[0]: + valid_uom_codes.add((u[0] or "").strip().upper()) + + for cur in session.query(CurrencyType.code).all(): + if cur[0]: + valid_currency_codes.add((cur[0] or "").strip().upper()) + + company = ( + session.query(Company) + .filter(Company.id == company_id) + .first() + ) + if company: + company_has_prosec = bool(company.prosec) + rfc = (company.rfc or "").strip().upper() + is_rfc_exception = rfc in RFC_EXCEPTION_SET + + for row in session.query(Country.m3_key).all(): + if row[0]: + valid_country_m3.add((row[0] or "").strip().upper()) + + for row in ( + session.query(Sector.key) + .filter(Sector.authorized == True) + .all() + ): + if row[0]: + authorized_sector_keys.add((row[0] or "").strip().upper()) + + for row in session.query(TariffFraction.code).all(): + if row[0]: + code = (row[0] or "").strip() + valid_fraction_mex_8.add(code[:8]) + + for row in ( + session.query(HistoricalTariffFraction.historical_fraction) + .filter( + HistoricalTariffFraction.tenant_id == tenant_id, + HistoricalTariffFraction.company_id == company_id, + HistoricalTariffFraction.historical_fraction.isnot(None), + ) + .distinct() + .all() + ): + if row[0] and (row[0] or "").strip(): + valid_fraction_mex_8.add((row[0] or "").strip()[:8]) + + except Exception as e: + logger.warning("Parts import: could not load FK sets: %s", e) + + return ( + valid_class_codes, + valid_uom_codes, + valid_currency_codes, + valid_fraction_mex_8, + valid_country_m3, + authorized_sector_keys, + company_has_prosec, + is_rfc_exception, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py new file mode 100644 index 00000000..75b32962 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py @@ -0,0 +1,204 @@ +""" +Mapeo fila CSV → datos para Part. Helpers de normalización de valores. +Clarion: ME/USD, MN/MXP, MC/ClaveMoneda; apóstrofes omitidos en NUMPARTE; RFC excepción rellena desde Class. +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, Set + + +def clean_dict(data_dict: dict) -> dict: + """Convierte strings vacíos a None; mantiene 0 en campos no-_id. Igual que a76/invoices.""" + cleaned = {} + for key, value in data_dict.items(): + if isinstance(value, str) and not value.strip(): + cleaned[key] = None + elif value == 0 and (key.endswith("_id") or key == "remesa"): + cleaned[key] = None + else: + cleaned[key] = value + return cleaned + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _bool_from_row(val: Any) -> bool: + if val is None or val == "": + return True + s = str(val).strip().upper() + if s in ("0", "F", "FALSE", "NO", "N"): + return False + return True + + +def _normalize_num_parte(val: Optional[str], max_len: int = 70) -> Optional[str]: + """NUMPARTE: mayúsculas, sin apóstrofes (Clarion: Se Omitirá el Apostrofe).""" + if val is None: + return None + s = str(val).strip().replace("'", "") + if not s: + return None + s = s.upper() + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _currency_from_row( + row_norm: Dict[str, Any], + valid_currency_codes: Set[str], +) -> tuple: + """ + Deriva currency_type y currency_key desde TIPOMONEDA (Col G) y CLAVEMONEDA (Col H). + Clarion: G vacío -> ME/USD; G=ME -> USD; G=MN -> MXP; G=MC -> H obligatoria. + """ + g = (row_norm.get("TIPOMONEDA") or row_norm.get("MONEDA") or "").strip().upper() + h = (row_norm.get("CLAVEMONEDA") or "").strip().upper() + if not g: + return "ME", "USD" + if g == "ME": + return "ME", "USD" + if g == "MN": + return "MN", "MXP" + if g == "MC" and h and h in valid_currency_codes: + return "MC", h + if h and h in valid_currency_codes: + return g[:2], h + return None, None + + +def row_to_part_data( + row_norm: Dict[str, Any], + valid_class_codes: Set[str], + valid_uom_codes: Set[str], + valid_currency_codes: Set[str], +) -> Dict[str, Any]: + """ + Mapea una fila normalizada del CSV a un diccionario de datos para Part. + Clarion: NUMPARTE sin apóstrofes; Tipo Moneda ME/MN/MC con Clave; PAIS, PREFERENCIA, SECTOR, RUTA IMAGEN. + """ + part_number = _normalize_num_parte(row_norm.get("NUMPARTE"), 70) + commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + part_class = _str_or_none(row_norm.get("CLASE"), 8) + if part_class and part_class.upper() not in valid_class_codes: + part_class = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure.upper() not in valid_uom_codes: + unit_of_measure = None + + currency_type, currency_key = _currency_from_row(row_norm, valid_currency_codes) + if currency_key and currency_key not in valid_currency_codes: + currency_key = None + currency_type = None + + unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT")) + unit_weight = _decimal_or_none(row_norm.get("PESOUNIT")) + weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6) + if weight_type and weight_type.upper() not in ("KILOS", "LIBRAS"): + weight_type = "KILOS" + fraction = _str_or_none(row_norm.get("FRACCION"), 10) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + part_photo = _str_or_none(row_norm.get("RUTAIMAGEN"), 255) + + fda_key = _str_or_none(row_norm.get("FDAKEY"), 20) + fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30) + license_code = _str_or_none(row_norm.get("LICENCIA"), 3) + eccn = _str_or_none(row_norm.get("ECCN"), 20) + export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2) + exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19) + is_active = _bool_from_row(row_norm.get("ACTIVO")) + + return { + "part_number": part_number, + "commercial_part_number": commercial, + "description_spanish": desc_es, + "description_english": desc_en, + "part_class": part_class, + "unit_of_measure": unit_of_measure, + "unit_cost": unit_cost, + "currency_type": currency_type, + "currency_key": currency_key, + "unit_weight": unit_weight, + "weight_type": weight_type, + "fraction": fraction, + "us_fraction": us_fraction, + "part_photo": part_photo, + "fda_key": fda_key, + "fcc_key": fcc_key, + "license_code": license_code, + "eccn": eccn, + "export_code": export_code, + "exclusion_symbol": exclusion_symbol, + "is_active": is_active, + } + + +def row_to_part_data_merge_existing( + row_norm: Dict[str, Any], + existing_data: Dict[str, Any], + valid_class_codes: Set[str], + valid_uom_codes: Set[str], + valid_currency_codes: Set[str], +) -> Dict[str, Any]: + """ + Para modo Actualizar (parcial): valores del CSV si no vacíos, sino los de la parte existente (Clarion VALIDA_PARCIAL_PARTES). + """ + data = row_to_part_data( + row_norm, valid_class_codes, valid_uom_codes, valid_currency_codes + ) + if not data.get("part_number"): + return data + merge_keys = ( + "description_spanish", + "description_english", + "part_class", + "unit_of_measure", + "unit_cost", + "currency_type", + "currency_key", + "unit_weight", + "weight_type", + "fraction", + "us_fraction", + "part_photo", + ) + for key in merge_keys: + val = data.get(key) + if val is None or (isinstance(val, str) and not val.strip()): + data[key] = existing_data.get(key) + return data + + +def apply_rfc_exception_from_class( + data: Dict[str, Any], + class_obj: Any, +) -> Dict[str, Any]: + """ + Clarion LLENA_PARTES: si empresa CLA/CTE y part_class informado, sobrescribir desde la clase: + UniMed, DescripcionE, DescripcionI, Fraccion, FraccionAme, TipoFraccion='GENERAL'. + """ + data["unit_of_measure"] = getattr(class_obj, "unit_of_measure", None) or data.get("unit_of_measure") + data["description_spanish"] = getattr(class_obj, "description_es", None) or data.get("description_spanish") + data["description_english"] = getattr(class_obj, "description_en", None) or data.get("description_english") + data["fraction"] = getattr(class_obj, "fraction", None) or data.get("fraction") + data["us_fraction"] = getattr(class_obj, "us_fraction", None) or data.get("us_fraction") + return data diff --git a/backend/api/v1/modules/a76/parts/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/parts/routes.py similarity index 78% rename from backend/api/v1/modules/a76/parts/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/parts/routes.py index bffb87ac..364b6ee7 100644 --- a/backend/api/v1/modules/a76/parts/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/parts/routes.py @@ -14,6 +14,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -39,6 +40,8 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo actualizar (ACT): validación parcial si la parte existe"), + reemplazar_sin_preguntar: bool = Query(True, description="Si False, solo agregar nuevas (no sobrescribir existentes)"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -59,6 +62,8 @@ async def upload_import_file( "company_id": company_id, "user_id": current_user.get("id"), "template_id": "part_numbers", + "actualizar": actualizar, + "reemplazar_sin_preguntar": reemplazar_sin_preguntar, } try: @@ -78,7 +83,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f: f.write(contents) @@ -119,6 +124,27 @@ async def get_import_status(job_id: str): if isinstance(result, dict) and result.get("status") in ("finished", "warning"): return result + # Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica + if isinstance(result, str): + try: + parsed = json.loads(result) + if isinstance(parsed, dict) and ( + parsed.get("status") == "waiting_confirmation" + or (parsed.get("job_id") and "total_rows" in parsed) + ): + return parsed + if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"): + return parsed + except (json.JSONDecodeError, TypeError): + pass + + # Si el resultado tiene forma de escaneo (waiting_confirmation), devolverlo para que el front muestre el modal + if isinstance(result, dict) and ( + result.get("status") == "waiting_confirmation" + or (result.get("job_id") and "total_rows" in result) + ): + return result + logger.warning("Parts import task %s failed: state=%s", job_id, task_result.state) err_msg = None tb = getattr(task_result, "traceback", None) diff --git a/backend/api/v1/modules/a76/parts/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/parts/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/parts/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/parts/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py b/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py new file mode 100644 index 00000000..7c878b19 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py @@ -0,0 +1,360 @@ +""" +Tareas Celery para importación CSV de Números de Parte. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Paridad Clarion: actualizar (ACT), validación full/parcial, merge existente, reemplazar_sin_preguntar, RFC desde clase. +""" +import json +import logging +import os +from typing import Dict, Any, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import csv_reader as common_csv +from ..common import meta as common_meta +from ..common import responses as common_responses +from .template_config import row_from_template, detect_headers_or_data +from .validators import validate_row_part, get_row_warnings +from .common.mappers import ( + row_to_part_data, + row_to_part_data_merge_existing, + apply_rfc_exception_from_class, +) +from .common.fk_loader import load_parts_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "part" + +# Para routes.py (upload guarda con estos prefijos) +PART_IMPORT_FILE_PREFIX = "part_import_file:" +PART_IMPORT_META_PREFIX = "part_import_meta:" +PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:" +PART_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Parts import: starting scan for job %s", job_id) + + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header) + try: + total_rows = common_csv.count_csv_rows(file_path, has_header=has_header) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + actualizar = meta.get("actualizar", False) + + ( + valid_class_codes, + valid_uom_codes, + valid_currency_codes, + valid_fraction_mex_8, + valid_country_m3, + authorized_sector_keys, + company_has_prosec, + is_rfc_exception, + ) = load_parts_fk_sets(tenant_id, company_id) + + from api.v1.modules.a76.parts.models import Part + existing_part_numbers = set() + try: + with CoreSessionLocal() as session: + for p in session.query(Part.part_number).filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ).all(): + if p[0]: + existing_part_numbers.add((p[0] or "").strip().upper()) + except Exception as e: + logger.warning("Parts import: could not load existing part numbers: %s", e) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_country_m3=valid_country_m3, + authorized_sector_keys=authorized_sector_keys, + company_has_prosec=company_has_prosec, + is_rfc_exception=is_rfc_exception, + actualizar=actualizar, + existing_part_numbers=existing_part_numbers, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + else: + for w in get_row_warnings(row_norm, i): + if len(errors_detail) < 500: + errors_detail.append({ + "line": w.get("line"), + "col": w.get("col", ""), + "msg": w.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Parts import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail + ) + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Parts import: starting commit for job %s", job_id) + + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + actualizar = meta.get("actualizar", False) + reemplazar_sin_preguntar = meta.get("reemplazar_sin_preguntar", True) + + from api.v1.modules.a76.parts.models import Part + from api.v1.modules.a76.classes.models import Class + + ( + valid_class_codes, + valid_uom_codes, + valid_currency_codes, + valid_fraction_mex_8, + valid_country_m3, + authorized_sector_keys, + company_has_prosec, + is_rfc_exception, + ) = load_parts_fk_sets(tenant_id, company_id) + + inserted_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + meta_path = common_meta.get_meta_path(file_path) + + fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header) + + try: + with CoreSessionLocal() as session: + existing_by_part_number = {} + for p in session.query(Part).filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ).all(): + key = (p.part_number or "").strip().upper() + if key: + existing_by_part_number[key] = p + + for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + part_number_raw = (row_norm.get("NUMPARTE") or "").strip().upper() + use_partial = ( + actualizar + and part_number_raw + and part_number_raw in existing_by_part_number + and reemplazar_sin_preguntar + ) + + err = validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_country_m3=valid_country_m3, + authorized_sector_keys=authorized_sector_keys, + company_has_prosec=company_has_prosec, + is_rfc_exception=is_rfc_exception, + actualizar=actualizar, + existing_part_numbers=set(existing_by_part_number.keys()), + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + if use_partial: + existing = existing_by_part_number.get(part_number_raw) + existing_data = { + "description_spanish": existing.description_spanish, + "description_english": existing.description_english, + "part_class": existing.part_class, + "unit_of_measure": existing.unit_of_measure, + "unit_cost": existing.unit_cost, + "currency_type": existing.currency_type, + "currency_key": existing.currency_key, + "unit_weight": existing.unit_weight, + "weight_type": existing.weight_type, + "fraction": existing.fraction, + "us_fraction": existing.us_fraction, + "part_photo": existing.part_photo, + } + data = row_to_part_data_merge_existing( + row_norm, + existing_data, + valid_class_codes, + valid_uom_codes, + valid_currency_codes, + ) + else: + data = row_to_part_data( + row_norm, + valid_class_codes, + valid_uom_codes, + valid_currency_codes, + ) + + part_number = data.get("part_number") + if not part_number: + skipped_invalid += 1 + continue + + if is_rfc_exception and data.get("part_class"): + class_obj = ( + session.query(Class) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + Class.class_code == (data.get("part_class") or "").strip().upper(), + ) + .first() + ) + if class_obj: + data = apply_rfc_exception_from_class(data, class_obj) + + existing = existing_by_part_number.get(part_number) + if existing: + if not reemplazar_sin_preguntar: + skipped_duplicate += 1 + continue + for key, value in data.items(): + if hasattr(existing, key): + setattr(existing, key, value) + session.add(existing) + inserted_count += 1 + else: + new_part = Part( + tenant_id=tenant_id, + company_id=company_id, + client_id=company_id, + **data, + ) + session.add(new_part) + existing_by_part_number[part_number] = new_part + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("Parts import DB error: %s", db_err) + return common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, error=str(db_err), + ) + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + response = common_responses.commit_result( + "warning", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, + message=f"No se insertaron registros. {total_skipped} rechazados.", + ) + elif inserted_count == 0: + response = common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, + error="No hay registros válidos en el archivo CSV", + ) + else: + response = common_responses.commit_result( + "finished", inserted_count, skipped_invalid, skipped_missing_fk, + skipped_duplicate, skipped_details, + ) + + except Exception as e: + logger.exception("Parts import task failed") + response = common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, error=str(e), + ) + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if response is None: + response = common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, error="Error inesperado", + ) + return response diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/template_config.py b/backend/api/v1/modules/a76/layouts_csv/parts/template_config.py new file mode 100644 index 00000000..5a5e9da7 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/template_config.py @@ -0,0 +1,142 @@ +""" +Configuración de plantilla CSV para Números de Parte (EstructuraCatPartesAF). +Cabeceras de descarga = Clarion: NUMERO DE PARTE, DESCRIPCION EN ESPAÑOL, etc. +""" +import csv +import io +from typing import Dict, List, Any, Optional, Tuple + + +# Valores que indican que la primera fila es cabecera (primera columna normalizada) +FIRST_COLUMN_HEADER_VALUES = ("NUMERO DE PARTE", "NUMPARTE") + + +def detect_headers_or_data( + file_path: str, + normalize_header_fn, + encoding: str = "utf-8-sig", +) -> Tuple[Optional[List[str]], bool]: + """ + Lee la primera línea del CSV y decide si es cabecera o dato. + - Si la primera celda normalizada está en FIRST_COLUMN_HEADER_VALUES -> has_header=True, fieldnames=None. + - Si no -> has_header=False, fieldnames=TEMPLATE_DOWNLOAD_HEADERS (primera fila = dato). + """ + try: + with open(file_path, "r", encoding=encoding) as f: + sample = f.read(2048) + except Exception: + return None, True + lines = sample.splitlines() + if not lines: + return None, True + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = csv.excel + reader = csv.reader(io.StringIO(lines[0]), dialect=dialect) + first_row = next(reader, None) + if not first_row: + return None, True + first_cell = (first_row[0] or "").strip() + first_cell_norm = normalize_header_fn(first_cell) + if first_cell_norm in FIRST_COLUMN_HEADER_VALUES: + return None, True + return list(TEMPLATE_FIELDNAMES_FOR_READING), False + + +# Cabeceras que se escriben al descargar la plantilla CSV (igual que Clarion EstructuraCatPartesAF) +# Columnas I y J vacías en Clarion: dos comas entre CLAVE MONEDA y PESO UNITARIO (espacios en blanco separados por coma) +TEMPLATE_DOWNLOAD_HEADERS: List[str] = [ + "NUMERO DE PARTE", + "DESCRIPCION EN ESPAÑOL", + "DESCRIPCION EN INGLES", + "CLASE", + "UNIDAD DE MEDIDA COMERCIAL", + "COSTO UNITARIO", + "TIPO MONEDA COSTO", + "CLAVE MONEDA", + "", # Col I vacía + "", # Col J vacía + "PESO UNITARIO", + "TIPO PESO", + "FRACCION", + "PAIS", + "PREFERENCIA", + "SECTOR", + "RUTA DE LA IMAGEN", +] + +# Para lectura cuando la primera fila es dato (sin cabecera): nombres únicos para columnas I y J +TEMPLATE_FIELDNAMES_FOR_READING: List[str] = [ + "NUMERO DE PARTE", + "DESCRIPCION EN ESPAÑOL", + "DESCRIPCION EN INGLES", + "CLASE", + "UNIDAD DE MEDIDA COMERCIAL", + "COSTO UNITARIO", + "TIPO MONEDA COSTO", + "CLAVE MONEDA", + "_COL_I_", + "_COL_J_", + "PESO UNITARIO", + "TIPO PESO", + "FRACCION", + "PAIS", + "PREFERENCIA", + "SECTOR", + "RUTA DE LA IMAGEN", +] + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "part_numbers": [ + {"canonical": "NUMPARTE", "aliases": ["NUMERO DE PARTE", "NUMERO PARTE", "PART NUMBER", "NUM PARTE"]}, + {"canonical": "NUMPARTECOM", "aliases": ["NUMERO PARTE COMERCIAL", "COMMERCIAL PART", "PARTE COMERCIAL"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION EN ESPAÑOL", "DESCRIPCION EN ESPANOL", "DESCRIPCION", "DESCRIPCION ES", "DESC ESPANOL"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN INGLES", "DESCRIPCION EN", "DESCRIPTION", "DESC INGLES"]}, + {"canonical": "CLASE", "aliases": ["CLASS", "CLASE MATERIAL", "PART CLASS"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD DE MEDIDA COMERCIAL", "UNIDAD MEDIDA", "UNIT", "UOM", "UNIT OF MEASURE"]}, + {"canonical": "COSTOUNIT", "aliases": ["COSTO UNITARIO", "UNIT COST", "COSTO"]}, + {"canonical": "TIPOMONEDA", "aliases": ["TIPO MONEDA COSTO", "TIPO MONEDA", "MONEDA"]}, + {"canonical": "CLAVEMONEDA", "aliases": ["CLAVE MONEDA", "CURRENCY", "MONEDA CLAVE", "CURRENCY KEY"]}, + {"canonical": "PESOUNIT", "aliases": ["PESO UNITARIO", "UNIT WEIGHT", "PESO"]}, + {"canonical": "TIPOPESO", "aliases": ["TIPO PESO", "WEIGHT TYPE"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, + {"canonical": "PAIS", "aliases": ["PAIS", "COUNTRY"]}, + {"canonical": "PREFERENCIA", "aliases": ["PREFERENCIA", "PREFERENCIA ARANCELARIA"]}, + {"canonical": "SECTOR", "aliases": ["SECTOR"]}, + {"canonical": "RUTAIMAGEN", "aliases": ["RUTA DE LA IMAGEN", "RUTA IMAGEN", "FOTO"]}, + {"canonical": "FDAKEY", "aliases": ["FDA", "FDA KEY"]}, + {"canonical": "FCCKEY", "aliases": ["FCC", "FCC KEY"]}, + {"canonical": "LICENCIA", "aliases": ["LICENSE CODE", "LICENSE"]}, + {"canonical": "ECCN", "aliases": ["ECCN CODE"]}, + {"canonical": "EXPORTCODE", "aliases": ["EXPORT CODE", "CODIGO EXPORT"]}, + {"canonical": "EXCLUSION", "aliases": ["EXCLUSION SYMBOL", "SIMBOLO EXCLUSION"]}, + {"canonical": "ACTIVO", "aliases": ["IS ACTIVE", "ACTIVE", "ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("part_numbers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py new file mode 100644 index 00000000..18aa9521 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py @@ -0,0 +1,4 @@ +# validators for parts CSV import row validation +from .create import validate_row_part, validate_row_part_partial, get_row_warnings + +__all__ = ["validate_row_part", "validate_row_part_partial", "get_row_warnings"] diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py new file mode 100644 index 00000000..ef48142f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py @@ -0,0 +1,404 @@ +""" +Validaciones comunes de fila para import CSV de partes. +Paridad Clarion: VALIDA_TODA_PARTES (obligatorios A, B, E salvo excepción RFC), VALIDACIONES_PARTES. +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import check_max_length, check_decimal + + +MSG_NUMPARTE_VACIO = ( + "Error: (Col. A) La columna de Número de Parte esta vacio y no se pueden hacer las validaciones. " + "Capturar en la Columna A un Número de Parte nuevo o una ya existente a la cual desee actualizar campos" +) +NUMPARTE_MAX_LEN_CLARION = 30 +NUMPARTE_MAX_LEN_DB = 70 + + +def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col A (NUMPARTE) obligatorio; mensaje Clarion si está vacío.""" + val = (row.get("NUMPARTE") or "").strip() + if not val: + return {"line": line_num, "col": "NUMPARTE", "msg": MSG_NUMPARTE_VACIO} + if len(val) > NUMPARTE_MAX_LEN_DB: + return { + "line": line_num, + "col": "NUMPARTE", + "msg": f"Error: (Col. A) El Número de Parte: {val} supera la longitud de caracteres. " + f"Capturar en la columna A un Número de Parte de {NUMPARTE_MAX_LEN_DB} caracteres como máximo.", + } + return None + + +def validate_row_required_full( + row: Dict[str, Any], + line_num: int, + is_rfc_exception: bool = False, +) -> Optional[Dict[str, Any]]: + """Obligatorios en validación completa: A, B; y E (U.M. Comercial) salvo excepción RFC con D vacía.""" + err = validate_row_required(row, line_num) + if err: + return err + if not (row.get("DESCRIPCIONE") or "").strip(): + return { + "line": line_num, + "col": "DESCRIPCIONE", + "msg": "Existen campos vacios que son obligatorios, es la (Col.B) Descripción Español. " + "Revisar la línea del archivo y capturar los campos con la información correcta.", + } + # E obligatorio salvo que empresa sea excepción RFC y Col D (Clase) esté vacía + col_d = (row.get("CLASE") or "").strip() + if not is_rfc_exception or col_d: + if not (row.get("UNIMED") or "").strip(): + return { + "line": line_num, + "col": "UNIMED", + "msg": "Existen campos vacios que son obligatorios, es la (Col.E) U.M. Comercial. " + "Revisar la línea del archivo y capturar los campos con la información correcta.", + } + return None + + +def validate_row_required_act_part_exists( + row: Dict[str, Any], + line_num: int, + existing_part_numbers: Set[str], +) -> Optional[Dict[str, Any]]: + """Cuando actualizar=True y se hace validación full: el número de parte debe existir en catálogo.""" + num_parte = (row.get("NUMPARTE") or "").strip().upper() + if not num_parte: + return None + if num_parte not in existing_part_numbers: + return { + "line": line_num, + "col": "NUMPARTE", + "msg": "(Col.A) El número de parte no existe.", + } + return None + + +def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Longitudes máximas (Clarion: Col A 30; resto según modelo).""" + err = check_max_length(row, "NUMPARTE", NUMPARTE_MAX_LEN_DB, line_num) + if err: + return err + checks = [ + ("DESCRIPCIONE", 500), + ("DESCRIPCIONI", 500), + ("CLASE", 8), + ("UNIMED", 5), + ("TIPOMONEDA", 3), + ("CLAVEMONEDA", 3), + ("FRACCION", 10), + ("PAIS", 3), + ("PREFERENCIA", 7), + ("SECTOR", 5), + ("RUTAIMAGEN", 255), + ("TIPOPESO", 6), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Tipos numéricos (decimales) de la fila.""" + err = check_decimal(row, "COSTOUNIT", line_num) + if err: + return err + err = check_decimal(row, "PESOUNIT", line_num) + if err: + return err + return None + + +def validate_row_class_fk( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col D (Clase): si no vacía, debe existir en catálogo de clases.""" + val = (row.get("CLASE") or "").strip() + if not val or valid_class_codes is None: + return None + if val.upper() in valid_class_codes: + return None + return { + "line": line_num, + "col": "CLASE", + "msg": f"Error: (Col. D) La Clase: {val} no existe en el Catálogo de Clases de Activo Fijo. " + "Dar de alta la Clase en el Catálogo de Clases de Activo Fijo.", + } + + +def validate_row_uom_fk( + row: Dict[str, Any], + line_num: int, + valid_uom_codes: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col E (U.M. Comercial): si no vacía, debe existir en catálogo UOM.""" + val = (row.get("UNIMED") or "").strip() + if not val or valid_uom_codes is None: + return None + if val.upper() in valid_uom_codes: + return None + return { + "line": line_num, + "col": "UNIMED", + "msg": f"Error: (Col. E) La Unidad de Medida Comercial: {val} no existe en el Catálogo de U.M. " + "Revisar esta Unidad de Medida en el archivo, en caso de ser correcta dar la de alta en el Catálogo de U.M.", + } + + +def validate_row_tipo_moneda( + row: Dict[str, Any], + line_num: int, +) -> Optional[Dict[str, Any]]: + """Col G (Tipo Moneda): solo ME, MN o MC (o vacío).""" + val = (row.get("TIPOMONEDA") or "").strip().upper() + if not val: + return None + if val in ("ME", "MN", "MC"): + return None + return { + "line": line_num, + "col": "TIPOMONEDA", + "msg": f"Error: (Col. G) La opción de Tipo Moneda: {val} no es valida. " + "Capturar una opción valida: ME para Dolares, MN para Pesos, MC para Moneda de Captura o dejar el campo vacio.", + } + + +def validate_row_clave_moneda( + row: Dict[str, Any], + line_num: int, + valid_currency_codes: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col H (Clave Moneda): si G = MC, H obligatoria y debe existir en catálogo.""" + g = (row.get("TIPOMONEDA") or "").strip().upper() + if g != "MC": + return None + val = (row.get("CLAVEMONEDA") or "").strip() + if not val: + return { + "line": line_num, + "col": "CLAVEMONEDA", + "msg": "Error: (Col. H) La Clave de la Moneda es obligatoria cuando Tipo Moneda es MC.", + } + if valid_currency_codes is not None and val.upper() not in valid_currency_codes: + return { + "line": line_num, + "col": "CLAVEMONEDA", + "msg": f"Error: (Col. H) La Clave de la Moneda: {val} no existe en el Catálogo de Claves de Moneda. " + "Revisar esta clave de la Moneda en el archivo, en caso de ser correcta Actualizar los Catálogos Fijos.", + } + return None + + +def validate_row_tipo_peso(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col L (Tipo Peso): solo KILOS o LIBRAS (o vacío).""" + val = (row.get("TIPOPESO") or "").strip() + if not val: + return None + if val.upper() in ("KILOS", "LIBRAS"): + return None + return { + "line": line_num, + "col": "TIPOPESO", + "msg": f"Error: (Col. L) La opción del Tipo de Peso {val} no es valida. " + "Capturar una opción valida: KILOS, LIBRAS o dejar el campo vacio y automaticamente se asigna KILOS.", + } + + +def _normalize_fraction_mex_8(value: str) -> str: + """Primeros 8 caracteres si len>=10, sino hasta 8 (Clarion SUB).""" + if not value: + return "" + v = value.strip() + if len(v) >= 10: + return v[:8] + return v[:8] if len(v) > 8 else v + + +def validate_row_fraction_mex_catalog( + row: Dict[str, Any], + line_num: int, + valid_fraction_mex_8: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col M (Fracción Mex): si no vacía, debe existir en catálogo Mex o histórico.""" + val = (row.get("FRACCION") or "").strip() + if not val or valid_fraction_mex_8 is None: + return None + code_8 = _normalize_fraction_mex_8(val) + if not code_8: + return None + if code_8 in valid_fraction_mex_8: + return None + return { + "line": line_num, + "col": "FRACCION", + "msg": f"Error: (Col. M) La Fraccion Mexicana: {val} no existe en el Catálogo de Fracciones Arancelarias Sifr@ ni en el Historico. " + "Revisar esta Fracción Arancelaria en el archivo, en caso de ser correcta Actualizar las Fracciones Arancelarias.", + } + + +def validate_row_country_fk( + row: Dict[str, Any], + line_num: int, + valid_country_m3: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col N (País): si no vacía, debe existir en catálogo de países (m3_key).""" + val = (row.get("PAIS") or "").strip() + if not val or valid_country_m3 is None: + return None + if val.upper() in valid_country_m3: + return None + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. N) La Clave del País: {val} no existe en el Catálogo de Paises. Dar lo de alta en el Catálogo de Paises.", + } + + +def validate_row_preferencia( + row: Dict[str, Any], + line_num: int, + is_rfc_exception: bool, +) -> Optional[Dict[str, Any]]: + """Col O (Preferencia): solo GENERAL, TLCS, PROSEC o ALADI. Excepción RFC: solo aplica si D no vacía.""" + col_d = (row.get("CLASE") or "").strip() + if is_rfc_exception and not col_d: + return None + val = (row.get("PREFERENCIA") or "").strip().upper() + if not val: + return None + if val in ("GENERAL", "TLCS", "PROSEC", "ALADI"): + return None + return { + "line": line_num, + "col": "PREFERENCIA", + "msg": "Error: (Col. O) La opción de Preferencia Arancelaria no es valida. " + "Capturar una opción valida: GENERAL, TLCS, PROSEC, ALADI.", + } + + +def validate_row_sector( + row: Dict[str, Any], + line_num: int, + company_has_prosec: bool, + authorized_sector_keys: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """Col P (Sector): si O=PROSEC entonces P obligatorio, empresa PROSEC, sector autorizado; si O≠PROSEC y P no vacío error.""" + pref = (row.get("PREFERENCIA") or "").strip().upper() + sector = (row.get("SECTOR") or "").strip() + if pref == "PROSEC": + if not sector: + return { + "line": line_num, + "col": "SECTOR", + "msg": "Error: (Col. P) no hay un sector capturado y se tiene la opción de Preferencia Arancelaria: PROSEC. Capturar un Sector valido en la columna P", + } + if not company_has_prosec: + return { + "line": line_num, + "col": "SECTOR", + "msg": f"Error: (Col. P) Se quiere aplicar el sector {sector} pero la empresa no cuenta con Permiso PROSEC. " + "Marcar la opción que cuenta con un Permiso PROSEC en los Datos de la Empresa.", + } + if authorized_sector_keys is not None and sector.upper() not in authorized_sector_keys: + return { + "line": line_num, + "col": "SECTOR", + "msg": f"Error: (Col. P) El sector {sector} no esta promovido para esta Empresa. " + "Capturar un Sector en la columna P que este promovido en los Datos de la Empresa o active el Sector en el Catálogo de Sectores.", + } + else: + if sector: + return { + "line": line_num, + "col": "SECTOR", + "msg": f"Error: (Col. P) Hay un sector capturado y se tiene la opción de Preferencia Arancelaria: {pref or '(vacío)'}. " + "Borra el sector en la columna P o cambiar la preferencia en la columna O.", + } + return None + + +def validaciones_parte( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_currency_codes: Optional[Set[str]] = None, + valid_fraction_mex_8: Optional[Set[str]] = None, + valid_country_m3: Optional[Set[str]] = None, + authorized_sector_keys: Optional[Set[str]] = None, + company_has_prosec: bool = False, + is_rfc_exception: bool = False, +) -> Optional[Dict[str, Any]]: + """Reglas compartidas Clarion (VALIDACIONES_PARTES): longitudes, tipos, FKs, preferencia, sector.""" + err = validate_row_lengths(row, line_num) + if err: + return err + err = validate_row_types(row, line_num) + if err: + return err + err = validate_row_class_fk(row, line_num, valid_class_codes) + if err: + return err + err = validate_row_uom_fk(row, line_num, valid_uom_codes) + if err: + return err + err = validate_row_tipo_moneda(row, line_num) + if err: + return err + err = validate_row_clave_moneda(row, line_num, valid_currency_codes) + if err: + return err + err = validate_row_tipo_peso(row, line_num) + if err: + return err + err = validate_row_fraction_mex_catalog(row, line_num, valid_fraction_mex_8) + if err: + return err + err = validate_row_country_fk(row, line_num, valid_country_m3) + if err: + return err + err = validate_row_preferencia(row, line_num, is_rfc_exception) + if err: + return err + err = validate_row_sector( + row, line_num, company_has_prosec, authorized_sector_keys + ) + if err: + return err + return None + + +def validate_row_warnings( + row: Dict[str, Any], + line_num: int, +) -> list: + """ + Devuelve lista de advertencias (no bloquean; no se añaden a error_lines). + Clarion: desfase cuando Col Q tiene valor; apóstrofes en Número de Parte. + """ + warnings: list = [] + # Advertencia desfase: Clarion cuando CSVArc:ColumnaQ <> '' + if (row.get("RUTAIMAGEN") or "").strip(): + warnings.append({ + "line": line_num, + "col": "RUTAIMAGEN", + "msg": "Advertencia: Podría existir un desfase en esta línea.", + "severity": "warning", + }) + # Advertencia apóstrofes en Número de Parte + num_parte = (row.get("NUMPARTE") or "").strip() + if "'" in num_parte: + warnings.append({ + "line": line_num, + "col": "NUMPARTE", + "msg": f"Advertencia: El Número de Parte: {num_parte} Contiene Apostrofes. Se Omitirá el Apostrofe para Subir el Número de Parte.", + "severity": "warning", + }) + return warnings diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py new file mode 100644 index 00000000..6ab21b04 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py @@ -0,0 +1,116 @@ +""" +Punto de entrada de validación para import de una fila de parte. +Flujo Clarion: no ACT → siempre VALIDA_TODA_PARTES; ACT y parte existe → VALIDA_PARCIAL_PARTES; +ACT y parte no existe → VALIDA_TODA_PARTES (obligatorios + validaciones) y en commit se crea (ADD). +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_required, + validate_row_required_full, + validaciones_parte, + validate_row_warnings, +) + + +def validate_row_part( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_currency_codes: Optional[Set[str]] = None, + valid_fraction_mex_8: Optional[Set[str]] = None, + valid_country_m3: Optional[Set[str]] = None, + authorized_sector_keys: Optional[Set[str]] = None, + company_has_prosec: bool = False, + is_rfc_exception: bool = False, + actualizar: bool = False, + existing_part_numbers: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de partes para import. + - No actualizar: siempre validación completa (obligatorios A,B,E + validaciones_parte). + - Actualizar y número de parte ya existe: validación parcial (solo validaciones_parte). + - Actualizar y número de parte no existe: validación completa; si pasa, en commit se crea (ADD). + Si actualizar y validación full, además exige que el número de parte exista en catálogo. + """ + err = validate_row_required(row, line_num) + if err: + return err + + part_number = (row.get("NUMPARTE") or "").strip().upper() + use_partial = ( + actualizar + and existing_part_numbers is not None + and part_number in existing_part_numbers + ) + + if use_partial: + return validate_row_part_partial( + row, + line_num, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_country_m3=valid_country_m3, + authorized_sector_keys=authorized_sector_keys, + company_has_prosec=company_has_prosec, + is_rfc_exception=is_rfc_exception, + ) + + # Full validation (también cuando actualizar y parte no existe → ADD en commit) + err = validate_row_required_full(row, line_num, is_rfc_exception=is_rfc_exception) + if err: + return err + # No exigir que exista en actualizar: si no existe se valida completa y en commit se crea (ADD) + return validaciones_parte( + row, + line_num, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_country_m3=valid_country_m3, + authorized_sector_keys=authorized_sector_keys, + company_has_prosec=company_has_prosec, + is_rfc_exception=is_rfc_exception, + ) + + +def validate_row_part_partial( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_currency_codes: Optional[Set[str]] = None, + valid_fraction_mex_8: Optional[Set[str]] = None, + valid_country_m3: Optional[Set[str]] = None, + authorized_sector_keys: Optional[Set[str]] = None, + company_has_prosec: bool = False, + is_rfc_exception: bool = False, +) -> Optional[Dict[str, Any]]: + """ + Validación parcial (modo Actualizar, parte existente): solo NUMPARTE + validaciones_parte. + Los campos vacíos se rellenan desde la parte existente en el mapper. + """ + err = validate_row_required(row, line_num) + if err: + return err + return validaciones_parte( + row, + line_num, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + valid_fraction_mex_8=valid_fraction_mex_8, + valid_country_m3=valid_country_m3, + authorized_sector_keys=authorized_sector_keys, + company_has_prosec=company_has_prosec, + is_rfc_exception=is_rfc_exception, + ) + + +def get_row_warnings(row: Dict[str, Any], line_num: int) -> list: + """Devuelve lista de advertencias para la fila (desfase, apóstrofes). No bloquean.""" + return validate_row_warnings(row, line_num) diff --git a/backend/api/v1/modules/a76/pedmientos/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/pedmientos/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/pedmientos/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py new file mode 100644 index 00000000..47ef2168 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers, fk_loader diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py new file mode 100644 index 00000000..3f9609ba --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py @@ -0,0 +1,150 @@ +""" +Validadores reutilizables para import CSV de pedimentos. +Incluye parseo de fechas (paridad exchange_rate) para FECHA_INICIO, FECHA_FINAL, FECHA_PAGO. +""" +from datetime import datetime, time +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, Set, List + +DATE_FORMATS: List[str] = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] +DATE_FORMAT_PREFERENCE_MAP: Dict[str, str] = { + "dd/mm/yyyy": "%d/%m/%Y", + "mm/dd/yyyy": "%m/%d/%Y", + "yyyy-mm-dd": "%Y-%m-%d", +} + + +def parse_date( + val: Optional[str], date_format_preference: Optional[str] = None +) -> Optional[datetime]: + """Parsea fecha; si date_format_preference está definido, solo se acepta ese formato.""" + if not val or not str(val).strip(): + return None + raw = str(val).strip() + if date_format_preference and date_format_preference in DATE_FORMAT_PREFERENCE_MAP: + fmt = DATE_FORMAT_PREFERENCE_MAP[date_format_preference] + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + return None + for fmt in DATE_FORMATS: + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + continue + return None + + +def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_int_in_set( + row: Dict[str, Any], col: str, valid_ids: Set[int], line_num: int +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + try: + client_id = int(val) + except ValueError: + return {"line": line_num, "col": col, "msg": "Debe ser número entero"} + if client_id not in valid_ids: + return {"line": line_num, "col": col, "msg": "Cliente no existe en catálogo"} + return None + + +def check_optional_int_in_set( + row: Dict[str, Any], col: str, valid_ids: Set[int], line_num: int +) -> Optional[Dict[str, Any]]: + """Si la columna viene vacía no se valida; si trae valor debe ser entero y estar en valid_ids.""" + val = (row.get(col) or "").strip() + if not val: + return None + try: + client_id = int(val) + except ValueError: + return {"line": line_num, "col": col, "msg": "Debe ser número entero"} + if valid_ids and client_id not in valid_ids: + return {"line": line_num, "col": col, "msg": "Cliente no existe en catálogo"} + return None + + +def check_optional_short_name( + row: Dict[str, Any], + col: str, + short_name_to_id: Dict[str, int], + line_num: int, +) -> Optional[Dict[str, Any]]: + """Si la columna viene vacía no se valida; si trae valor debe ser short_name existente en catálogo. Solo string, no ID.""" + val = (row.get(col) or "").strip() + if not val: + return None + key = val.upper() + if key not in short_name_to_id: + return { + "line": line_num, + "col": col, + "msg": "Cliente no encontrado (short_name no existe en catálogo)", + } + return None + + +def check_in_set( + row: Dict[str, Any], + col: str, + valid_set: Set[str], + line_num: int, + max_len: int = 10, + catalog_name: str = "catálogo", +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + if valid_set and val not in valid_set: + return {"line": line_num, "col": col, "msg": f"No existe en {catalog_name}"} + return None + + +def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_optional_decimal(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + try: + Decimal(val.replace(",", ".")) + except (InvalidOperation, ValueError): + return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} + return None + + +def check_optional_in_set( + row: Dict[str, Any], col: str, allowed: Set[str], line_num: int, msg: str +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip().lower() + if not val: + return None + if val not in allowed: + return {"line": line_num, "col": col, "msg": msg} + return None + + +PEDIMENTO_CODE_MAX = 2 +REGIMEN_MAX = 3 diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py new file mode 100644 index 00000000..cb55fefe --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py @@ -0,0 +1,149 @@ +""" +Carga de conjuntos FK para validación de import CSV de pedimentos. +Clarion: clave+regimen+tipo (CodePedimentoRegimen), aduana (CustomsSection), patente (CustomsBroker.license), existing_pedimento_keys. +Legacy: valid_client_ids, valid_regimes, valid_pedimento_codes. Cliente por short_name: short_name_to_id. +""" +from typing import Dict, Set, Tuple + +from sqlalchemy.orm import Session + + +def _pedimento_key(year: str, customs_office: str, license_val: str, pedimento_number: str) -> str: + """Clave única para identificar un pedimento (tenant/company se filtra en query).""" + return f"{year}|{customs_office}|{license_val}|{pedimento_number}" + + +def load_pedimentos_fk_sets( + session: Session, tenant_id: int, company_id: int +) -> Tuple[ + Set[int], + Set[str], + Set[str], + Set[Tuple[str, str, str]], + Set[str], + Set[str], + Set[str], + Set[str], + Dict[str, int], +]: + """ + Carga todos los conjuntos FK para validación Clarion y legacy. + Returns: + valid_client_ids, + valid_regimes, + valid_pedimento_codes, + valid_clave_regimen_tipo, # (pedimento_code, regimen_code, type_code) + valid_aduana_seccion, # customs_code + existing_pedimento_keys, # key strings para actualizar + valid_anexo22_claves, # stub vacío hasta tener catálogo + valid_patentes, # CustomsBroker.license (tenant/company) + short_name_to_id, # short_name normalizado (upper) -> client id (primera aparición gana) + """ + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, + ) + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + valid_clave_regimen_tipo: Set[Tuple[str, str, str]] = set() + valid_aduana_seccion: Set[str] = set() + existing_pedimento_keys: Set[str] = set() + valid_anexo22_claves: Set[str] = set() + valid_patentes: Set[str] = set() + short_name_to_id: Dict[str, int] = {} + + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .order_by(ClientProvider.id) + .all() + ): + valid_client_ids.add(cp.id) + sn = (cp.short_name or "").strip() + if sn: + key = sn.upper() + if key not in short_name_to_id: + short_name_to_id[key] = cp.id + + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + + for cpr in session.query(CodePedimentoRegimen).all(): + # type_code puede ser None; Clarion usa I/E. Normalizar a mayúsculas para coincidir con CSV. + t = (cpr.type_code or "").strip().upper() + if t: + valid_clave_regimen_tipo.add( + ( + (cpr.pedimento_code or "").strip().upper(), + (cpr.regimen_code or "").strip().upper(), + t, + ) + ) + + for cs in session.query(CustomsSection).all(): + valid_aduana_seccion.add(cs.customs_code.strip()) + + for cb in ( + session.query(CustomsBroker) + .filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ) + .all() + ): + if (cb.license or "").strip(): + valid_patentes.add((cb.license or "").strip()) + + for p in ( + session.query( + Pedimentos.year, + Pedimentos.customs_office, + Pedimentos.license, + Pedimentos.pedimento_number, + ) + .filter( + Pedimentos.tenant_id == tenant_id, + Pedimentos.company_id == company_id, + ) + .all() + ): + existing_pedimento_keys.add( + _pedimento_key( + (p.year or "").strip(), + (p.customs_office or "").strip(), + (p.license or "").strip(), + (p.pedimento_number or "").strip(), + ) + ) + + return ( + valid_client_ids, + valid_regimes, + valid_pedimento_codes, + valid_clave_regimen_tipo, + valid_aduana_seccion, + existing_pedimento_keys, + valid_anexo22_claves, + valid_patentes, + short_name_to_id, + ) + + +def pedimento_key_from_parsed( + year: str, customs_office: str, license_val: str, pedimento_number: str +) -> str: + """Construye la clave para comparar con existing_pedimento_keys.""" + return _pedimento_key(year, customs_office, license_val, pedimento_number) diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py new file mode 100644 index 00000000..3975f19a --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py @@ -0,0 +1,187 @@ +""" +Mapeo fila CSV → datos para PedimentosCreate. +Soporta layout Clarion (PEDIMENTO, TIPO_OPERACION, CLAVE_PEDIMENTO, etc.) y legacy (AÑO, ADUANA, PATENTE, NUMERO). +""" +from datetime import datetime +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional + +from ..template_config import is_clarion_layout, parse_pedimento_col_a +from ..common.common_validators import parse_date + + +def parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + return Decimal(str(val).strip().replace(",", ".")) + except (InvalidOperation, ValueError): + return None + + +def resolve_client_id_from_short_name( + valor: str, short_name_to_id: Dict[str, int] +) -> Optional[int]: + """Resuelve short_name (string) a client_id. Vacío → None. Misma normalización que fk_loader (strip + upper).""" + if not valor or not str(valor).strip(): + return None + key = str(valor).strip().upper() + return short_name_to_id.get(key) + + +def _row_to_pedimento_data_clarion( + row_norm: Dict[str, Any], + short_name_to_id: Dict[str, int], + date_format_preference: Optional[str] = None, +) -> Dict[str, Any]: + """Desde layout Clarion: Cols 1-3 (AÑO, PATENTE, NUMERO) o Col A (PEDIMENTO); Col H → customs_office; B,C,D,J y fechas. CLIENTE_SHORT_NAME opcional.""" + año = (row_norm.get("AÑO") or "").strip() + patente = (row_norm.get("PATENTE") or "").strip() + numero = (row_norm.get("NUMERO") or "").strip() + if año and patente and numero: + year = año[:2] + license_val = patente[:4] + pedimento_number = numero[:7] + else: + ped = (row_norm.get("PEDIMENTO") or "").strip() + parsed = parse_pedimento_col_a(ped) + if not parsed: + raise ValueError("PEDIMENTO inválido o vacío") + year, license_val, pedimento_number = parsed + aduana = (row_norm.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] + clave = (row_norm.get("CLAVE_PEDIMENTO") or "").strip()[:2] + regime = (row_norm.get("REGIMEN") or "").strip()[:3] + client_short_name = (row_norm.get("CLIENTE_SHORT_NAME") or "").strip() + client_id = resolve_client_id_from_short_name(client_short_name, short_name_to_id) + + data: Dict[str, Any] = { + "year": year, + "customs_office": aduana, + "license": license_val, + "pedimento_number": pedimento_number, + "pedimento_code": clave, + "regime": regime, + } + if client_id is not None: + data["client_id"] = client_id + + tipo = (row_norm.get("TIPO_OPERACION") or "").strip().upper() + if tipo == "I": + data["operation_type"] = "imp" + elif tipo == "E": + data["operation_type"] = "exp" + + ind_con = (row_norm.get("INDIVIDUAL_CONSOLIDADO") or "").strip().upper() or "CON" + data["pedimento_type"] = "normal" if ind_con == "IND" else "consolidated" + + status = (row_norm.get("ESTATUS") or "").strip() + if status: + data["status"] = status[:30] + + data["usd_value"] = parse_decimal(row_norm.get("VALOR_USD")) + data["paid_price"] = parse_decimal(row_norm.get("PRECIO_PAGADO")) + data["gross_weight"] = parse_decimal(row_norm.get("PESO_BRUTO")) + data["exchange_rate"] = parse_decimal(row_norm.get("TIPO_CAMBIO")) + obs = (row_norm.get("OBSERVACIONES") or "").strip() + if obs: + data["observations"] = obs + + # Fechas E, F, G → pedimento_dates + start_str = (row_norm.get("FECHA_INICIO") or "").strip() + end_str = (row_norm.get("FECHA_FINAL") or "").strip() + payment_str = (row_norm.get("FECHA_PAGO") or "").strip() + base = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + start_dt = parse_date(start_str, date_format_preference) if start_str else base + end_dt = parse_date(end_str, date_format_preference) if end_str else base + payment_dt = parse_date(payment_str, date_format_preference) if payment_str else base + if start_dt and end_dt: + data["pedimento_dates"] = { + "entry_date": start_dt, + "end_date": end_dt, + "start_date": start_dt, + "payment_date": payment_dt, + } + return data + + +def _row_to_pedimento_data_legacy( + row_norm: Dict[str, Any], short_name_to_id: Dict[str, int] +) -> Dict[str, Any]: + """Layout legacy: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_SHORT_NAME, CODIGO_PEDIMENTO/CLAVE_PEDIMENTO, REGIMEN.""" + year = (row_norm.get("AÑO") or "").strip()[:2] + customs_office = (row_norm.get("ADUANA_SECCION_CRUCE") or row_norm.get("ADUANA") or "").strip()[:3] + license_val = (row_norm.get("PATENTE") or "").strip()[:4] + pedimento_number = (row_norm.get("NUMERO") or "").strip()[:7] + client_short_name = (row_norm.get("CLIENTE_SHORT_NAME") or "").strip() + client_id = resolve_client_id_from_short_name(client_short_name, short_name_to_id) + pedimento_code = (row_norm.get("CLAVE_PEDIMENTO") or row_norm.get("CODIGO_PEDIMENTO") or "").strip()[:2] + regime = (row_norm.get("REGIMEN") or "").strip()[:3] + + data = { + "year": year, + "customs_office": customs_office, + "license": license_val, + "pedimento_number": pedimento_number, + "client_id": client_id, + "pedimento_code": pedimento_code, + "regime": regime, + } + + op = (row_norm.get("TIPO_OPERACION") or "").strip().lower() + if op in ("imp", "exp"): + data["operation_type"] = op + ptype = (row_norm.get("TIPO_PEDIMENTO") or "").strip().lower() + if ptype in ("normal", "consolidated", "complementary", "automobile"): + data["pedimento_type"] = ptype + status = (row_norm.get("ESTATUS") or "").strip() + if status: + data["status"] = status[:30] + data["usd_value"] = parse_decimal(row_norm.get("VALOR_USD")) + data["paid_price"] = parse_decimal(row_norm.get("PRECIO_PAGADO")) + data["gross_weight"] = parse_decimal(row_norm.get("PESO_BRUTO")) + data["exchange_rate"] = parse_decimal(row_norm.get("TIPO_CAMBIO")) + obs = (row_norm.get("OBSERVACIONES") or "").strip() + if obs: + data["observations"] = obs + return data + + +def row_to_pedimento_data( + row_norm: Dict[str, Any], + short_name_to_id: Dict[str, int], + date_format_preference: Optional[str] = None, +) -> Dict[str, Any]: + """ + Build dict for PedimentosCreate from normalized CSV row. + Usa layout Clarion si existe PEDIMENTO; si no, layout legacy. Cliente por short_name → client_id. + """ + if is_clarion_layout(row_norm): + return _row_to_pedimento_data_clarion( + row_norm, short_name_to_id, date_format_preference + ) + return _row_to_pedimento_data_legacy(row_norm, short_name_to_id) + + +def row_to_pedimento_data_merge_existing( + row_norm: Dict[str, Any], + existing: Dict[str, Any], + short_name_to_id: Dict[str, int], + date_format_preference: Optional[str] = None, +) -> Dict[str, Any]: + """ + Para modo actualizar: campos vacíos en la fila se rellenan con el pedimento existente. + """ + if is_clarion_layout(row_norm): + new_data = _row_to_pedimento_data_clarion( + row_norm, short_name_to_id, date_format_preference + ) + else: + new_data = _row_to_pedimento_data_legacy(row_norm, short_name_to_id) + + for key, val in new_data.items(): + if val is None or (isinstance(val, str) and not val.strip()): + if key in existing and existing[key] is not None: + new_data[key] = existing[key] + if not new_data.get("pedimento_dates") and existing.get("pedimento_dates"): + new_data["pedimento_dates"] = existing["pedimento_dates"] + return new_data diff --git a/backend/api/v1/modules/a76/pedmientos/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py similarity index 84% rename from backend/api/v1/modules/a76/pedmientos/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py index f624e5dc..deb0b9d9 100644 --- a/backend/api/v1/modules/a76/pedmientos/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py @@ -10,20 +10,21 @@ from uuid import uuid4 from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends from sqlalchemy.orm import Session -from typing import Dict, Any +from typing import Dict, Any, Optional from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - PED_IMPORT_FILE_PREFIX, - PED_IMPORT_META_PREFIX, + JOB_TYPE as PED_JOB_TYPE, PED_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage router = APIRouter() logger = logging.getLogger(__name__) @@ -39,6 +40,8 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo actualizar: validación parcial si el pedimento existe"), + dateFormat: Optional[str] = Query(None, description="Formato de fecha: dd/mm/yyyy, mm/dd/yyyy, yyyy-mm-dd"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -57,22 +60,26 @@ async def upload_import_file( job_id = str(uuid4()) contents = await file.read() - meta_data = { + file_key, meta_key, _ = common_storage.storage_keys(PED_JOB_TYPE, job_id) + meta_data: Dict[str, Any] = { "tenant_id": tenant_id, "company_id": company_id, "user_id": current_user.get("id"), "template_id": "pedimentos", + "actualizar": actualizar, } + if dateFormat: + meta_data["dateFormat"] = dateFormat try: r = _get_redis() r.set( - f"{PED_IMPORT_FILE_PREFIX}{job_id}", + file_key, base64.b64encode(contents), ex=PED_IMPORT_REDIS_TTL, ) r.set( - f"{PED_IMPORT_META_PREFIX}{job_id}", + meta_key, json.dumps(meta_data).encode("utf-8"), ex=PED_IMPORT_REDIS_TTL, ) @@ -81,11 +88,13 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f: + csv_path = common_storage.file_path_for_job(PED_JOB_TYPE, job_id) + with open(csv_path, "wb") as f: f.write(contents) - with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f: + meta_path = csv_path.replace(".csv", ".meta.json") + with open(meta_path, "w") as f: json.dump(meta_data, f) except Exception as e: logger.warning(f"Pedimentos import: local file save failed: {e}") diff --git a/backend/api/v1/modules/a76/pedmientos/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/pedmientos/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/pedmientos/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py new file mode 100644 index 00000000..18ddb766 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py @@ -0,0 +1,414 @@ +""" +Tareas Celery para importación CSV de Pedimentos. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader), fk_loader, validators, mappers. +""" +import json +import logging +import os +from datetime import datetime +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import ( + row_from_template, + is_clarion_layout, + parse_pedimento_col_a, + detect_headers_or_data, +) +from .validators import validate_row_pedimento +from .common.mappers import row_to_pedimento_data, row_to_pedimento_data_merge_existing +from .common.fk_loader import load_pedimentos_fk_sets, pedimento_key_from_parsed + +logger = logging.getLogger(__name__) + +JOB_TYPE = "ped" +TEMPLATE_ID = "pedimentos" + +# Para routes.py +PED_IMPORT_FILE_PREFIX = "ped_import_file:" +PED_IMPORT_META_PREFIX = "ped_import_meta:" +PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" +PED_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: + return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + if os.path.getsize(file_path) == 0: + return {"status": "failed", "error": "El archivo está vacío. Verifica que el CSV tenga contenido."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + fieldnames, has_header = detect_headers_or_data( + file_path, + common_normalize.normalize_header, + parse_pedimento_col_a, + ) + try: + total_rows = common_csv_reader.count_csv_rows(file_path, has_header=has_header) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + date_format_preference = meta.get("dateFormat") or meta.get("date_format") + + try: + with CoreSessionLocal() as session: + ( + valid_client_ids, + valid_regimes, + valid_pedimento_codes, + valid_clave_regimen_tipo, + valid_aduana_seccion, + existing_pedimento_keys, + valid_anexo22_claves, + valid_patentes, + short_name_to_id, + ) = load_pedimentos_fk_sets(session, tenant_id, company_id) + except Exception as e: + logger.error("Pedimentos import: failed to load FK sets: %s", e) + return {"status": "failed", "error": "No se pudo cargar catálogos"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = _norm_row(row) + err = validate_row_pedimento( + row_norm, + i, + short_name_to_id, + valid_regimes, + valid_pedimento_codes, + valid_clave_regimen_tipo, + valid_aduana_seccion, + existing_pedimento_keys, + valid_anexo22_claves, + valid_patentes, + actualizar=actualizar, + raw_row=row, + date_format_preference=date_format_preference, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Pedimentos import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Pedimentos import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + date_format_preference = meta.get("dateFormat") or meta.get("date_format") + + try: + with CoreSessionLocal() as session: + ( + valid_client_ids, + valid_regimes, + valid_pedimento_codes, + valid_clave_regimen_tipo, + valid_aduana_seccion, + existing_pedimento_keys, + valid_anexo22_claves, + valid_patentes, + short_name_to_id, + ) = load_pedimentos_fk_sets(session, tenant_id, company_id) + except Exception as e: + logger.error("Pedimentos import: failed to load FK sets: %s", e) + return {"status": "failed", "error": "No se pudo cargar catálogos"} + + from api.v1.modules.a76.pedmientos.dtos.pedimentos import ( + PedimentosCreate, + PedimentosUpdate, + ) + from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate + from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService + from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + meta_path = common_meta.get_meta_path(file_path) + + fieldnames_commit, _ = detect_headers_or_data( + file_path, + common_normalize.normalize_header, + parse_pedimento_col_a, + ) + + def _key_from_row(r: Dict[str, Any]) -> Optional[str]: + if is_clarion_layout(r): + año = (r.get("AÑO") or "").strip() + patente = (r.get("PATENTE") or "").strip() + numero = (r.get("NUMERO") or "").strip() + if año and patente and numero: + adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] + return pedimento_key_from_parsed(año[:2], adu, patente[:4], numero[:7]) + parsed = parse_pedimento_col_a((r.get("PEDIMENTO") or "").strip()) + if not parsed: + return None + y, lic, num = parsed + adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] + return pedimento_key_from_parsed(y, adu, lic, num) + y = (r.get("AÑO") or "").strip()[:2] + adu = (r.get("ADUANA_SECCION_CRUCE") or r.get("ADUANA") or "").strip()[:3] + lic = (r.get("PATENTE") or "").strip()[:4] + num = (r.get("NUMERO") or "").strip()[:7] + return pedimento_key_from_parsed(y, adu, lic, num) + + try: + with CoreSessionLocal() as session: + for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames_commit): + if i in error_lines: + continue + + row_norm = _norm_row(row) + err = validate_row_pedimento( + row_norm, + i, + short_name_to_id, + valid_regimes, + valid_pedimento_codes, + valid_clave_regimen_tipo, + valid_aduana_seccion, + existing_pedimento_keys, + valid_anexo22_claves, + valid_patentes, + actualizar=actualizar, + raw_row=row, + date_format_preference=date_format_preference, + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + key = _key_from_row(row_norm) + try: + # Si el pedimento ya existe: actualizar (merge) o reemplazar (crear). Si no existe: crear. + if key and key in existing_pedimento_keys: + existing = ( + session.query(Pedimentos) + .filter( + Pedimentos.tenant_id == tenant_id, + Pedimentos.company_id == company_id, + Pedimentos.year == key.split("|")[0], + Pedimentos.customs_office == key.split("|")[1], + Pedimentos.license == key.split("|")[2], + Pedimentos.pedimento_number == key.split("|")[3], + ) + .first() + ) + if existing: + if actualizar: + # Modo actualizar: merge con existente + existing_dict = { + "year": existing.year, + "customs_office": existing.customs_office, + "license": existing.license, + "pedimento_number": existing.pedimento_number, + "client_id": existing.client_id, + "pedimento_code": existing.pedimento_code, + "regime": existing.regime, + "operation_type": existing.operation_type, + "pedimento_type": existing.pedimento_type, + "status": existing.status, + "usd_value": existing.usd_value, + "paid_price": existing.paid_price, + "gross_weight": existing.gross_weight, + "exchange_rate": existing.exchange_rate, + "observations": existing.observations, + } + if existing.pedimento_dates: + existing_dict["pedimento_dates"] = { + "entry_date": existing.pedimento_dates.entry_date, + "end_date": existing.pedimento_dates.end_date, + "start_date": getattr( + existing.pedimento_dates, "start_date", existing.pedimento_dates.entry_date + ), + "payment_date": getattr( + existing.pedimento_dates, "payment_date", existing.pedimento_dates.entry_date + ), + } + data = row_to_pedimento_data_merge_existing( + row_norm, + existing_dict, + short_name_to_id, + date_format_preference, + ) + else: + # Modo crear: reemplazar con datos del CSV + data = row_to_pedimento_data( + row_norm, short_name_to_id, date_format_preference + ) + if "pedimento_dates" not in data or data.get("pedimento_dates") is None: + data["pedimento_dates"] = PedimentoDatesCreate( + entry_date=datetime.now(), + end_date=datetime.now(), + ) + update_data = {k: v for k, v in data.items() if k != "pedimento_dates"} + if data.get("pedimento_dates"): + update_data["pedimento_dates"] = PedimentoDatesCreate( + **data["pedimento_dates"] + ) + update_schema = PedimentosUpdate(**update_data) + PedimentosService.update( + session, existing.id, tenant_id, update_schema, company_id + ) + updated_count += 1 + else: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "Pedimento no encontrado"}) + else: + # No existe: crear (ambos modos) + data = row_to_pedimento_data( + row_norm, short_name_to_id, date_format_preference + ) + if "pedimento_dates" not in data or data.get("pedimento_dates") is None: + data["pedimento_dates"] = PedimentoDatesCreate( + entry_date=datetime.now(), + end_date=datetime.now(), + ) + create_data = PedimentosCreate(**data) + PedimentosService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except ValueError as ve: + if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): + skipped_duplicate += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + else: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + except Exception as e: + logger.warning("Pedimentos import line %s: %s", i, e) + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(e)}) + + except Exception as e: + logger.exception("Pedimentos import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and updated_count == 0 and total_skipped > 0: + reasons = "; ".join( + f"Línea {d.get('line', '?')}: {d.get('reason', '')}" for d in skipped_details[:5] + ) + if len(skipped_details) > 5: + reasons += f" (+{len(skipped_details) - 5} más)" + return { + "status": "warning", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados. Motivos: {reasons}", + } + if inserted_count == 0 and updated_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Pedimentos import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/template_config.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/template_config.py new file mode 100644 index 00000000..ffea72b9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/template_config.py @@ -0,0 +1,219 @@ +""" +Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls). +Layout Clarion: Cols 1-3 = AÑO (2), PATENTE (4), NUMERO (7); luego TIPO, CLAVE PEDIMENTO, REGIMEN, +fechas, ADUANA Y SECCION CRUCE, etc. Compatibilidad: PEDIMENTO (##-####-#######) y layout legacy. +""" +import csv +import io +from typing import Dict, List, Any, Optional, Tuple + +# Longitudes para validación (sin afectar modelos) +AÑO_LEN = 2 +PATENTE_LEN = 4 +NUMERO_LEN = 7 + +# Columnas Clarion (canónicos) + aliases exactos de la plantilla y legacy +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "pedimentos": [ + # Cols 1-3: AÑO, PATENTE, NUMERO (reemplazan Col A única PEDIMENTO) + {"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]}, + {"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]}, + {"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER"]}, + # Col B + {"canonical": "TIPO_OPERACION", "aliases": ["TIPO MOV(I=Importación,E=Exportación)", "TIPO MOV(I=Impotación,E=Expotación)", "TIPO MOV", "TIPO", "OPERATION_TYPE", "OPERACION"]}, + # Col C + {"canonical": "CLAVE_PEDIMENTO", "aliases": ["CLAVE PEDIMENTO", "CODIGO_PEDIMENTO", "PEDIMENTO_CODE", "CODIGO"]}, + # Col D + {"canonical": "REGIMEN", "aliases": ["REGIME", "CLAVE RÉGIMEN"]}, + # Col E, F, G + {"canonical": "FECHA_INICIO", "aliases": ["FECHA INICIO", "FECHA INICIAL"]}, + {"canonical": "FECHA_FINAL", "aliases": ["FECHA FINAL", "FECHA FIN"]}, + {"canonical": "FECHA_PAGO", "aliases": ["FECHA DE PAGO", "FECHA PAGO"]}, + # Col H + {"canonical": "ADUANA_SECCION_CRUCE", "aliases": ["ADUANA Y SECCION DE CRUCE", "ADUANA Y SECCION CRUCE", "ADUANA", "CUSTOMS_OFFICE", "CUSTOMS OFFICE"]}, + # Col I + {"canonical": "ACUSE_ELECTRONICO", "aliases": ["ACUSE ELECTRONICO", "ACUSE ELECTRÓNICO"]}, + # Col J + {"canonical": "INDIVIDUAL_CONSOLIDADO", "aliases": ["INDIVIDUAL o CONSOLIDADO (IND,CON)", "INDIVIDUAL o CONSOLIDADO", "IND/CON", "INDIVIDUAL CONSOLIDADO"]}, + # Col K, L, M + {"canonical": "MET_TRANSP_ENTRADA", "aliases": ["MET TRANS ENTRADA", "METODO TRANSP ENTRADA"]}, + {"canonical": "MET_TRANSP_ARRIVO", "aliases": ["MET TRANS ARRIVO", "METODO TRANSP ARRIVO"]}, + {"canonical": "MET_TRANSP_SALIDA", "aliases": ["MET TRANS SALIDA", "METODO TRANSP SALIDA"]}, + # Col N (desfase) y resto de columnas de la plantilla + {"canonical": "IEPS", "aliases": []}, + {"canonical": "DTA", "aliases": []}, + {"canonical": "CNT", "aliases": []}, + {"canonical": "PREVALIDACION", "aliases": []}, + {"canonical": "MONTO_TIGIE", "aliases": ["MONTO TIGIE"]}, + {"canonical": "PAGO_IMPUESTO", "aliases": ["PAGO IMPUESTO? (S/N)"]}, + {"canonical": "ES_MIXTO", "aliases": ["ES MIXTO (SI/NO)"]}, + {"canonical": "OBS_RECTIFICA", "aliases": ["OBS RECTIFICA"]}, + {"canonical": "OPCION_DESTINO", "aliases": ["OPCION DESTINO(Interior del Pais/Región Fronteriza/Franja Fronteriza)", "OPCION DESTINO"]}, + {"canonical": "VALOR_IVA", "aliases": ["VALOR IVA"]}, + {"canonical": "VALOR_ME", "aliases": ["VALOR ME"]}, + {"canonical": "VALOR_ADUANAS", "aliases": ["VALOR ADUANAS"]}, + {"canonical": "FLETE", "aliases": []}, + {"canonical": "VALOR_SEGUROS", "aliases": ["VALOR SEGUROS"]}, + {"canonical": "SEGUROS", "aliases": []}, + {"canonical": "EMBALAJES", "aliases": []}, + {"canonical": "OTROS_INCREMENTABLES", "aliases": ["OTROS INCREMENTABLES"]}, + {"canonical": "ESTATUS", "aliases": ["ESTATUS (ABIERTO/CERRADO)", "STATUS", "ESTADO"]}, + {"canonical": "PERSONA_REV", "aliases": ["PERSONA REV"]}, + {"canonical": "FECHA_CIERRE", "aliases": ["FECHA CIERRE"]}, + {"canonical": "FECHA_REVISION", "aliases": ["FECHA REVISION"]}, + {"canonical": "FECHA_AUTORIZACION", "aliases": ["FECHA AUTORIZACION"]}, + {"canonical": "FECHA_RECIBIDO", "aliases": ["FECHA RECIBIDO"]}, + {"canonical": "REPRESENTANTE_AA", "aliases": ["REPRESENTANTE AA"]}, + {"canonical": "CLAVE_DEST_ORIGEN", "aliases": ["CLAVE DEST ORIGEN"]}, + {"canonical": "FECHA_ENTRADA_RECINTO", "aliases": ["FECHA ENTRADA RECINTO"]}, + {"canonical": "FECHA_EXTRACCION_RECINTO", "aliases": ["FECHA EXTRACCION RECINTO"]}, + {"canonical": "ERRORES", "aliases": []}, + {"canonical": "FORMA_PAGO_DTA", "aliases": ["FORMA PAGO DTA"]}, + {"canonical": "FORMA_PAGO_IGI", "aliases": ["FORMA PAGO IGI"]}, + {"canonical": "FORMA_PAGO_PREVAL", "aliases": ["FORMA PAGO PREVAL"]}, + {"canonical": "FORMA_PAGO_IVA", "aliases": ["FORMA PAGO IVA"]}, + {"canonical": "RECARGOS", "aliases": []}, + {"canonical": "MULTAS", "aliases": []}, + {"canonical": "IVA_DE_PREV", "aliases": ["IVA DE PREV"]}, + {"canonical": "CUOTAS_COMPENSATORIAS", "aliases": ["CUOTAS CONPENSATORIAS"]}, + {"canonical": "IDENTIFICADORES", "aliases": []}, + {"canonical": "IEPS_2", "aliases": ["IEPS 2"]}, + {"canonical": "FORMA_PAGO_IEPS_2", "aliases": ["FORMA DE PAGO IEPS 2"]}, + {"canonical": "DTA_2", "aliases": ["DTA 2"]}, + {"canonical": "FORMA_PAGO_DTA_2", "aliases": ["FORMA DE PAGO DTA 2"]}, + {"canonical": "IVA_2", "aliases": ["IVA 2"]}, + {"canonical": "FORMA_PAGO_IVA_2", "aliases": ["FORMA DE PAGO IVA 2"]}, + {"canonical": "IGI_2", "aliases": ["IGI 2"]}, + {"canonical": "FORMA_PAGO_IGI_2", "aliases": ["FORMA DE PAGO IGI 2"]}, + {"canonical": "FORMA_PAGO_PREVALIDACION_2", "aliases": ["FORMA DE PAGO PREVALIDACION 2"]}, + {"canonical": "CNT_2", "aliases": ["CNT 2"]}, + {"canonical": "FORMA_PAGO_CNT_2", "aliases": ["FORMA DE PAGO CNT 2"]}, + # Legacy / compatibilidad (PEDIMENTO una columna ##-####-#######, y otros) + {"canonical": "PEDIMENTO", "aliases": ["NUMERO DE PEDIMENTO (##-####-######)", "NUMERO DE PEDIMENTO", "NUMERO PEDIMENTO", "PED"]}, + {"canonical": "CLIENTE_SHORT_NAME", "aliases": ["CLIENTE", "SHORT_NAME", "CLIENTE_ID", "CLIENT_ID", "ID CLIENTE"]}, + {"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO PED"]}, + {"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]}, + {"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]}, + {"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]}, + {"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]}, + {"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]}, + ], +} + +# Orden de columnas para CSV sin cabecera (primera fila = datos). Usado por detect_headers_or_data. +PEDIMENTOS_TEMPLATE_ORDER: List[str] = [ + item["canonical"] for item in TEMPLATE_COLUMNS["pedimentos"] +] + + +def _first_row_looks_like_three_cols_data(first_row: List[str]) -> bool: + """True si las primeras 3 celdas son AÑO (2 dígitos), PATENTE (4), NUMERO (7).""" + if not first_row or len(first_row) < 3: + return False + c0 = (first_row[0] or "").strip() + c1 = (first_row[1] or "").strip() + c2 = (first_row[2] or "").strip() + return ( + len(c0) <= AÑO_LEN and c0.isdigit() + and len(c1) <= PATENTE_LEN and c1.isdigit() + and len(c2) <= NUMERO_LEN and c2.isdigit() + ) + + +def detect_headers_or_data( + file_path: str, + normalize_header_fn, + parse_pedimento_fn, + encoding: str = "utf-8-sig", +) -> Tuple[Optional[List[str]], bool]: + """ + Lee la primera línea del CSV y decide si es cabecera o dato. + - Si las primeras 3 celdas son 2, 4 y 7 dígitos (AÑO, PATENTE, NUMERO) -> has_header=False. + - Si la primera celda tiene formato ##-####-####### (compatibilidad) -> has_header=False. + - Si no -> has_header=True. Devuelve (fieldnames, has_header). + """ + try: + with open(file_path, "r", encoding=encoding) as f: + sample = f.read(2048) + except Exception: + return None, True + lines = sample.splitlines() + if not lines: + return None, True + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = csv.excel + reader = csv.reader(io.StringIO(lines[0]), dialect=dialect) + first_row = next(reader, None) + if not first_row: + return None, True + if _first_row_looks_like_three_cols_data(first_row): + return list(PEDIMENTOS_TEMPLATE_ORDER), False + first_cell = (first_row[0] or "").strip() + if parse_pedimento_fn(first_cell) is not None: + return list(PEDIMENTOS_TEMPLATE_ORDER), False + return None, True + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla pedimentos.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out + + +# Formato Clarion Col A: ##-####-####### (15 chars, guiones en posiciones 3 y 8, 1-based) +PEDIMENTO_LEN = 15 +PEDIMENTO_DASH_POSITIONS = (2, 7) # 0-based: pos 2 y 7 deben ser '-' + + +def parse_pedimento_col_a(pedimento_str: str) -> Optional[Tuple[str, str, str]]: + """ + Parsea Col A (PEDIMENTO) formato ##-####-#######. + Retorna (year_2, license_4, pedimento_number_7) o None si formato inválido. + """ + if not pedimento_str or not isinstance(pedimento_str, str): + return None + s = (pedimento_str or "").strip() + if len(s) != PEDIMENTO_LEN: + return None + if s[PEDIMENTO_DASH_POSITIONS[0]] != "-" or s[PEDIMENTO_DASH_POSITIONS[1]] != "-": + return None + year = s[0:2] + license_val = s[3:7] + pedimento_number = s[8:15] + if not year.isdigit() or not license_val.isdigit() or not pedimento_number.isdigit(): + return None + return (year, license_val, pedimento_number) + + +def is_clarion_layout(row_norm: Dict[str, Any]) -> bool: + """True si la fila tiene las 3 columnas AÑO, PATENTE, NUMERO con valor, o PEDIMENTO con valor (compatibilidad).""" + año = (row_norm.get("AÑO") or "").strip() + patente = (row_norm.get("PATENTE") or "").strip() + numero = (row_norm.get("NUMERO") or "").strip() + if año and patente and numero: + return True + ped = (row_norm.get("PEDIMENTO") or "").strip() + return bool(ped) diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py new file mode 100644 index 00000000..6ea80cef --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_pedimento + +__all__ = ["validate_row_pedimento"] diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py new file mode 100644 index 00000000..e1311509 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py @@ -0,0 +1,458 @@ +""" +Validaciones comunes de fila para import CSV de pedimentos. +Paridad Clarion: desfase (Col N), Cols 1-3 (AÑO, PATENTE, NUMERO) u Col A PEDIMENTO, obligatorios B–H (VALIDA_TODA), +VALIDACIONES_PEDIMENTO (Tipo I/E, Clave+Régimen+Tipo, Aduana, Patente, IND/CON, Método transporte). +""" +from typing import Dict, Any, Optional, Set, Tuple + +from ..template_config import ( + AÑO_LEN, + PATENTE_LEN, + NUMERO_LEN, + PEDIMENTO_LEN, + PEDIMENTO_DASH_POSITIONS, + parse_pedimento_col_a, +) +from ..common.common_validators import ( + check_required_max, + check_in_set, + check_int_in_set, + check_optional_int_in_set, + check_optional_short_name, + check_optional_max_length, + check_optional_decimal, + check_optional_in_set, + parse_date, + DATE_FORMAT_PREFERENCE_MAP, +) + +# --- Desfase +MSG_DESFASE = "Advertencia: Podría existir un desfase en esta línea." +MSG_DESFASE_SOLUCION = "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta." + +# --- Col A --- +MSG_PEDIMENTO_VACIO = ( + "Error: (Col. A) La columna de Pedimento está vacío y no se pueden hacer las validaciones. " + "Capturar en la Columna A un Pedimento nuevo o uno ya existente al cual desee actualizar campos" +) +MSG_PEDIMENTO_LONGITUD = ( + "Error: (Col. A) La columna de Pedimento no cumple con la longitud o sintaxis correcta Ej. (XX-XXXX-XXXXXXX)" +) +MSG_PEDIMENTO_FORMATO = "Error: (Col. A) El Formato del Pedimento: {val} es incorrecto." +MSG_PEDIMENTO_FORMATO_SOLUCION = "Capturar en la columna A el campo Pedimento con este formato ##-####-#######." + +# --- Cols 1-3 (AÑO, PATENTE, NUMERO) --- +MSG_ANIO_PATENTE_NUMERO = "Error: (Cols 1-3) AÑO, PATENTE y NUMERO son obligatorios (AÑO 2 dígitos, PATENTE 4, NUMERO 7)." +MSG_PATENTE_NO_EXISTE = "Error: (Patente) La patente {val} no está dada de alta en el catálogo de agentes aduanales." + +# --- Obligatorios B–H --- +MSG_OBLIGATORIOS = "Existen campos vacios que son obligatorios, es la {campos}." +MSG_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta." + +# --- Tipo B --- +MSG_TIPO_INVALIDO = "Error: (Col. B) El Tipo de Operación: {val} no es valido." +MSG_TIPO_SOLUCION = "Capturar una opción valida, que seria I para Importación, E para Exportación." + +# --- Combinación B+C+D --- +MSG_COMBINACION = "Error, (Col. B, C y D) La combinación de una operación de {b} con clave de pedimento {c} y régimen {d} no es posible." +MSG_COMBINACION_SOLUCION = "Las posibles combinaciones son {comb}." + +# --- Aduana H --- +MSG_ADUANA_NO_EXISTE = "Error: (Col. H) La aduana y sección: {val} no existe." +MSG_ADUANA_SOLUCION = "Revisar que sea correcta esta aduana y seccion de cruce, de ser así actualice los catálogos fijos." + +# --- Col J IND/CON --- +MSG_IND_CON_INVALIDO = "Error: (Col. J) La opción de Individual/Consolidado: {val} no es valido." +MSG_IND_CON_SOLUCION = "Capturar una opción valida: IND para Individual, CON para Consolidado: o dejar el campo vacio y automaticamente se asigna Consolidado." + +# --- Transporte K/L/M --- +MSG_TRANSP_NO_EXISTE = "Error: (Col. {col}) El Metodo de Transporte: {val} no existe en el Catálogo del Anexo 22 Apendice 3." +MSG_TRANSP_SOLUCION = "Revisar si esta asignado Correctamente." + +# --- Fechas --- +FECHA_MAX_LEN = 10 + + +def validate_row_desfase_pedimento( + raw_row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Si la fila tiene al menos 16 columnas y la 16ª (Col N, IEPS/desfase) tiene valor, error de desfase. Orden: AÑO,PATENTE,NUMERO,TIPO,...,IEPS en índice 15.""" + values_ordered = list(raw_row.values()) if raw_row else [] + desfase_idx = 15 # IEPS en PEDIMENTOS_TEMPLATE_ORDER (tras AÑO,PATENTE,NUMERO + 12 columnas más) + if len(values_ordered) >= (desfase_idx + 1) and (values_ordered[desfase_idx] or "").strip(): + return { + "line": line_num, + "col": "", + "msg": f"{MSG_DESFASE} {MSG_DESFASE_SOLUCION}", + } + return None + + +def validate_row_pedimento_required_col_a( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Col A (PEDIMENTO) obligatorio y longitud 15.""" + val = (row.get("PEDIMENTO") or "").strip() + if not val: + return {"line": line_num, "col": "PEDIMENTO", "msg": MSG_PEDIMENTO_VACIO} + if len(val) != PEDIMENTO_LEN: + return { + "line": line_num, + "col": "PEDIMENTO", + "msg": f"{MSG_PEDIMENTO_LONGITUD} {MSG_PEDIMENTO_FORMATO_SOLUCION}", + } + return None + + +def validate_row_pedimento_format( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Formato ##-####-#######: guiones en posiciones 3 y 8 (1-based). Solo aplica cuando la fila trae PEDIMENTO.""" + val = (row.get("PEDIMENTO") or "").strip() + if not val or len(val) != PEDIMENTO_LEN: + return None + if val[PEDIMENTO_DASH_POSITIONS[0]] != "-" or val[PEDIMENTO_DASH_POSITIONS[1]] != "-": + return { + "line": line_num, + "col": "PEDIMENTO", + "msg": f"{MSG_PEDIMENTO_FORMATO.format(val=val)} {MSG_PEDIMENTO_FORMATO_SOLUCION}", + } + return None + + +def validate_row_pedimento_required_three_cols( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Cols 1-3: AÑO (2 dígitos), PATENTE (4), NUMERO (7) obligatorios y numéricos.""" + cols_missing = [] + año = (row.get("AÑO") or "").strip() + patente = (row.get("PATENTE") or "").strip() + numero = (row.get("NUMERO") or "").strip() + if not año: + cols_missing.append("Col.1) AÑO") + elif len(año) > AÑO_LEN or not año.isdigit(): + return {"line": line_num, "col": "AÑO", "msg": "AÑO debe ser 2 dígitos numéricos."} + if not patente: + cols_missing.append("Col.2) PATENTE") + elif len(patente) > PATENTE_LEN or not patente.isdigit(): + return {"line": line_num, "col": "PATENTE", "msg": "PATENTE debe ser 4 dígitos numéricos."} + if not numero: + cols_missing.append("Col.3) NUMERO") + elif len(numero) > NUMERO_LEN or not numero.isdigit(): + return {"line": line_num, "col": "NUMERO", "msg": "NUMERO debe ser 7 dígitos numéricos."} + if cols_missing: + campos = ", ".join(cols_missing) + return { + "line": line_num, + "col": "", + "msg": f"{MSG_ANIO_PATENTE_NUMERO} Faltan: {campos}.", + } + return None + + +def validate_row_patente( + row: Dict[str, Any], + line_num: int, + valid_patentes: Set[str], +) -> Optional[Dict[str, Any]]: + """Si la fila trae PATENTE o PEDIMENTO (parseado), valida que la patente esté en el catálogo de agentes aduanales.""" + patente_val = (row.get("PATENTE") or "").strip() + if not patente_val: + ped = (row.get("PEDIMENTO") or "").strip() + parsed = parse_pedimento_col_a(ped) if ped else None + if parsed: + _, patente_val, _ = parsed + if not patente_val: + return None + if valid_patentes and patente_val not in valid_patentes: + return { + "line": line_num, + "col": "PATENTE", + "msg": MSG_PATENTE_NO_EXISTE.format(val=patente_val), + } + return None + + +def validate_row_pedimento_required_full( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Obligatorios cuando no es actualizar: TIPO (B), CLAVE_PEDIMENTO (C), REGIMEN (D), FECHA_INICIO (E), FECHA_FINAL (F), FECHA_PAGO (G), ADUANA_SECCION_CRUCE (H).""" + cols_missing = [] + col_labels = [ + ("TIPO_OPERACION", "Col.B) Tipo Operación"), + ("CLAVE_PEDIMENTO", "Col.C) Clave Pedimento"), + ("REGIMEN", "Col.D) Clave Régimen"), + ("FECHA_INICIO", "Col.E) Fecha Inicio"), + ("FECHA_FINAL", "Col.F) Fecha Final"), + ("FECHA_PAGO", "Col.G) Fecha de Pago"), + ("ADUANA_SECCION_CRUCE", "Col.H) Aduana y Sección de Cruce"), + ] + for key, label in col_labels: + if not (row.get(key) or "").strip(): + cols_missing.append(label) + if not cols_missing: + return None + campos = ", ".join(cols_missing) + return { + "line": line_num, + "col": "", + "msg": f"{MSG_OBLIGATORIOS.format(campos=campos)} {MSG_OBLIGATORIOS_SOLUCION}", + } + + +def validate_row_fecha_pedimento( + row: Dict[str, Any], + col: str, + line_num: int, + date_format_preference: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Valida que el campo de fecha sea válido (longitud ≤10, parseable).""" + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > FECHA_MAX_LEN: + return { + "line": line_num, + "col": col, + "msg": f"Error: ({col}) La fecha supera la longitud de caracteres.", + } + if parse_date(val, date_format_preference) is None: + format_label = ( + DATE_FORMAT_PREFERENCE_MAP.get(date_format_preference, "##/##/####") + if date_format_preference + else "##/##/####" + ) + return { + "line": line_num, + "col": col, + "msg": f"Error: ({col}) La fecha no coincide con el formato ({format_label}).", + } + return None + + +def validate_row_tipo_operacion(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col B: si tiene valor debe ser I o E.""" + val = (row.get("TIPO_OPERACION") or "").strip().upper() + if not val: + return None + if val not in ("I", "E"): + return { + "line": line_num, + "col": "TIPO_OPERACION", + "msg": f"{MSG_TIPO_INVALIDO.format(val=row.get('TIPO_OPERACION'))} {MSG_TIPO_SOLUCION}", + } + return None + + +def validate_row_clave_regimen_tipo( + row: Dict[str, Any], + line_num: int, + valid_clave_regimen_tipo: Set[Tuple[str, str, str]], +) -> Optional[Dict[str, Any]]: + """Combinación B+C+D debe existir en CodePedimentoRegimen.""" + b = (row.get("TIPO_OPERACION") or "").strip().upper() + c = (row.get("CLAVE_PEDIMENTO") or "").strip().upper() + d = (row.get("REGIMEN") or "").strip().upper() + if not b or not c or not d: + return None + key = (c, d, b) + if valid_clave_regimen_tipo and key not in valid_clave_regimen_tipo: + # Opcional: listar combinaciones válidas para esa clave + comb = ", ".join( + f"operación {t} con clave {cp} y régimen {r}" + for cp, r, t in valid_clave_regimen_tipo + if cp == c + ) or "ninguna para esta clave" + return { + "line": line_num, + "col": "TIPO_OPERACION", + "msg": f"{MSG_COMBINACION.format(b=b, c=c, d=d)} {MSG_COMBINACION_SOLUCION.format(comb=comb)}", + } + return None + + +def validate_row_aduana( + row: Dict[str, Any], + line_num: int, + valid_aduana_seccion: Set[str], +) -> Optional[Dict[str, Any]]: + """Col H: si tiene valor debe existir en CustomsSection.""" + val = (row.get("ADUANA_SECCION_CRUCE") or "").strip() + if not val: + return None + # customs_code puede ser 3 caracteres; si Col H trae más, tomar primeros 3 + code = val[:3] if len(val) >= 3 else val + if valid_aduana_seccion and code not in valid_aduana_seccion: + return { + "line": line_num, + "col": "ADUANA_SECCION_CRUCE", + "msg": f"{MSG_ADUANA_NO_EXISTE.format(val=val)} {MSG_ADUANA_SOLUCION}", + } + return None + + +def validate_row_ind_con(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col J: si tiene valor debe ser IND o CON.""" + val = (row.get("INDIVIDUAL_CONSOLIDADO") or "").strip().upper() + if not val: + return None + if val not in ("IND", "CON"): + return { + "line": line_num, + "col": "INDIVIDUAL_CONSOLIDADO", + "msg": f"{MSG_IND_CON_INVALIDO.format(val=val)} {MSG_IND_CON_SOLUCION}", + } + return None + + +def validate_row_transporte( + row: Dict[str, Any], + line_num: int, + col_key: str, + col_label: str, + valid_anexo22_claves: Set[str], +) -> Optional[Dict[str, Any]]: + """Valida una columna de método de transporte (K, L o M) contra Anexo 22. Si el conjunto está vacío, no se valida.""" + val = (row.get(col_key) or "").strip().upper() + if not val: + return None + if valid_anexo22_claves and val not in valid_anexo22_claves: + return { + "line": line_num, + "col": col_key, + "msg": f"{MSG_TRANSP_NO_EXISTE.format(col=col_label, val=val)} {MSG_TRANSP_SOLUCION}", + } + return None + + +def validaciones_pedimento( + row: Dict[str, Any], + line_num: int, + valid_clave_regimen_tipo: Set[Tuple[str, str, str]], + valid_aduana_seccion: Set[str], + valid_anexo22_claves: Set[str], + valid_patentes: Set[str], + short_name_to_id: Optional[Dict[str, int]] = None, + date_format_preference: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Orquestador: formato PEDIMENTO si aplica, tipo B, combinación B+C+D, aduana H, patente, IND/CON J, transporte K/L/M, fechas E/F/G. CLIENTE_SHORT_NAME opcional.""" + if (row.get("PEDIMENTO") or "").strip(): + err = validate_row_pedimento_format(row, line_num) + if err: + return err + # CLIENTE_SHORT_NAME opcional: si viene valor se valida que exista en catálogo; si viene vacío no se exige + if short_name_to_id is not None: + err = check_optional_short_name(row, "CLIENTE_SHORT_NAME", short_name_to_id, line_num) + if err: + return err + err = validate_row_patente(row, line_num, valid_patentes) + if err: + return err + err = validate_row_tipo_operacion(row, line_num) + if err: + return err + err = validate_row_clave_regimen_tipo(row, line_num, valid_clave_regimen_tipo) + if err: + return err + err = validate_row_aduana(row, line_num, valid_aduana_seccion) + if err: + return err + err = validate_row_ind_con(row, line_num) + if err: + return err + err = validate_row_transporte( + row, line_num, "MET_TRANSP_ENTRADA", "K", valid_anexo22_claves + ) + if err: + return err + err = validate_row_transporte( + row, line_num, "MET_TRANSP_ARRIVO", "L", valid_anexo22_claves + ) + if err: + return err + err = validate_row_transporte( + row, line_num, "MET_TRANSP_SALIDA", "M", valid_anexo22_claves + ) + if err: + return err + for col in ("FECHA_INICIO", "FECHA_FINAL", "FECHA_PAGO"): + err = validate_row_fecha_pedimento( + row, col, line_num, date_format_preference + ) + if err: + return err + return None + + +# --- Legacy (layout sin PEDIMENTO único): mantener para compatibilidad --- +def validate_row_pedimento_required_legacy( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Requeridos para layout legacy: AÑO, ADUANA (ADUANA_SECCION_CRUCE), PATENTE, NUMERO.""" + for col, max_len in [ + ("AÑO", 2), + ("ADUANA_SECCION_CRUCE", 3), + ("PATENTE", 4), + ("NUMERO", 7), + ]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def validate_row_pedimento_fk_legacy( + row: Dict[str, Any], + line_num: int, + short_name_to_id: Dict[str, int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], +) -> Optional[Dict[str, Any]]: + """FK para layout legacy (CLAVE_PEDIMENTO o CODIGO_PEDIMENTO, REGIMEN). CLIENTE_SHORT_NAME opcional.""" + err = check_optional_short_name(row, "CLIENTE_SHORT_NAME", short_name_to_id, line_num) + if err: + return err + # Template puede dar CLAVE_PEDIMENTO o CODIGO_PEDIMENTO según alias + code = (row.get("CLAVE_PEDIMENTO") or row.get("CODIGO_PEDIMENTO") or "").strip() + if not code: + return {"line": line_num, "col": "CLAVE_PEDIMENTO", "msg": "Requerido"} + if len(code) > 2: + return {"line": line_num, "col": "CLAVE_PEDIMENTO", "msg": "Máximo 2 caracteres"} + if valid_pedimento_codes and code not in valid_pedimento_codes: + return {"line": line_num, "col": "CLAVE_PEDIMENTO", "msg": "No existe en código pedimento"} + err = check_in_set( + row, "REGIMEN", valid_regimes, line_num, max_len=3, catalog_name="régimen" + ) + if err: + return err + return None + + +def validate_row_pedimento_optionals_legacy( + row: Dict[str, Any], line_num: int +) -> Optional[Dict[str, Any]]: + """Opcionales para layout legacy.""" + err = check_optional_max_length(row, "ESTATUS", 30, line_num) + if err: + return err + for col in ("VALOR_USD", "PRECIO_PAGADO", "PESO_BRUTO", "TIPO_CAMBIO"): + err = check_optional_decimal(row, col, line_num) + if err: + return err + err = check_optional_in_set( + row, "TIPO_OPERACION", {"imp", "exp"}, line_num, "Debe ser imp o exp" + ) + if err: + return err + err = check_optional_in_set( + row, + "TIPO_PEDIMENTO", + {"normal", "consolidated", "complementary", "automobile"}, + line_num, + "Tipo no válido", + ) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py new file mode 100644 index 00000000..107daf66 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py @@ -0,0 +1,126 @@ +""" +Punto de entrada de validación para import de una fila pedimento. +Flujo Clarion: + - Modo actualizar: crear si no existe, actualizar si existe. Si existe → VALIDA_PARCIAL; si no existe → VALIDA_TODA. + - Modo crear: crear siempre; si existe reemplazar. Siempre VALIDA_TODA. +Legacy: si la fila no tiene PEDIMENTO (Col A Clarion), se usa validación por AÑO/ADUANA/PATENTE/NUMERO. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from ..template_config import is_clarion_layout, parse_pedimento_col_a +from ..common.fk_loader import pedimento_key_from_parsed +from .common import ( + validate_row_desfase_pedimento, + validate_row_pedimento_required_col_a, + validate_row_pedimento_format, + validate_row_pedimento_required_three_cols, + validate_row_pedimento_required_full, + validaciones_pedimento, + validate_row_pedimento_required_legacy, + validate_row_pedimento_fk_legacy, + validate_row_pedimento_optionals_legacy, +) + + +def _clarion_pedimento_key(row: Dict[str, Any]) -> Optional[str]: + """Construye la clave (year|customs_office|license|pedimento_number) desde fila Clarion (3 cols o PEDIMENTO).""" + año = (row.get("AÑO") or "").strip() + patente = (row.get("PATENTE") or "").strip() + numero = (row.get("NUMERO") or "").strip() + if año and patente and numero: + aduana = (row.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] + return pedimento_key_from_parsed(año[:2], aduana, patente[:4], numero[:7]) + parsed = parse_pedimento_col_a((row.get("PEDIMENTO") or "").strip()) + if not parsed: + return None + year, license_val, pedimento_number = parsed + aduana = (row.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] + return pedimento_key_from_parsed(year, aduana, license_val, pedimento_number) + + +def validate_row_pedimento( + row: Dict[str, Any], + line_num: int, + short_name_to_id: Dict[str, int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], + valid_clave_regimen_tipo: Set[Tuple[str, str, str]], + valid_aduana_seccion: Set[str], + existing_pedimento_keys: Set[str], + valid_anexo22_claves: Set[str], + valid_patentes: Set[str], + actualizar: bool = False, + raw_row: Optional[Dict[str, Any]] = None, + date_format_preference: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de pedimentos. + - Si raw_row está presente, se valida desfase (Col N) primero. + - Layout Clarion (PEDIMENTO con valor): flujo VALIDA_TODA / VALIDA_PARCIAL según actualizar y existing_pedimento_keys. + - Layout legacy (sin PEDIMENTO): requeridos AÑO, ADUANA, PATENTE, NUMERO, CODIGO_PEDIMENTO, REGIMEN; CLIENTE_SHORT_NAME opcional. + """ + if raw_row is not None: + err = validate_row_desfase_pedimento(raw_row, line_num) + if err: + return err + + if not is_clarion_layout(row): + err = validate_row_pedimento_required_legacy(row, line_num) + if err: + return err + err = validate_row_pedimento_fk_legacy( + row, line_num, short_name_to_id, valid_regimes, valid_pedimento_codes + ) + if err: + return err + return validate_row_pedimento_optionals_legacy(row, line_num) + + # Layout Clarion: validar Cols 1-3 (AÑO, PATENTE, NUMERO) o Col A (PEDIMENTO) si compatibilidad + has_three_cols = ( + (row.get("AÑO") or "").strip() + and (row.get("PATENTE") or "").strip() + and (row.get("NUMERO") or "").strip() + ) + if has_three_cols: + err = validate_row_pedimento_required_three_cols(row, line_num) + else: + err = validate_row_pedimento_required_col_a(row, line_num) + if not err: + err = validate_row_pedimento_format(row, line_num) + if err: + return err + + key = _clarion_pedimento_key(row) + # Actualizar: si existe → validación parcial; si no existe → validación completa (crear). + # Crear: siempre validación completa (crear siempre; si existe reemplazar). + use_partial = ( + actualizar + and key is not None + and key in existing_pedimento_keys + ) + + if use_partial: + return validaciones_pedimento( + row, + line_num, + valid_clave_regimen_tipo, + valid_aduana_seccion, + valid_anexo22_claves, + valid_patentes, + short_name_to_id=short_name_to_id, + date_format_preference=date_format_preference, + ) + + err = validate_row_pedimento_required_full(row, line_num) + if err: + return err + return validaciones_pedimento( + row, + line_num, + valid_clave_regimen_tipo, + valid_aduana_seccion, + valid_anexo22_claves, + valid_patentes, + short_name_to_id=short_name_to_id, + date_format_preference=date_format_preference, + ) diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/trailers/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/trailers/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py new file mode 100644 index 00000000..47ef2168 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers, fk_loader diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py new file mode 100644 index 00000000..6948bf3c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py @@ -0,0 +1,183 @@ +""" +Validadores reutilizables para import CSV de trailers (longitudes, requerido). +Paridad Clarion: VALIDACIONES_TRAILER, código entidad C/I/A/B, catálogos tipo/país/estado, desfase. +""" +from typing import Dict, Any, Optional, Set, Tuple + +# Max lengths from Trailer model (a76.trailer) +MAX_LEN = { + "trailer_number": 20, + "ace_trailer_number": 10, + "trailer_type_key": 2, + "seal": 15, + "entity_code": 1, + "plate_number": 17, + "state": 30, + "country": 3, + "container_key": 3, +} + +CODIGO_ENTIDAD_VALIDOS = {"C", "I", "A", "B"} + + +def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length( + row: Dict[str, Any], col: str, max_len: int, line_num: int, required: bool = False +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def _get_codigo_entidad(row: Dict[str, Any]) -> str: + return (row.get("CODIGO DE ENTIDAD") or row.get("CODIGO ENTIDAD") or "").strip() + + +def check_codigo_entidad_valores(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col E: Si CODIGO DE ENTIDAD no vacío, debe ser C, I, A o B (Clarion).""" + val = _get_codigo_entidad(row).upper() + if not val: + return None + if val in CODIGO_ENTIDAD_VALIDOS: + return None + return { + "line": line_num, + "col": "CODIGO DE ENTIDAD", + "msg": f"Error: (Col. E) El Codigo de Entidad: {val} es incorrecto.", + "solution": "Capturar en columna E el Codigo de Entidad correcto, C, I, A ó B.", + } + + +def check_trailer_type_catalog( + row: Dict[str, Any], + line_num: int, + valid_trailer_type_keys: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col C: Si TIPO TRAILER no vacío, debe existir en catálogo (GTipoTrailer).""" + val = (row.get("TIPO TRAILER") or "").strip() + if not val or valid_trailer_type_keys is None: + return None + if val.upper() in valid_trailer_type_keys: + return None + return { + "line": line_num, + "col": "TIPO TRAILER", + "msg": f"Error: (Col. C) El Tipo de Trailer: {val} es incorrecto.", + "solution": "Capturar en columna C un Tipo del Trailer existente.", + } + + +def check_pais_catalog_trailers( + row: Dict[str, Any], + line_num: int, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col H: Si PAIS no vacío, debe ser clave americana (≤2 chars) y existir en catálogo (GPaises.Pais_Ame).""" + val = (row.get("PAIS") or "").strip() + if not val or valid_country_ame is None: + return None + val_upper = val.upper() + if len(val) > 2: + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Celda H{line_num}) El Pais: {val} Es Incorrecto", + "solution": "Capturar en columna H un Pais en Clave Americana(US = Estados Unidos, MX = Mexico, ES = España, etc).", + } + if val_upper in valid_country_ame: + return None + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. H) El Pais: {val} es incorrecto.", + "solution": "Capturar en columna H el Pais del Transporte en Clave Americana.", + } + + +def check_estado_catalog_trailers( + row: Dict[str, Any], + line_num: int, + state_descriptions_upper: Optional[Set[str]] = None, + state_ame_to_description: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + Col G: Si ESTADO no vacío, debe existir en catálogo (GEstados). Si hay estado, PAIS no puede estar vacío. + Devuelve (error, estado_resuelto_upper). estado_resuelto_upper es la descripción en mayúsculas para estado-país. + """ + state_ame_to_description = state_ame_to_description or {} + val = (row.get("ESTADO") or "").strip() + if not val or state_descriptions_upper is None: + return None, None + val_upper = val.upper() + resolved_upper: Optional[str] = None + if val_upper in state_descriptions_upper: + resolved_upper = val_upper + elif len(val) <= 2 and val_upper in state_ame_to_description: + resolved_upper = state_ame_to_description[val_upper] + if resolved_upper is None: + return { + "line": line_num, + "col": "ESTADO", + "msg": f"Error: (Celda G{line_num}) El Estado: {val} es incorrecto.", + "solution": "Capturar en columna G el Estado del Trailer en Clave Americana o Nombre Completo.", + }, None + # Si existe estado, PAIS no puede estar vacío (Clarion) + pais = (row.get("PAIS") or "").strip() + if not pais: + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Celda G{line_num}) El Estado: {val} No esta ligado a ningun Pais.", + "solution": "Capturar en columna H un Pais en Clave Americana(US = Estados Unidos, MX = Mexico, ES = España, etc).", + }, None + return None, resolved_upper + + +def check_estado_pais_consistency_trailers( + row: Dict[str, Any], + line_num: int, + state_country_set: Optional[Set[Tuple[str, str]]] = None, + estado_resuelto_upper: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Si hay ESTADO y PAIS, validar que el estado pertenezca al país (Clarion GEstados-GPaises).""" + estado = (row.get("ESTADO") or "").strip() + pais = (row.get("PAIS") or "").strip().upper() + if not estado or not pais or state_country_set is None: + return None + desc_upper = estado_resuelto_upper or estado.upper() + key = (pais, desc_upper) + if key in state_country_set: + return None + return { + "line": line_num, + "col": "ESTADO", + "msg": f"Error: (Col. G) EL Estado: {estado} no pertenece al Pais: {pais}.", + "solution": "Capturar en columna G un Estado que pertenesca al Pais de la columna H.", + } + + +def check_desfase_trailers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA tiene valor → advertencia de desfase (Clarion, no bloqueante).""" + val = (row.get("COL_EXTRA") or "").strip() + if not val: + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": "Advertencia: Podría existir un desfase en esta línea.", + "solution": "Revisar esta línea del archivo CSV y verificar cada campo este en la posicion correcta.", + "warning": True, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py new file mode 100644 index 00000000..c87adcaa --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py @@ -0,0 +1,78 @@ +""" +Carga de conjuntos FK para validación de import CSV de trailers/cajas. +Clarion: GTipoTrailer, GPaises (Pais_Ame), GEstados (Descripcion / Clave_Ame), relación Estado-País. +""" +from typing import Set, Tuple, Optional, Dict +import logging + +from core.database import CoreSessionLocal + +logger = logging.getLogger(__name__) + + +def load_trailers_fk_sets( + tenant_id: Optional[int] = None, + company_id: Optional[int] = None, +) -> Tuple[ + Set[str], + Set[str], + Set[str], + Set[Tuple[str, str]], + Dict[str, str], +]: + """ + Carga conjuntos para validación CSV de trailers (paridad Clarion). + Devuelve: + - valid_trailer_type_keys: códigos de trailer_type (GTipoTrailer) + - valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas + - state_descriptions_upper: descripciones de estados en mayúsculas (GEstados) + - state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país" + - state_ame_to_description: dict clave_ame_upper -> description_upper (opcional; vacío si State no tiene ame_key) + """ + valid_trailer_type_keys: Set[str] = set() + valid_country_ame: Set[str] = set() + state_descriptions_upper: Set[str] = set() + state_country_set: Set[Tuple[str, str]] = set() + state_ame_to_description: Dict[str, str] = {} + + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.trailer_types.models import TrailerType + from api.v1.modules.public.reference_data.countries.models import Country + from api.v1.modules.public.reference_data.states.models import State + + for row in session.query(TrailerType.trailer_type_key).all(): + if row[0]: + valid_trailer_type_keys.add((row[0] or "").strip().upper()) + + for row in session.query(Country.ame_key).all(): + if row[0]: + valid_country_ame.add((row[0] or "").strip().upper()) + + for state in session.query(State).all(): + desc = (state.description or "").strip() + if desc: + state_descriptions_upper.add(desc.upper()) + country = ( + session.query(Country) + .filter(Country.m3_key == state.m3_key) + .first() + ) + if country and (country.ame_key or "").strip(): + state_country_set.add( + ((country.ame_key or "").strip().upper(), desc.upper()) + ) + ame = getattr(state, "ame_key", None) + if ame and (ame or "").strip(): + state_ame_to_description[(ame or "").strip().upper()] = desc.upper() if desc else (ame or "").strip().upper() + + except Exception as e: + logger.warning("Trailers import: could not load FK sets: %s", e) + + return ( + valid_trailer_type_keys, + valid_country_ame, + state_descriptions_upper, + state_country_set, + state_ame_to_description, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py new file mode 100644 index 00000000..032a8edb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py @@ -0,0 +1,94 @@ +""" +Mapeo fila CSV → datos para Trailer. +Paridad Clarion LLENA_TRAILER: en actualización, campo vacío en CSV usa valor existente del trailer. +""" +from typing import Dict, Any, Optional, Union + +from .common_validators import MAX_LEN + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _get_existing_val(existing: Union[Any, Dict[str, Any]], key: str) -> Any: + """Obtiene valor del trailer existente (modelo ORM o dict).""" + if existing is None: + return None + if isinstance(existing, dict): + return existing.get(key) + return getattr(existing, key, None) + + +def _get_entity_code(row_norm: Dict[str, Any]) -> Optional[str]: + return _str_or_none( + row_norm.get("CODIGO ENTIDAD") or row_norm.get("CODIGO DE ENTIDAD"), + MAX_LEN["entity_code"], + ) + + +def row_to_trailer_data(row_norm: Dict[str, Any]) -> Dict[str, Any]: + """Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row.""" + trailer_number = _str_or_none( + row_norm.get("NUMERO TRAILER") or row_norm.get("CLAVE TRAILER"), + MAX_LEN["trailer_number"], + ) + if not trailer_number: + return {} + return { + "trailer_number": trailer_number, + "ace_trailer_number": _str_or_none(row_norm.get("CLAVE ACE"), MAX_LEN["ace_trailer_number"]), + "trailer_type_key": _str_or_none(row_norm.get("TIPO TRAILER"), MAX_LEN["trailer_type_key"]), + "seal": _str_or_none(row_norm.get("PRECINTO"), MAX_LEN["seal"]), + "entity_code": _get_entity_code(row_norm), + "plate_number": _str_or_none(row_norm.get("PLACAS"), MAX_LEN["plate_number"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "container_key": _str_or_none(row_norm.get("CLAVE CONTENEDOR"), MAX_LEN["container_key"]), + } + + +def row_to_trailer_data_for_update( + row_norm: Dict[str, Any], + existing_trailer: Union[Any, Dict[str, Any]], +) -> Dict[str, Any]: + """ + Build dict for TrailerUpdateDTO: CSV value if non-empty, else existing trailer value (Clarion VALIDA_PARCIAL / LLENA_TRAILER). + """ + trailer_number = _str_or_none( + row_norm.get("NUMERO TRAILER") or row_norm.get("CLAVE TRAILER"), + MAX_LEN["trailer_number"], + ) or _get_existing_val(existing_trailer, "trailer_number") + if not trailer_number: + return {} + + def _csv_or_existing(csv_key: str, dto_key: str, max_len: Optional[int] = None): + v = _str_or_none(row_norm.get(csv_key), max_len) + if v is not None and v != "": + return v + return _get_existing_val(existing_trailer, dto_key) + + def _entity_code_or_existing(): + v = _get_entity_code(row_norm) + if v is not None and v != "": + return v + return _get_existing_val(existing_trailer, "entity_code") + + return { + "trailer_number": trailer_number, + "ace_trailer_number": _csv_or_existing("CLAVE ACE", "ace_trailer_number", MAX_LEN["ace_trailer_number"]), + "trailer_type_key": _csv_or_existing("TIPO TRAILER", "trailer_type_key", MAX_LEN["trailer_type_key"]), + "seal": _csv_or_existing("PRECINTO", "seal", MAX_LEN["seal"]), + "entity_code": _entity_code_or_existing(), + "plate_number": _csv_or_existing("PLACAS", "plate_number", MAX_LEN["plate_number"]), + "state": _csv_or_existing("ESTADO", "state", MAX_LEN["state"]), + "country": _csv_or_existing("PAIS", "country", MAX_LEN["country"]), + "container_key": _csv_or_existing("CLAVE CONTENEDOR", "container_key", MAX_LEN["container_key"]), + } diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py similarity index 95% rename from backend/api/v1/modules/a76/transportation/trailers/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/trailers/routes.py index 7e51aca1..0c9475fd 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py @@ -15,6 +15,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -42,6 +43,7 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo Agregar/Actualizar (ACT); si True, validación parcial para claves existentes"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -62,6 +64,7 @@ async def upload_import_file( "company_id": company_id, "user_id": current_user.get("id"), "template_id": "trailers", + "actualizar": actualizar, } try: @@ -81,7 +84,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/trailers/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/trailers/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py new file mode 100644 index 00000000..3102971d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py @@ -0,0 +1,426 @@ +""" +Tareas Celery para importación CSV de Trailers y Cajas. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe). +""" +import csv +import json +import logging +import os +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from .template_config import row_from_template +from .validators import validate_row_trailer, validate_row_trailer_desfase +from .common.mappers import row_to_trailer_data, row_to_trailer_data_for_update +from .common.fk_loader import load_trailers_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "trl" + +# Para routes.py +TRL_IMPORT_FILE_PREFIX = "trl_import_file:" +TRL_IMPORT_META_PREFIX = "trl_import_meta:" +TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:" +TRL_IMPORT_STATUS_PREFIX = "trl_import_status:" +TRL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _dedupe_headers(headers: List[str]) -> List[str]: + counts: Dict[str, int] = {} + unique: List[str] = [] + for header in headers: + name = str(header or "").strip() or "COL" + count = counts.get(name, 0) + 1 + counts[name] = count + unique.append(name if count == 1 else f"{name} {count}") + return unique + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Trailers import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Trailers import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_trailer_numbers: Set[str] = set() + if actualizar: + try: + from api.v1.modules.a76.transportation.trailers.models import Trailer + with CoreSessionLocal() as session: + for t in ( + session.query(Trailer.trailer_number) + .filter( + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .all() + ): + if t[0] and (t[0] or "").strip(): + existing_trailer_numbers.add((t[0] or "").strip()) + except Exception as e: + logger.warning("Trailers import: could not load existing trailer_numbers for actualizar: %s", e) + + ( + valid_trailer_type_keys, + valid_country_ame, + state_descriptions_upper, + state_country_set, + state_ame_to_description, + ) = load_trailers_fk_sets(tenant_id, company_id) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f_in, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, common_normalize.normalize_header) + # Desfase: advertencia no bloqueante (no se añade a error_lines) + _ = validate_row_trailer_desfase(row_norm, i) + err = validate_row_trailer( + row_norm, + i, + actualizar=actualizar, + existing_trailer_numbers=existing_trailer_numbers, + valid_trailer_type_keys=valid_trailer_type_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + state_ame_to_description=state_ame_to_description, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Trailers import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Trailers import: failed to store scan status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Trailers import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Trailers import: failed to store scan status in Redis: %s", e) + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Trailers import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Trailers import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_trailer_numbers: Set[str] = set() + if actualizar: + try: + from api.v1.modules.a76.transportation.trailers.models import Trailer + with CoreSessionLocal() as session: + for t in ( + session.query(Trailer.trailer_number) + .filter( + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .all() + ): + if t[0] and (t[0] or "").strip(): + existing_trailer_numbers.add((t[0] or "").strip()) + except Exception as e: + logger.warning("Trailers import: could not load existing trailer_numbers for actualizar: %s", e) + + ( + valid_trailer_type_keys, + valid_country_ame, + state_descriptions_upper, + state_country_set, + state_ame_to_description, + ) = load_trailers_fk_sets(tenant_id, company_id) + + from api.v1.modules.a76.transportation.trailers.services import TrailerService + from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_trailer( + row_norm, + i, + actualizar=actualizar, + existing_trailer_numbers=existing_trailer_numbers, + valid_trailer_type_keys=valid_trailer_type_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + state_ame_to_description=state_ame_to_description, + ) + if err: + skipped_invalid += 1 + tn = (row_norm.get("NUMERO TRAILER") or row_norm.get("CLAVE TRAILER") or "").strip()[:20] or "-" + skipped_details.append({ + "line": i, + "trailer_number": tn, + "invoice": tn, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + tn = (row_norm.get("NUMERO TRAILER") or row_norm.get("CLAVE TRAILER") or "").strip()[:20] or "" + if not tn: + skipped_invalid += 1 + continue + + if tn in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append({ + "line": i, + "trailer_number": tn, + "invoice": tn, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) + continue + seen_keys_in_file[tn] = i + + existing = TrailerService.get_by_id(session, tn, tenant_id, company_id) + try: + if existing: + if actualizar: + data = row_to_trailer_data_for_update(row_norm, existing) + else: + data = row_to_trailer_data(row_norm) + if not data or not data.get("trailer_number"): + skipped_invalid += 1 + continue + update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"}) + TrailerService.update(session, tn, tenant_id, update_data, company_id) + updated_count += 1 + else: + data = row_to_trailer_data(row_norm) + if not data or not data.get("trailer_number"): + skipped_invalid += 1 + continue + create_data = TrailerCreateDTO(**data) + TrailerService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append({ + "line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err), + }) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("Trailers import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("Trailers import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + try: + r = _get_redis() + r.delete(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}") + except Exception as e: + logger.warning("Trailers import: failed to delete status key: %s", e) + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Trailers import: failed to store commit status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Trailers import: starting commit for job %s", job_id) + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Trailers import: failed to store commit status in Redis: %s", e) + return result diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/trailers/template_config.py similarity index 95% rename from backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/trailers/template_config.py index be0eb4a7..31c35f12 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/template_config.py @@ -16,6 +16,7 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "ESTADO", "aliases": ["STATE"]}, {"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]}, {"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]}, + {"canonical": "COL_EXTRA", "aliases": ["COLUMNA I", "COL I", "COL 9"]}, ], } diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py new file mode 100644 index 00000000..456065a3 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_trailer, validate_row_trailer_desfase + +__all__ = ["validate_row_trailer", "validate_row_trailer_desfase"] diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py new file mode 100644 index 00000000..056ce6f2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py @@ -0,0 +1,122 @@ +""" +Validaciones comunes de fila para import CSV de trailers. +Paridad Clarion: VALIDACIONES_TRAILER, VALIDA_TODA_TRAILER, VALIDA_PARCIAL_TRAILER. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from ..common.common_validators import ( + MAX_LEN, + check_required, + check_max_length, + check_codigo_entidad_valores, + check_trailer_type_catalog, + check_pais_catalog_trailers, + check_estado_catalog_trailers, + check_estado_pais_consistency_trailers, +) + + +def validate_row_trailer_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_required(row, "NUMERO TRAILER", MAX_LEN["trailer_number"], line_num) + + +def validate_row_trailer_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("CLAVE ACE", MAX_LEN["ace_trailer_number"]), + ("TIPO TRAILER", MAX_LEN["trailer_type_key"]), + ("PRECINTO", MAX_LEN["seal"]), + ("CODIGO ENTIDAD", MAX_LEN["entity_code"]), + ("CODIGO DE ENTIDAD", MAX_LEN["entity_code"]), + ("PLACAS", MAX_LEN["plate_number"]), + ("ESTADO", MAX_LEN["state"]), + ("PAIS", MAX_LEN["country"]), + ("CLAVE CONTENEDOR", MAX_LEN["container_key"]), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validaciones_trailer( + row: Dict[str, Any], + line_num: int, + valid_trailer_type_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, + state_ame_to_description: Optional[Dict[str, str]] = None, +) -> Optional[Dict[str, Any]]: + """ + VALIDACIONES_TRAILER: reglas compartidas (longitudes, tipo trailer, código entidad, + país, estado, estado-país). No exige CODIGO DE ENTIDAD obligatorio. + """ + err = validate_row_trailer_required(row, line_num) + if err: + return err + err = validate_row_trailer_lengths(row, line_num) + if err: + return err + err = check_codigo_entidad_valores(row, line_num) + if err: + return err + err = check_trailer_type_catalog(row, line_num, valid_trailer_type_keys) + if err: + return err + err = check_pais_catalog_trailers(row, line_num, valid_country_ame) + if err: + return err + err_estado, estado_resuelto_upper = check_estado_catalog_trailers( + row, line_num, state_descriptions_upper, state_ame_to_description + ) + if err_estado: + return err_estado + err = check_estado_pais_consistency_trailers( + row, line_num, state_country_set, estado_resuelto_upper + ) + if err: + return err + return None + + +def valida_toda_trailer( + row: Dict[str, Any], + line_num: int, + valid_trailer_type_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, + state_ame_to_description: Optional[Dict[str, str]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_TODA_TRAILER: mismo que validaciones_trailer (Col A ya validada antes).""" + return validaciones_trailer( + row, + line_num, + valid_trailer_type_keys=valid_trailer_type_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + state_ame_to_description=state_ame_to_description, + ) + + +def valida_parcial_trailer( + row: Dict[str, Any], + line_num: int, + valid_trailer_type_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, + state_ame_to_description: Optional[Dict[str, str]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_PARCIAL_TRAILER: solo validaciones_trailer (actualizar registro existente).""" + return validaciones_trailer( + row, + line_num, + valid_trailer_type_keys=valid_trailer_type_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + state_ame_to_description=state_ame_to_description, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py new file mode 100644 index 00000000..cc8bdf39 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py @@ -0,0 +1,68 @@ +""" +Punto de entrada de validación para import de una fila trailer. +Paridad Clarion: desfase (advertencia), CLAVE vacía, VALIDA_TODA vs VALIDA_PARCIAL según actualizar y clave existente. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from .common import ( + validate_row_trailer_required, + valida_toda_trailer, + valida_parcial_trailer, +) +from ..common.common_validators import check_desfase_trailers + + +def validate_row_trailer( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, + existing_trailer_numbers: Optional[Set[str]] = None, + valid_trailer_type_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, + state_ame_to_description: Optional[Dict[str, str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de trailers. + 1. Clave (NUMERO TRAILER) vacía → error. + 2. Si actualizar y clave en existing_trailer_numbers → valida_parcial_trailer. + 3. Si no actualizar o clave no existe → valida_toda_trailer. + Desfase (COL_EXTRA) no se valida aquí; el caller puede llamar validate_row_trailer_desfase para advertencias no bloqueantes. + """ + err = validate_row_trailer_required(row, line_num) + if err: + return err + + existing = existing_trailer_numbers or set() + clave = (row.get("NUMERO TRAILER") or row.get("CLAVE TRAILER") or "").strip() + use_partial = actualizar and bool(clave and clave in existing) + + if use_partial: + err = valida_parcial_trailer( + row, + line_num, + valid_trailer_type_keys=valid_trailer_type_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + state_ame_to_description=state_ame_to_description, + ) + else: + err = valida_toda_trailer( + row, + line_num, + valid_trailer_type_keys=valid_trailer_type_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + state_ame_to_description=state_ame_to_description, + ) + return err + + +def validate_row_trailer_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Advertencia de desfase si COL_EXTRA tiene valor. No bloqueante; el caller puede acumular en warnings. + """ + return check_desfase_trailers(row, line_num) diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/__init__.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/__init__.py new file mode 100644 index 00000000..be468a87 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/__init__.py @@ -0,0 +1 @@ +# layouts_csv.transportistas: CSV import for Transportistas (carriers catalog). Paridad Clarion. diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/__init__.py new file mode 100644 index 00000000..e2a4a863 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/__init__.py @@ -0,0 +1 @@ +# transportistas common: fk_loader, common_validators, mappers diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/common_validators.py new file mode 100644 index 00000000..7f7630b2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/common_validators.py @@ -0,0 +1,143 @@ +""" +Validadores reutilizables para import CSV de transportistas. +Paridad Clarion: VALIDACIONES_TRANSPORTISTAS (estado, país, estado-país), desfase Col S. +""" +from typing import Dict, Any, Optional, Set, Tuple + +# Max lengths from Transporter model (a76.transporter) +MAX_LEN = { + "transporter_key": 23, + "name": 256, + "short_name": 10, + "responsible": 100, + "rfc": 30, + "streets": 100, + "postal_code": 15, + "city": 30, + "state": 30, + "country": 3, + "loader_code": 9, + "caat_code": 49, + "transport_code": 8, + "transport_interface_type": 20, + "ftp_server": 200, + "ftp_user": 200, + "ftp_password": 100, + "ftp_directory": 1000, +} + + +def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if max_len and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_desfase_transportistas(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA (columna 19 / Col S) tiene valor → advertencia de desfase (Clarion, no bloqueante).""" + val = (row.get("COL_EXTRA") or "").strip() + if not val: + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": "Advertencia: Podría existir un desfase en esta línea.", + "solution": "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta.", + "warning": True, + } + + +def check_pais_catalog_transportistas( + row: Dict[str, Any], + line_num: int, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col J (PAIS): Si no vacío, debe ser clave americana (2 chars) y existir en catálogo (GPaises.Pais_Ame).""" + val = (row.get("PAIS") or "").strip() + if not val or valid_country_ame is None: + return None + val_upper = val.upper() + if len(val) > 2: + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Celda J{line_num}) El Pais: {val} Es Incorrecto", + "solution": "Capturar en columna J un Pais en Clave Americana(US = Estados Unidos, MX = Mexico, ES = España, etc).", + } + if val_upper in valid_country_ame: + return None + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. J) El Pais: {val} es incorrecto.", + "solution": "Capturar en columna J el Pais del Transportista en Clave Americana.", + } + + +def check_estado_catalog_transportistas( + row: Dict[str, Any], + line_num: int, + state_descriptions_upper: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col I (ESTADO): Si no vacío, debe existir en catálogo (GEstados por descripción). Si hay estado, PAIS no puede estar vacío.""" + val = (row.get("ESTADO") or "").strip() + if not val or state_descriptions_upper is None: + return None + val_upper = val.upper() + if val_upper not in state_descriptions_upper: + return { + "line": line_num, + "col": "ESTADO", + "msg": f"Error: (Celda I{line_num}) El Estado: {val} es incorrecto.", + "solution": "Capturar en columna I el Estado del Transportista en Clave Americana o Nombre Completo.", + } + pais = (row.get("PAIS") or "").strip() + if not pais: + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Celda I{line_num}) El Estado: {val} No esta ligado a ningun Pais.", + "solution": "Capturar en columna J un Pais en Clave Americana(US = Estados Unidos, MX = Mexico, ES = España, etc).", + } + return None + + +def check_estado_pais_consistency_transportistas( + row: Dict[str, Any], + line_num: int, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """Si hay ESTADO y PAIS, validar que el estado pertenezca al país (Clarion GEstados-GPaises).""" + estado = (row.get("ESTADO") or "").strip() + pais = (row.get("PAIS") or "").strip().upper() + if not estado or not pais or state_country_set is None: + return None + key = (pais, estado.upper()) + if key in state_country_set: + return None + return { + "line": line_num, + "col": "ESTADO", + "msg": f"Error: (Col. I) EL Estado: {estado} no pertenece al Pais: {pais}.", + "solution": "Capturar en columna I un Estado que pertenesca al Pais de la columna J.", + } diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py new file mode 100644 index 00000000..86ad4740 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py @@ -0,0 +1,79 @@ +""" +Carga de conjuntos FK para validación de import CSV de transportistas. +Clarion: GTransportista (ClaveTrans), GPaises (Pais_Ame), GEstados (Descripcion), relación Estado-País. +""" +from typing import Set, Tuple, Optional +import logging + +from core.database import CoreSessionLocal + +logger = logging.getLogger(__name__) + + +def load_transportistas_fk_sets( + tenant_id: Optional[int] = None, + company_id: Optional[int] = None, +) -> Tuple[ + Set[str], + Set[str], + Set[str], + Set[Tuple[str, str]], +]: + """ + Carga conjuntos para validación CSV de transportistas (paridad Clarion). + Devuelve: + - existing_transporter_keys: claves de transportistas existentes (tenant/company) en mayúsculas + - valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas + - state_descriptions_upper: descripciones de estados en mayúsculas (GEstados) + - state_country_set: set de (ame_key_pais, description_estado_upper) para validar estado pertenece a país + """ + existing_transporter_keys: Set[str] = set() + valid_country_ame: Set[str] = set() + state_descriptions_upper: Set[str] = set() + state_country_set: Set[Tuple[str, str]] = set() + + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.transportation.transporters.models import Transporter + from api.v1.modules.public.reference_data.countries.models import Country + from api.v1.modules.public.reference_data.states.models import State + + if tenant_id is not None and company_id is not None: + for row in ( + session.query(Transporter.transporter_key) + .filter( + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) + .all() + ): + if row[0] and (row[0] or "").strip(): + existing_transporter_keys.add((row[0] or "").strip().upper()) + + for row in session.query(Country.ame_key).all(): + if row[0]: + valid_country_ame.add((row[0] or "").strip().upper()) + + for state in session.query(State).all(): + desc = (state.description or "").strip() + if desc: + state_descriptions_upper.add(desc.upper()) + country = ( + session.query(Country) + .filter(Country.m3_key == state.m3_key) + .first() + ) + if country and (country.ame_key or "").strip(): + state_country_set.add( + ((country.ame_key or "").strip().upper(), desc.upper()) + ) + + except Exception as e: + logger.warning("Transportistas import: could not load FK sets: %s", e) + + return ( + existing_transporter_keys, + valid_country_ame, + state_descriptions_upper, + state_country_set, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/mappers.py new file mode 100644 index 00000000..0b93f40d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/mappers.py @@ -0,0 +1,95 @@ +""" +Mapeo fila CSV → datos para Transporter. +Paridad Clarion LLENA_TRANSPORTISTAS / VALIDA_PARCIAL: en actualización, campo vacío en CSV usa valor existente. +UPPER aplicado según Clarion: NOMBRE CORTO, CODIGO POSTAL, PAIS, CODIGO CARGADOR, CODIGO CAAT, CODIGO TRANS, etc. +""" +from typing import Dict, Any, Optional, Union + +from .common_validators import MAX_LEN + + +def _str_or_none(val: Any, max_len: Optional[int] = None, upper: bool = False) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if upper: + s = s.upper() + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _get_existing_val(existing: Union[Any, Dict[str, Any]], key: str) -> Any: + if existing is None: + return None + if isinstance(existing, dict): + return existing.get(key) + return getattr(existing, key, None) + + +def row_to_transporter_data(row_norm: Dict[str, Any]) -> Dict[str, Any]: + """Build dict suitable for TransporterCreateDTO from normalized row (Clarion QueCSV → GenTra).""" + transporter_key = _str_or_none(row_norm.get("CLAVE TRANSPORTISTA"), MAX_LEN["transporter_key"]) + if not transporter_key: + return {} + return { + "transporter_key": transporter_key, + "name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]), + "short_name": _str_or_none(row_norm.get("NOMBRE CORTO"), MAX_LEN["short_name"], upper=True), + "responsible": _str_or_none(row_norm.get("RESPONSABLE"), MAX_LEN["responsible"]), + "rfc": _str_or_none(row_norm.get("RFC"), MAX_LEN["rfc"]), + "streets": _str_or_none(row_norm.get("CALLES"), MAX_LEN["streets"]), + "postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"], upper=True), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"], upper=True), + "loader_code": _str_or_none(row_norm.get("CODIGO CARGADOR"), MAX_LEN["loader_code"], upper=True), + "caat_code": _str_or_none(row_norm.get("CODIGO CAAT"), MAX_LEN["caat_code"], upper=True), + "transport_code": _str_or_none(row_norm.get("CODIGO TRANS"), MAX_LEN["transport_code"], upper=True), + "transport_interface_type": _str_or_none(row_norm.get("TIPO INTERFASE TRANS"), MAX_LEN["transport_interface_type"]), + "ftp_server": _str_or_none(row_norm.get("SERVIDOR FTP"), MAX_LEN["ftp_server"]), + "ftp_user": _str_or_none(row_norm.get("USUARIO FTP"), MAX_LEN["ftp_user"]), + "ftp_password": _str_or_none(row_norm.get("CLAVE ACCESO FTP"), MAX_LEN["ftp_password"]), + "ftp_directory": _str_or_none(row_norm.get("DIRECTORIO FTP"), MAX_LEN["ftp_directory"]), + } + + +def row_to_transporter_data_for_update( + row_norm: Dict[str, Any], + existing_transporter: Union[Any, Dict[str, Any]], +) -> Dict[str, Any]: + """ + Build dict for TransporterUpdateDTO: CSV value if non-empty, else existing (Clarion VALIDA_PARCIAL). + """ + transporter_key = _str_or_none(row_norm.get("CLAVE TRANSPORTISTA"), MAX_LEN["transporter_key"]) or _get_existing_val(existing_transporter, "transporter_key") + if not transporter_key: + return {} + + def _csv_or_existing(csv_key: str, dto_key: str, max_len: Optional[int] = None, upper: bool = False): + v = _str_or_none(row_norm.get(csv_key), max_len, upper=upper) + if v is not None and v != "": + return v + return _get_existing_val(existing_transporter, dto_key) + + return { + "transporter_key": transporter_key, + "name": _csv_or_existing("NOMBRE", "name", MAX_LEN["name"]), + "short_name": _csv_or_existing("NOMBRE CORTO", "short_name", MAX_LEN["short_name"], upper=True), + "responsible": _csv_or_existing("RESPONSABLE", "responsible", MAX_LEN["responsible"]), + "rfc": _csv_or_existing("RFC", "rfc", MAX_LEN["rfc"]), + "streets": _csv_or_existing("CALLES", "streets", MAX_LEN["streets"]), + "postal_code": _csv_or_existing("CODIGO POSTAL", "postal_code", MAX_LEN["postal_code"], upper=True), + "city": _csv_or_existing("CIUDAD", "city", MAX_LEN["city"]), + "state": _csv_or_existing("ESTADO", "state", MAX_LEN["state"]), + "country": _csv_or_existing("PAIS", "country", MAX_LEN["country"], upper=True), + "loader_code": _csv_or_existing("CODIGO CARGADOR", "loader_code", MAX_LEN["loader_code"], upper=True), + "caat_code": _csv_or_existing("CODIGO CAAT", "caat_code", MAX_LEN["caat_code"], upper=True), + "transport_code": _csv_or_existing("CODIGO TRANS", "transport_code", MAX_LEN["transport_code"], upper=True), + "transport_interface_type": _csv_or_existing("TIPO INTERFASE TRANS", "transport_interface_type", MAX_LEN["transport_interface_type"]), + "ftp_server": _csv_or_existing("SERVIDOR FTP", "ftp_server", MAX_LEN["ftp_server"]), + "ftp_user": _csv_or_existing("USUARIO FTP", "ftp_user", MAX_LEN["ftp_user"]), + "ftp_password": _csv_or_existing("CLAVE ACCESO FTP", "ftp_password", MAX_LEN["ftp_password"]), + "ftp_directory": _csv_or_existing("DIRECTORIO FTP", "ftp_directory", MAX_LEN["ftp_directory"]), + } diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py new file mode 100644 index 00000000..26af415f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py @@ -0,0 +1,190 @@ +""" +Rutas de importación CSV para Transportistas. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.paths import layout_path +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + TRP_IMPORT_FILE_PREFIX, + TRP_IMPORT_META_PREFIX, + TRP_IMPORT_STATUS_PREFIX, + TRP_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo Agregar/Actualizar (ACT); si True, validación parcial para claves existentes"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error("Transportistas import: access validation failed: %s", e) + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "transporters", + "actualizar": actualizar, + } + + try: + r = _get_redis() + r.set( + f"{TRP_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=TRP_IMPORT_REDIS_TTL, + ) + r.set( + f"{TRP_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=TRP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error("Transportistas import: Redis store error: %s", e) + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = layout_path("imports", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"trp_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"trp_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning("Transportistas import: local file save failed: %s", e) + + scan_file.apply_async(args=[job_id], task_id=job_id) + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception("Transportistas import: background scan failed: %s", e) + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + try: + r = _get_redis() + raw = r.get(f"{TRP_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug("Transportistas import: could not read status from Redis: %s", e) + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Transportistas import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + try: + r = _get_redis() + r.set( + f"{TRP_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=TRP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug("Transportistas import: could not write processing status: %s", e) + + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception("Transportistas import: background commit failed: %s", e) + + threading.Thread(target=run_commit_background, daemon=True).start() + + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/schemas.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/schemas.py new file mode 100644 index 00000000..dc0f5dc5 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + updated: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py new file mode 100644 index 00000000..44128278 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py @@ -0,0 +1,344 @@ +""" +Tareas Celery para importación CSV de Transportistas. +Flujo: scan_file (validación) → commit. +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import json +import logging +import os +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template +from .validators import validate_row_transporter, validate_row_transporter_desfase +from .common.mappers import row_to_transporter_data, row_to_transporter_data_for_update +from .common.fk_loader import load_transportistas_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "trp" + +# Para routes.py (deben coincidir con common_storage.storage_keys(JOB_TYPE, job_id)) +TRP_IMPORT_FILE_PREFIX = "trp_import_file:" +TRP_IMPORT_META_PREFIX = "trp_import_meta:" +TRP_IMPORT_ERROR_LINES_PREFIX = "trp_import_error_lines:" +TRP_IMPORT_STATUS_PREFIX = "trp_import_status:" +TRP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Transportistas import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Transportistas import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv_reader.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + ( + existing_transporter_keys, + valid_country_ame, + state_descriptions_upper, + state_country_set, + ) = load_transportistas_fk_sets(tenant_id, company_id) + + if actualizar: + pass # existing_transporter_keys ya cargado + else: + existing_transporter_keys = set() + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, common_normalize.normalize_header) + _ = validate_row_transporter_desfase(row_norm, i) + err = validate_row_transporter( + row_norm, + i, + actualizar=actualizar, + existing_transporter_keys=existing_transporter_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Transportistas import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{TRP_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Transportistas import: failed to store scan status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Transportistas import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{TRP_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Transportistas import: failed to store scan status in Redis: %s", e) + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Transportistas import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Transportistas import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + ( + existing_transporter_keys, + valid_country_ame, + state_descriptions_upper, + state_country_set, + ) = load_transportistas_fk_sets(tenant_id, company_id) + + from api.v1.modules.a76.transportation.transporters.services import TransporterService + from api.v1.modules.a76.transportation.transporters.dto import TransporterCreateDTO, TransporterUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_transporter( + row_norm, + i, + actualizar=actualizar, + existing_transporter_keys=existing_transporter_keys, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + if err: + skipped_invalid += 1 + tk = (row_norm.get("CLAVE TRANSPORTISTA") or "").strip()[:23] or "-" + skipped_details.append({ + "line": i, + "transporter_key": tk, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + tk = (row_norm.get("CLAVE TRANSPORTISTA") or "").strip()[:23] or "" + if not tk: + skipped_invalid += 1 + continue + tk_upper = tk.upper() + if tk_upper in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append({ + "line": i, + "transporter_key": tk, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) + continue + seen_keys_in_file[tk_upper] = i + + existing = TransporterService.get_by_id(session, tk, tenant_id, company_id) + if not existing: + existing = TransporterService.get_by_id_ignore_case(session, tk, tenant_id, company_id) + try: + if existing: + if actualizar: + data = row_to_transporter_data_for_update(row_norm, existing) + else: + data = row_to_transporter_data(row_norm) + if not data or not data.get("transporter_key"): + skipped_invalid += 1 + continue + update_data = TransporterUpdateDTO(**{k: v for k, v in data.items() if k != "transporter_key"}) + TransporterService.update(session, existing.transporter_key, tenant_id, update_data, company_id) + updated_count += 1 + else: + data = row_to_transporter_data(row_norm) + if not data or not data.get("transporter_key"): + skipped_invalid += 1 + continue + create_data = TransporterCreateDTO(**data) + TransporterService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "transporter_key": tk, + "reason": str(db_err), + }) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("Transportistas import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("Transportistas import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + try: + r = _get_redis() + r.delete(f"{TRP_IMPORT_STATUS_PREFIX}{job_id}") + except Exception as e: + logger.warning("Transportistas import: failed to delete status key: %s", e) + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRP_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Transportistas import: failed to store commit status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Transportistas import: starting commit for job %s", job_id) + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRP_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Transportistas import: failed to store commit status in Redis: %s", e) + return result diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/template_config.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/template_config.py new file mode 100644 index 00000000..d2bf4418 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/template_config.py @@ -0,0 +1,55 @@ +""" +Configuración de plantilla CSV para Transportistas (EstructuraCatTransportistas). +Mapeo Clarion: Col A = CLAVE TRANSPORTISTA, B = NOMBRE, ... R = DIRECTORIO FTP, S = desfase. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "transporters": [ + {"canonical": "CLAVE TRANSPORTISTA", "aliases": ["TRANSPORTISTA", "CLAVE TRANS", "CARRIER KEY"]}, + {"canonical": "NOMBRE", "aliases": ["NOMBRE TRANSPORTISTA", "NAME"]}, + {"canonical": "NOMBRE CORTO", "aliases": ["NOMBRE CORTO TRANS", "SHORT NAME"]}, + {"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE TRANS"]}, + {"canonical": "RFC", "aliases": ["RFC TRANS"]}, + {"canonical": "CALLES", "aliases": ["STREETS", "DIRECCION"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGO POSTAL TRANS", "POSTAL CODE", "CP"]}, + {"canonical": "CIUDAD", "aliases": ["CITY", "CIUDAD TRANS"]}, + {"canonical": "ESTADO", "aliases": ["STATE", "ESTADO TRANS"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY", "PAIS TRANS"]}, + {"canonical": "CODIGO CARGADOR", "aliases": ["COD CARGADOR", "LOADER CODE"]}, + {"canonical": "CODIGO CAAT", "aliases": ["CAAT", "CODIGO CAAT TRANS"]}, + {"canonical": "CODIGO TRANS", "aliases": ["COD TRANS", "TRANSPORT CODE", "SCAC"]}, + {"canonical": "TIPO INTERFASE TRANS", "aliases": ["TIPO INTERFASE", "INTERFACE TYPE"]}, + {"canonical": "SERVIDOR FTP", "aliases": ["FTP SERVER", "SERVIDOR FTP TRANS"]}, + {"canonical": "USUARIO FTP", "aliases": ["FTP USER", "USUARIO FTP TRANS"]}, + {"canonical": "CLAVE ACCESO FTP", "aliases": ["CLAVE FTP", "FTP PASSWORD", "PASSWORD FTP"]}, + {"canonical": "DIRECTORIO FTP", "aliases": ["FTP DIRECTORY", "DIRECTORIO FTP TRANS"]}, + {"canonical": "COL_EXTRA", "aliases": ["COLUMNA S", "COL S", "COL 19"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("transporters") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/__init__.py new file mode 100644 index 00000000..d4bd5455 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_transporter, validate_row_transporter_desfase + +__all__ = ["validate_row_transporter", "validate_row_transporter_desfase"] diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/common.py new file mode 100644 index 00000000..b4c06dca --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/common.py @@ -0,0 +1,94 @@ +""" +Validaciones comunes de fila para import CSV de transportistas. +Paridad Clarion: VALIDACIONES_TRANSPORTISTAS, VALIDA_TODA_TRANSPORTISTAS, VALIDA_PARCIAL_TRANSPORTISTAS. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from ..common.common_validators import ( + MAX_LEN, + check_required, + check_max_length, + check_pais_catalog_transportistas, + check_estado_catalog_transportistas, + check_estado_pais_consistency_transportistas, +) + + +def validate_row_transporter_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col A (CLAVE TRANSPORTISTA) obligatoria.""" + return check_required(row, "CLAVE TRANSPORTISTA", MAX_LEN["transporter_key"], line_num) + + +def validate_row_transporter_required_full( + row: Dict[str, Any], + line_num: int, +) -> Optional[Dict[str, Any]]: + """VALIDA_TODA: Col A y Col B (NOMBRE) obligatorios cuando se agrega o se reemplaza.""" + err = validate_row_transporter_required(row, line_num) + if err: + return err + err = check_max_length( + row, "NOMBRE", MAX_LEN["name"], line_num, required=True + ) + if err: + return err + return None + + +def validaciones_transportistas( + row: Dict[str, Any], + line_num: int, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """ + VALIDACIONES_TRANSPORTISTAS: estado, país, estado-país (Clarion). + """ + err = check_estado_catalog_transportistas(row, line_num, state_descriptions_upper) + if err: + return err + err = check_pais_catalog_transportistas(row, line_num, valid_country_ame) + if err: + return err + err = check_estado_pais_consistency_transportistas(row, line_num, state_country_set) + if err: + return err + return None + + +def valida_toda_transportistas( + row: Dict[str, Any], + line_num: int, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_TODA: obligatorios A y B + VALIDACIONES_TRANSPORTISTAS (usado para agregar nuevo o reemplazar).""" + err = validate_row_transporter_required_full(row, line_num) + if err: + return err + return validaciones_transportistas( + row, + line_num, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + + +def valida_parcial_transportistas( + row: Dict[str, Any], + line_num: int, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_PARCIAL_TRANSPORTISTAS: solo VALIDACIONES_TRANSPORTISTAS (actualizar registro existente).""" + return validaciones_transportistas( + row, + line_num, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/create.py new file mode 100644 index 00000000..384e6e05 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/validators/create.py @@ -0,0 +1,60 @@ +""" +Punto de entrada de validación para import de una fila transportista. +Actualizar = merge: si no existe la clave se crea (valida_toda), si existe se actualizan solo campos enviados (valida_parcial). +Reemplazar = reescribir: siempre valida_toda; en commit se sustituye el registro completo o se agrega. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from .common import ( + validate_row_transporter_required, + valida_toda_transportistas, + valida_parcial_transportistas, +) +from ..common.common_validators import check_desfase_transportistas + + +def validate_row_transporter( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, + existing_transporter_keys: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de transportistas. + 1. CLAVE TRANSPORTISTA (Col A) vacía → error. + 2. Si actualizar y clave existe en catálogo → valida_parcial (solo validaciones de catálogo; campos vacíos = mantener actual). + 3. Si reemplazar o (actualizar y clave no existe) → valida_toda (A y B obligatorios + validaciones). + """ + err = validate_row_transporter_required(row, line_num) + if err: + return err + + existing = existing_transporter_keys or set() + clave = (row.get("CLAVE TRANSPORTISTA") or "").strip().upper() + use_partial = actualizar and bool(clave and clave in existing) + + if use_partial: + err = valida_parcial_transportistas( + row, + line_num, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + else: + err = valida_toda_transportistas( + row, + line_num, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + return err + + +def validate_row_transporter_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Advertencia de desfase si COL_EXTRA (Col S) tiene valor. No bloqueante.""" + return check_desfase_transportistas(row, line_num) diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py new file mode 100644 index 00000000..d7025515 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for us_tariff_fractions) diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py new file mode 100644 index 00000000..b77457c3 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py @@ -0,0 +1,83 @@ +""" +Validadores reutilizables para import CSV de fracciones arancelarias americanas. +Paridad Clarion: Col A max 16, Col C en catálogo U.M., Col E PO/ME o vacío (default PO). +""" +from decimal import Decimal +from typing import Dict, Any, Optional, Set + +CODE_MAX = 16 +PREFIX_MAX = 10 +UNIT_MAX = 10 +TYPE_MAX = 10 + +# Clarion: Tipo Advalorem PO (Porcentaje), ME (Costos Fijo Dlls) o vacío → PO +TIPO_ADVALOREM_VALIDOS = frozenset({"PO", "ME"}) + + +def normalize_code(raw: Optional[str]) -> str: + """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" + if not raw: + return "" + s = str(raw).strip().replace(".", "").replace("-", "") + return s[:CODE_MAX] if len(s) > CODE_MAX else s + + +def check_required_code(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + code_raw = (row.get(col) or "").strip() + if not code_raw: + return {"line": line_num, "col": col, "msg": "Requerido"} + code_norm = normalize_code(code_raw) + if not code_norm: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(code_norm) > CODE_MAX: + return {"line": line_num, "col": col, "msg": f"Máximo {CODE_MAX} caracteres"} + return None + + +def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def parse_float_min_zero(val: Any) -> Optional[float]: + if val is None or str(val).strip() == "": + return None + try: + v = float(str(val).strip().replace(",", ".")) + return v if v >= 0 else None + except (ValueError, TypeError): + return None + + +def check_optional_decimal_min_zero(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or not str(val).strip(): + return None + try: + v = float(str(val).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": col, "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": col, "msg": "Debe ser un número"} + return None + + +def check_tipo_advalorem(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + """Col E: solo PO, ME o vacío. Clarion mensaje.""" + val = (row.get(col) or "").strip().upper() + if not val: + return None + if val in TIPO_ADVALOREM_VALIDOS: + return None + return { + "line": line_num, + "col": col, + "msg": ( + f"Error: (Col. E) La opción de Tipo de Advalorem: {row.get(col) or ''} no es valido. " + "Capturar una opción valida: PO para Porcentaje, ME para Costos Fijo en Dlls o dejar el campo vacio y automaticamente se asigna Porcentaje." + ), + } diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/fk_loader.py new file mode 100644 index 00000000..e2da98a4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/fk_loader.py @@ -0,0 +1,52 @@ +""" +Carga de conjuntos FK para validación/mapeo de import CSV de Fracciones Americanas. +Clarion: U.M. (GUniMedida), códigos existentes de fracción americana (GFracAme). +""" +from typing import Set, Tuple + +from core.database import CoreSessionLocal + + +def load_fa_fk_sets( + tenant_id: int, + company_id: int, +) -> Tuple[Set[str], Set[str]]: + """ + Carga conjuntos para validación CSV Fracciones Americanas (paridad Clarion). + Devuelve (valid_uom_codes, existing_fraction_codes). + - valid_uom_codes: códigos de Unidad de Medida (a76.units_of_measure, code UPPER, max 5 chars). + - existing_fraction_codes: códigos de USTariffFraction ya existentes por tenant/company. + """ + valid_uom_codes: Set[str] = set() + existing_fraction_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + + for row in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + if row[0]: + valid_uom_codes.add((row[0].strip() or "").upper()[:5]) + + for row in ( + session.query(USTariffFraction.code) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + ) + .all() + ): + if row[0]: + existing_fraction_codes.add(row[0].strip()) + + except Exception as e: + import logging + logging.getLogger(__name__).warning("FA import: could not load FK sets: %s", e) + return valid_uom_codes, existing_fraction_codes diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py new file mode 100644 index 00000000..6f0be32e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py @@ -0,0 +1,90 @@ +""" +Mapeo fila CSV → datos para USTariffFraction. +""" +from decimal import Decimal +from typing import Dict, Any, Optional + +from .common_validators import ( + normalize_code, + parse_float_min_zero, +) + +MAX_LEN = {"prefix": 10, "unit_of_measure": 10, "type_code": 10} + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_us_tariff_fraction_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int, existing: Optional[Any] = None +) -> Dict[str, Any]: + """ + Build dict for USTariffFraction model. + Si existing está presente (modo Actualizar), campos vacíos en row_norm se rellenan desde existing (VALIDA_PARCIAL Clarion). + Col E vacío → type_code 'PO'. + """ + code = normalize_code(row_norm.get("FRACCION_ARANCELARIA")) + if not code: + return {} + + def _prefijo() -> Optional[str]: + v = _str_or_none(row_norm.get("PREFIJO"), MAX_LEN["prefix"]) + if v is not None: + return v + return getattr(existing, "prefix", None) if existing else None + + def _unit() -> Optional[str]: + v = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), MAX_LEN["unit_of_measure"]) + if v is not None: + return v + return getattr(existing, "unit_of_measure", None) if existing else None + + def _desc() -> Optional[str]: + v = _str_or_none(row_norm.get("DESCRIPCION")) + if v is not None: + return v + return getattr(existing, "description", None) if existing else None + + def _type_code() -> Optional[str]: + v = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), MAX_LEN["type_code"]) + if v is not None: + return v.upper() if v else "PO" + if existing is not None: + return getattr(existing, "type_code", None) or "PO" + return "PO" + + def _adv() -> Optional[float]: + x = parse_float_min_zero(row_norm.get("ADVALOREM_PCT")) + if x is not None: + return x + return getattr(existing, "ad_valorem", None) if existing else None + + def _fixed() -> Optional[Decimal]: + fixed_cost_raw = parse_float_min_zero(row_norm.get("ADVALOREM_DLLS")) + if fixed_cost_raw is not None: + return Decimal(str(round(fixed_cost_raw, 8))) + if existing is not None: + v = getattr(existing, "fixed_cost", None) + return Decimal(str(v)) if v is not None else None + return None + + fixed_cost = _fixed() + return { + "tenant_id": tenant_id, + "company_id": company_id, + "code": code, + "prefix": _prefijo(), + "type_code": _type_code(), + "ad_valorem": _adv(), + "fixed_cost": fixed_cost, + "unit_of_measure": _unit(), + "description": _desc(), + } diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py similarity index 94% rename from backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py index bf5083a3..87e4ad57 100644 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py @@ -14,6 +14,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -39,11 +40,13 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo Agregar/Actualizar (ACT); si False, Agregar/Reemplazar"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): """ Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + actualizar=True simula Clarion 'Agr./Actual.'; actualizar=False 'Agr./Reempl.'. """ try: tenant_id = validate_access_to_resource(db, company_id, current_user) @@ -62,6 +65,7 @@ async def upload_import_file( "company_id": company_id, "user_id": current_user.get("id"), "template_id": "us_tariff_fractions", + "actualizar": actualizar, } try: @@ -81,7 +85,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py new file mode 100644 index 00000000..a7714ada --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py @@ -0,0 +1,264 @@ +""" +Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import json +import logging +import os +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template +from .validators import validate_row_us_tariff_fraction, validate_row_desfase_fa +from .common.mappers import row_to_us_tariff_fraction_data +from .common.fk_loader import load_fa_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "fa" +TEMPLATE_ID = "us_tariff_fractions" + +# Para routes.py +FA_IMPORT_FILE_PREFIX = "fa_import_file:" +FA_IMPORT_META_PREFIX = "fa_import_meta:" +FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:" +FA_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: + return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "FA import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "FA import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv_reader.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta_data = common_meta.load_meta(file_path) + actualizar = meta_data.get("actualizar", False) + valid_uom_codes, existing_fraction_codes = load_fa_fk_sets(tenant_id, company_id) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + warnings_detail: List[Dict[str, Any]] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + warn = validate_row_desfase_fa(row, i) + if warn and len(warnings_detail) < 500: + warnings_detail.append({ + "line": warn["line"], + "col": warn.get("col", ""), + "msg": warn.get("msg", ""), + }) + + row_norm = _norm_row(row) + err = validate_row_us_tariff_fraction( + row_norm, + i, + actualizar=actualizar, + existing_fraction_codes=existing_fraction_codes, + valid_uom_codes=valid_uom_codes, + raw_row=row, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("FA import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + result = common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + if warnings_detail: + result["warnings"] = warnings_detail + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("FA import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "FA import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "FA import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta_data = common_meta.load_meta(file_path) + actualizar = meta_data.get("actualizar", False) + valid_uom_codes, existing_fraction_codes = load_fa_fk_sets(tenant_id, company_id) + + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = _norm_row(row) + err = validate_row_us_tariff_fraction( + row_norm, + i, + actualizar=actualizar, + existing_fraction_codes=existing_fraction_codes, + valid_uom_codes=valid_uom_codes, + raw_row=row, + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + data = row_to_us_tariff_fraction_data(row_norm, tenant_id, company_id) + if not data or not data.get("code"): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA vacío"}) + continue + + existing = ( + session.query(USTariffFraction) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + USTariffFraction.code == data["code"], + ) + .first() + ) + if existing: + data = row_to_us_tariff_fraction_data( + row_norm, tenant_id, company_id, existing=existing + ) + existing.prefix = data.get("prefix") + existing.type_code = data.get("type_code") + existing.ad_valorem = data.get("ad_valorem") + existing.fixed_cost = data.get("fixed_cost") + existing.unit_of_measure = data.get("unit_of_measure") + existing.description = data.get("description") + session.add(existing) + updated_count += 1 + else: + new_row = USTariffFraction(**data) + session.add(new_row) + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("FA import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("FA import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + + if inserted_count == 0 and updated_count == 0 and skipped_invalid > 0: + return { + "status": "warning", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0 and updated_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("FA import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/template_config.py similarity index 88% rename from backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/template_config.py index aa5a5e7f..8a34a31e 100644 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/template_config.py @@ -14,9 +14,14 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]}, {"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]}, {"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]}, + {"canonical": "COL_EXTRA", "aliases": ["DESFASE"]}, ], } +# Orden A..H para detectar desfase en 8ª columna (raw_row.values()[7]) +TEMPLATE_ORDER = [item["canonical"] for item in TEMPLATE_COLUMNS["us_tariff_fractions"]] +DESFASE_COLUMN_INDEX = 7 + def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]: """normalized_header -> canonical_name para plantilla us_tariff_fractions.""" diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py new file mode 100644 index 00000000..389af650 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_us_tariff_fraction, validate_row_desfase_fa + +__all__ = ["validate_row_us_tariff_fraction", "validate_row_desfase_fa"] diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py new file mode 100644 index 00000000..e0c20c6e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py @@ -0,0 +1,173 @@ +""" +Validaciones comunes de fila para import CSV de fracciones arancelarias americanas. +Paridad Clarion: desfase Col H, VALIDA_TODA_FRACCIONAME / VALIDA_PARCIAL_FRACCIONAME, VALIDACIONES_FRACCIONAME. +""" +from typing import Dict, Any, Optional, Set + +from ..template_config import DESFASE_COLUMN_INDEX +from ..common.common_validators import ( + normalize_code, + check_optional_max_length, + check_optional_decimal_min_zero, + check_tipo_advalorem, + PREFIX_MAX, + UNIT_MAX, + CODE_MAX, +) +from ..common.common_validators import TIPO_ADVALOREM_VALIDOS # noqa: F401 re-export + +# Mensajes Clarion +MSG_COL_A_VACIO = ( + "Error: (Col. A) La columna de Fracción Americana esta vacia y no se pueden hacer las validaciones. " + "Capturar en la Columna A una Fracción Americana nueva o una ya existente al cual desee actualizar campos" +) +MSG_COL_A_LONGITUD = ( + "Capturar en la columna A una Fraccion Arancelaria de 16 caracteres como máximo." +) +MSG_COL_D_OBLIGATORIO = ( + "Existen campos vacios que son obligatorios, es la (Col.D) Descripción. " + "Revisar la línea del archivo y capturar los campos con la información correcta." +) +MSG_FRACCION_NO_EXISTE = ( + "Error: (Col. A) La Fraccion Americana no existe." +) +MSG_DESFASE = "Advertencia: Podría existir un desfase en esta línea." +MSG_DESFASE_SOLUCION = "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta." + + +def validate_row_desfase_fa( + raw_row: Optional[Dict[str, Any]], line_num: int +) -> Optional[Dict[str, Any]]: + """ + Si la fila tiene 8+ columnas y la 8ª (Col H) tiene valor → advertencia (no bloqueante). + Devuelve dict con severity=warning para que el caller pueda acumularlo en warnings_detail. + """ + if not raw_row: + return None + values_ordered = list(raw_row.values()) + if len(values_ordered) <= DESFASE_COLUMN_INDEX: + return None + if not (values_ordered[DESFASE_COLUMN_INDEX] or "").strip(): + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": MSG_DESFASE, + "solution": MSG_DESFASE_SOLUCION, + "severity": "warning", + } + + +def _validate_col_a_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col A obligatoria; mensaje Clarion.""" + code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip() + if not code_raw: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": MSG_COL_A_VACIO} + code_norm = normalize_code(code_raw) + if not code_norm: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": MSG_COL_A_VACIO} + if len(code_norm) > CODE_MAX: + return { + "line": line_num, + "col": "FRACCION_ARANCELARIA", + "msg": f"Error: (Col. A) La Fraccion Americana: {code_raw} supera la longitud de caracteres. {MSG_COL_A_LONGITUD}", + } + return None + + +def _validate_col_d_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col D obligatoria en validación completa (VALIDA_TODA).""" + if (row.get("DESCRIPCION") or "").strip(): + return None + return {"line": line_num, "col": "DESCRIPCION", "msg": MSG_COL_D_OBLIGATORIO} + + +def _validate_uom_catalog( + row: Dict[str, Any], line_num: int, valid_uom_codes: Optional[Set[str]] +) -> Optional[Dict[str, Any]]: + """Col C: si no vacía, debe existir en catálogo U.M. (Clarion GUniMedida).""" + val = (row.get("UNIDAD_DE_MEDIDA") or "").strip().upper() + if not val or valid_uom_codes is None: + return None + if val in valid_uom_codes: + return None + return { + "line": line_num, + "col": "UNIDAD_DE_MEDIDA", + "msg": ( + f"Error: (Col. C) La Unidad de Medida: {row.get('UNIDAD_DE_MEDIDA') or ''} no existe en el Catálogo de U.M. " + "Revisar esta Unidad de Medida en el archivo, en caso de ser correcta dar la de alta en el Catálogo de U.M." + ), + } + + +def _validaciones_fraccioname( + row: Dict[str, Any], + line_num: int, + valid_uom_codes: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + """VALIDACIONES_FRACCIONAME: Col A len ≤16, Col C en catálogo U.M., Col E PO/ME/vacío, F/G ≥0.""" + err = _validate_col_a_required(row, line_num) + if err: + return err + err = check_optional_max_length(row, "PREFIJO", PREFIX_MAX, line_num) + if err: + return err + err = check_optional_max_length(row, "UNIDAD_DE_MEDIDA", UNIT_MAX, line_num) + if err: + return err + err = _validate_uom_catalog(row, line_num, valid_uom_codes) + if err: + return err + err = check_tipo_advalorem(row, "TIPO_DE_ADVALOREM", line_num) + if err: + return err + err = check_optional_decimal_min_zero(row, "ADVALOREM_PCT", line_num) + if err: + return err + err = check_optional_decimal_min_zero(row, "ADVALOREM_DLLS", line_num) + if err: + return err + return None + + +def validate_row_us_tariff_fraction( + row: Dict[str, Any], + line_num: int, + *, + actualizar: bool = False, + existing_fraction_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + raw_row: Optional[Dict[str, Any]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de fracción arancelaria americana. + - raw_row: si se pasa y la 8ª columna tiene valor, se devuelve advertencia de desfase (no bloqueante; el caller decide si la trata como error). + - Col A vacía → error. + - Si actualizar y fracción no existe en existing_fraction_codes → error "La Fracción Americana no existe". + - Si actualizar y fracción existe → VALIDA_PARCIAL (solo VALIDACIONES_FRACCIONAME; Col D no obligatoria). + - Si no actualizar → VALIDA_TODA (Col D obligatoria + VALIDACIONES_FRACCIONAME). + """ + # Desfase: advertencia (no bloqueante por defecto; se devuelve como error para que scan la registre en errors_detail pero no bloquea commit si no está en error_lines) + # Plan: "advertencia no bloqueante" → no añadimos a error_lines; guardamos en warnings. Para simplificar, desfase lo devolvemos como error de severidad warning y en tasks no lo añadimos a error_lines (solo a warnings_detail). Mejor: desfase retornamos None (no error) y el caller puede llamar a validate_row_desfase_fa por separado y acumular warnings. Así no bloqueamos. Entonces en validate_row_us_tariff_fraction no llamamos desfase como error; en tasks llamamos primero validate_row_desfase_fa y si hay warning lo guardamos en warnings_detail, luego llamamos validate_row_us_tariff_fraction que puede devolver error. OK. + # So we don't return desfase from validate_row_us_tariff_fraction; tasks will call validate_row_desfase_fa and collect warnings. So no change here for desfase inside this function. + + err = _validate_col_a_required(row, line_num) + if err: + return err + + code_norm = normalize_code((row.get("FRACCION_ARANCELARIA") or "").strip()) + fraction_exists = ( + existing_fraction_codes is not None and code_norm in existing_fraction_codes + ) + + if actualizar and not fraction_exists: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": MSG_FRACCION_NO_EXISTE} + + use_full = not actualizar or not fraction_exists + if use_full: + err = _validate_col_d_required(row, line_num) + if err: + return err + + return _validaciones_fraccioname(row, line_num, valid_uom_codes) diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py new file mode 100644 index 00000000..10ec95f4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py @@ -0,0 +1,10 @@ +""" +Punto de entrada de validación para import de una fila fracción arancelaria americana. +Firma: validate_row_us_tariff_fraction(row, line_num, *, actualizar=False, existing_fraction_codes=None, valid_uom_codes=None, raw_row=None). +""" +from .common import ( + validate_row_us_tariff_fraction, + validate_row_desfase_fa, +) + +__all__ = ["validate_row_us_tariff_fraction", "validate_row_desfase_fa"] diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/__init__.py similarity index 100% rename from backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py rename to backend/api/v1/modules/a76/layouts_csv/vehicles/__init__.py diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py new file mode 100644 index 00000000..0dd687d4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for vehicles) diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py new file mode 100644 index 00000000..fd588f6e --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py @@ -0,0 +1,248 @@ +""" +Validadores reutilizables para import CSV de vehículos (longitudes, decimal, fecha aseguradora). +Paridad Clarion: VALIDACIONES_TRANSPORTE, código entidad C/I/A/B, catálogos tipo/país/estado, desfase. +""" +import re +from decimal import Decimal +from typing import Dict, Any, Optional, Set, Tuple + +# Max lengths from Vehicle model (a76.vehicle). Clarion Col A máx 15; modelo actual 14. +MAX_LEN = { + "vehicle_key": 14, + "ace_vehicle_key": 10, + "transporter_key": 23, + "transport_identifier": 30, + "transport_type": 2, + "entity_code": 1, + "transponder_number": 16, + "dot_number": 8, + "plate_number": 17, + "city": 30, + "state": 30, + "country": 3, + "seal": 49, + "insurance_company_name": 30, + "insurance_number": 20, + "series": 30, +} + + +def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length( + row: Dict[str, Any], col: str, max_len: int, line_num: int, required: bool = False +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + s = str(val).strip().replace(",", "") + return Decimal(s) + except Exception: + return None + + +def parse_insurance_date(val: Any) -> Optional[int]: + """Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD.""" + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if not s: + return None + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + pass + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + pass + break + return None + + +def check_optional_decimal(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or (isinstance(val, str) and not val.strip()): + return None + if parse_decimal(val) is None: + return {"line": line_num, "col": col, "msg": "Debe ser numérico"} + return None + + +def check_optional_insurance_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or (isinstance(val, str) and not val.strip()): + return None + if parse_insurance_date(val) is None: + return {"line": line_num, "col": col, "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} + return None + + +# --- Clarion VALIDACIONES_TRANSPORTE: dominio y catálogos --- + +CODIGO_ENTIDAD_VALIDOS = {"C", "I", "A", "B"} + + +def check_codigo_entidad_valores(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Col F: Si CODIGO DE ENTIDAD no vacío, debe ser C, I, A o B (Clarion).""" + val = (row.get("CODIGO DE ENTIDAD") or "").strip().upper() + if not val: + return None + if val in CODIGO_ENTIDAD_VALIDOS: + return None + return { + "line": line_num, + "col": "CODIGO DE ENTIDAD", + "msg": f"Error: (Col. F) El Codigo de Entidad: {val} es incorrecto.", + "solution": "Capturar en columna F el Codigo de Entidad correcto, C, I, A ó B.", + } + + +def check_codigo_entidad_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """VALIDA_TODA: Col F (Codigo de Entidad) obligatorio cuando registro es nuevo.""" + val = (row.get("CODIGO DE ENTIDAD") or "").strip() + if val: + return None + return { + "line": line_num, + "col": "CODIGO DE ENTIDAD", + "msg": "Existen campos vacios que son obligatorios, es la (Col.F) Codigo de Entidad.", + "solution": "Revisar la línea del archivo y capturar los campos con la información correcta.", + } + + +def check_transport_type_catalog( + row: Dict[str, Any], + line_num: int, + valid_transport_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col E: Si TIPO TRANSPORTE no vacío, debe existir en catálogo (GTipoTransportes).""" + val = (row.get("TIPO TRANSPORTE") or "").strip() + if not val or valid_transport_codes is None: + return None + if val.upper() in valid_transport_codes: + return None + return { + "line": line_num, + "col": "TIPO TRANSPORTE", + "msg": f"Error: (Col. E) El Tipo de Transporte: {val} es incorrecto.", + "solution": "Capturar en columna E algun Tipo de Transporte correcto.", + } + + +def check_pais_catalog_vehicles( + row: Dict[str, Any], + line_num: int, + valid_country_ame: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col L: Si PAIS no vacío, debe ser clave americana (2 chars) y existir en catálogo (GPaises.Pais_Ame).""" + val = (row.get("PAIS") or "").strip() + if not val or valid_country_ame is None: + return None + val_upper = val.upper() + if len(val) > 2: + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Celda L{line_num}) El Pais: {val} Es Incorrecto", + "solution": "Capturar en columna L un Pais en Clave Americana(US = Estados Unidos, MX = Mexico, ES = España, etc).", + } + if val_upper in valid_country_ame: + return None + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Col. L) El Pais: {val} es incorrecto.", + "solution": "Capturar en columna L el Pais del Transporte en Clave Americana.", + } + + +def check_estado_catalog( + row: Dict[str, Any], + line_num: int, + state_descriptions_upper: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Col K: Si ESTADO no vacío, debe existir en catálogo (GEstados). Si hay estado, PAIS no puede estar vacío.""" + val = (row.get("ESTADO") or "").strip() + if not val or state_descriptions_upper is None: + return None + val_upper = val.upper() + if val_upper not in state_descriptions_upper: + return { + "line": line_num, + "col": "ESTADO", + "msg": f"Error: (Celda K{line_num}) El Estado: {val} es incorrecto.", + "solution": "Capturar en columna K el Estado del Transporte en Clave Americana o Nombre Completo.", + } + # Si existe estado, PAIS no puede estar vacío (Clarion) + pais = (row.get("PAIS") or "").strip() + if not pais: + return { + "line": line_num, + "col": "PAIS", + "msg": f"Error: (Celda K{line_num}) El Estado: {val} No esta ligado a ningun Pais.", + "solution": "Capturar en columna L un Pais en Clave Americana(US = Estados Unidos, MX = Mexico, ES = España, etc).", + } + return None + + +def check_estado_pais_consistency( + row: Dict[str, Any], + line_num: int, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """Si hay ESTADO y PAIS, validar que el estado pertenezca al país (Clarion GEstados-GPaises).""" + estado = (row.get("ESTADO") or "").strip() + pais = (row.get("PAIS") or "").strip().upper() + if not estado or not pais or state_country_set is None: + return None + key = (pais, estado.upper()) + if key in state_country_set: + return None + return { + "line": line_num, + "col": "ESTADO", + "msg": f"Error: (Col. K) EL Estado: {estado} no pertenece al Pais: {pais}.", + "solution": "Capturar en columna K un Estado que pertenesca al Pais de la columna L.", + } + + +def check_desfase_vehicles(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Si COL_EXTRA (columna 18 / Col R) tiene valor → advertencia de desfase (Clarion, no bloqueante por defecto).""" + val = (row.get("COL_EXTRA") or "").strip() + if not val: + return None + return { + "line": line_num, + "col": "COL_EXTRA", + "msg": "Advertencia: Podría existir un desfase en esta línea.", + "solution": "Revisar esta línea del archivo CSV y verificar cada campo este en la posicion correcta.", + "warning": True, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py new file mode 100644 index 00000000..f76aa77d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py @@ -0,0 +1,73 @@ +""" +Carga de conjuntos FK para validación de import CSV de vehículos (transportes). +Clarion: GTipoTransportes, GPaises (Pais_Ame), GEstados (Descripcion / Clave_Ame), relación Estado-País. +""" +from typing import Set, Tuple, Optional +import logging + +from core.database import CoreSessionLocal + +logger = logging.getLogger(__name__) + + +def load_vehicles_fk_sets( + tenant_id: Optional[int] = None, + company_id: Optional[int] = None, +) -> Tuple[ + Set[str], + Set[str], + Set[str], + Set[Tuple[str, str]], +]: + """ + Carga conjuntos para validación CSV de vehículos (paridad Clarion). + Devuelve: + - valid_transport_codes: códigos de transport_types (GTipoTransportes) + - valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas + - state_descriptions_upper: descripciones de estados en mayúsculas (GEstados), para "estado existe" + - state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país" + """ + valid_transport_codes: Set[str] = set() + valid_country_ame: Set[str] = set() + state_descriptions_upper: Set[str] = set() + state_country_set: Set[Tuple[str, str]] = set() + + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.transport_types.models import TransportType + from api.v1.modules.public.reference_data.countries.models import Country + from api.v1.modules.public.reference_data.states.models import State + + for row in session.query(TransportType.transport_code).all(): + if row[0]: + valid_transport_codes.add((row[0] or "").strip().upper()) + + for row in session.query(Country.ame_key).all(): + if row[0]: + valid_country_ame.add((row[0] or "").strip().upper()) + + # States: description (GEstados.Descripcion); State.m3_key = Country.m3_key + for state in session.query(State).all(): + desc = (state.description or "").strip() + if desc: + state_descriptions_upper.add(desc.upper()) + # País para este estado vía m3_key + country = ( + session.query(Country) + .filter(Country.m3_key == state.m3_key) + .first() + ) + if country and (country.ame_key or "").strip(): + state_country_set.add( + ((country.ame_key or "").strip().upper(), desc.upper()) + ) + + except Exception as e: + logger.warning("Vehicles import: could not load FK sets: %s", e) + + return ( + valid_transport_codes, + valid_country_ame, + state_descriptions_upper, + state_country_set, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py new file mode 100644 index 00000000..a44f7ff9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py @@ -0,0 +1,117 @@ +""" +Mapeo fila CSV → datos para Vehicle. +Paridad Clarion LLENA_TRANSPORTE: en actualización, campo vacío en CSV usa valor existente del vehículo. +""" +from typing import Dict, Any, Optional, Union + +from .common_validators import ( + MAX_LEN, + parse_decimal, + parse_insurance_date, +) + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _get_existing_val(existing: Union[Any, Dict[str, Any]], key: str) -> Any: + """Obtiene valor del vehículo existente (modelo ORM o dict).""" + if existing is None: + return None + if isinstance(existing, dict): + return existing.get(key) + return getattr(existing, key, None) + + +def row_to_vehicle_data(row_norm: Dict[str, Any]) -> Dict[str, Any]: + """Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row.""" + vehicle_key = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["vehicle_key"]) + if not vehicle_key: + return {} + return { + "vehicle_key": vehicle_key, + "ace_vehicle_key": _str_or_none(row_norm.get("CLAVE ACE"), MAX_LEN["ace_vehicle_key"]), + "transporter_key": _str_or_none(row_norm.get("CLAVE TRANSPORTE"), MAX_LEN["transporter_key"]), + "series": _str_or_none(row_norm.get("VIN"), MAX_LEN["series"]), + "transport_type": _str_or_none(row_norm.get("TIPO TRANSPORTE"), MAX_LEN["transport_type"]), + "entity_code": _str_or_none(row_norm.get("CODIGO DE ENTIDAD"), MAX_LEN["entity_code"]), + "transponder_number": _str_or_none(row_norm.get("TRANSPONDEDOR"), MAX_LEN["transponder_number"]), + "dot_number": _str_or_none(row_norm.get("NUMERO DOT"), MAX_LEN["dot_number"]), + "plate_number": _str_or_none(row_norm.get("PLACAS"), MAX_LEN["plate_number"]), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "seal": _str_or_none(row_norm.get("PRECINTO"), MAX_LEN["seal"]), + "insurance_company_name": _str_or_none(row_norm.get("EMPRESA ASEGURADORA"), MAX_LEN["insurance_company_name"]), + "insurance_number": _str_or_none(row_norm.get("NUM. ASEGURADORA"), MAX_LEN["insurance_number"]), + "insurance_amount": parse_decimal(row_norm.get("MONTO ASEGURADO") or row_norm.get("MONTO")), + "insurance_date": parse_insurance_date(row_norm.get("FECHA DE ASEGURADORA") or row_norm.get("FECHA ASEGURADORA")), + } + + +def row_to_vehicle_data_for_update( + row_norm: Dict[str, Any], + existing_vehicle: Union[Any, Dict[str, Any]], +) -> Dict[str, Any]: + """ + Build dict for VehicleUpdateDTO: CSV value if non-empty, else existing vehicle value (Clarion VALIDA_PARCIAL / LLENA_TRANSPORTE). + """ + vehicle_key = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["vehicle_key"]) or _get_existing_val(existing_vehicle, "vehicle_key") + if not vehicle_key: + return {} + + def _csv_or_existing(csv_key: str, dto_key: str, max_len: Optional[int] = None): + v = _str_or_none(row_norm.get(csv_key), max_len) + if v is not None and v != "": + return v + return _get_existing_val(existing_vehicle, dto_key) + + def _csv_decimal_or_existing(csv_key: str, dto_key: str): + raw = row_norm.get("MONTO ASEGURADO") or row_norm.get("MONTO") if csv_key == "MONTO ASEGURADO" else row_norm.get(csv_key) + if raw is not None and str(raw).strip(): + d = parse_decimal(raw) + if d is not None: + return float(d) + val = _get_existing_val(existing_vehicle, dto_key) + if val is not None and hasattr(val, "__float__"): + try: + return float(val) + except (TypeError, ValueError): + pass + return val + + def _csv_date_or_existing(csv_key: str, dto_key: str): + raw = row_norm.get("FECHA DE ASEGURADORA") or row_norm.get("FECHA ASEGURADORA") + if raw is not None and str(raw).strip(): + d = parse_insurance_date(raw) + if d is not None: + return d + return _get_existing_val(existing_vehicle, dto_key) + + return { + "vehicle_key": vehicle_key, + "ace_vehicle_key": _csv_or_existing("CLAVE ACE", "ace_vehicle_key", MAX_LEN["ace_vehicle_key"]), + "transporter_key": _csv_or_existing("CLAVE TRANSPORTE", "transporter_key", MAX_LEN["transporter_key"]), + "series": _csv_or_existing("VIN", "series", MAX_LEN["series"]), + "transport_type": _csv_or_existing("TIPO TRANSPORTE", "transport_type", MAX_LEN["transport_type"]), + "entity_code": _csv_or_existing("CODIGO DE ENTIDAD", "entity_code", MAX_LEN["entity_code"]), + "transponder_number": _csv_or_existing("TRANSPONDEDOR", "transponder_number", MAX_LEN["transponder_number"]), + "dot_number": _csv_or_existing("NUMERO DOT", "dot_number", MAX_LEN["dot_number"]), + "plate_number": _csv_or_existing("PLACAS", "plate_number", MAX_LEN["plate_number"]), + "city": _csv_or_existing("CIUDAD", "city", MAX_LEN["city"]), + "state": _csv_or_existing("ESTADO", "state", MAX_LEN["state"]), + "country": _csv_or_existing("PAIS", "country", MAX_LEN["country"]), + "seal": _csv_or_existing("PRECINTO", "seal", MAX_LEN["seal"]), + "insurance_company_name": _csv_or_existing("EMPRESA ASEGURADORA", "insurance_company_name", MAX_LEN["insurance_company_name"]), + "insurance_number": _csv_or_existing("NUM. ASEGURADORA", "insurance_number", MAX_LEN["insurance_number"]), + "insurance_amount": _csv_decimal_or_existing("MONTO ASEGURADO", "insurance_amount"), + "insurance_date": _csv_date_or_existing("FECHA DE ASEGURADORA", "insurance_date"), + } diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py similarity index 96% rename from backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py rename to backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py index 20960753..1d6c2e5f 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py @@ -15,6 +15,7 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db +from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse @@ -42,6 +43,7 @@ def _get_redis(): async def upload_import_file( file: UploadFile = File(...), company_id: int = Query(..., description="Company ID"), + actualizar: bool = Query(False, description="Modo Agregar/Actualizar (ACT); si True, validación parcial para claves existentes"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -62,6 +64,7 @@ async def upload_import_file( "company_id": company_id, "user_id": current_user.get("id"), "template_id": "vehicles", + "actualizar": actualizar, } try: @@ -81,7 +84,7 @@ async def upload_import_file( raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") try: - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + upload_dir = layout_path("imports", "temp") os.makedirs(upload_dir, exist_ok=True) with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f: f.write(contents) diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/schemas.py similarity index 100% rename from backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py rename to backend/api/v1/modules/a76/layouts_csv/vehicles/schemas.py diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py new file mode 100644 index 00000000..30436b5f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py @@ -0,0 +1,378 @@ +""" +Tareas Celery para importación CSV de Vehículos (Transportes). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). +""" +import json +import logging +import os +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader +from .template_config import row_from_template +from .validators import validate_row_vehicle, validate_row_vehicle_desfase +from .common.mappers import row_to_vehicle_data, row_to_vehicle_data_for_update +from .common.fk_loader import load_vehicles_fk_sets + +logger = logging.getLogger(__name__) + +JOB_TYPE = "veh" + +# Para routes.py +VEHL_IMPORT_FILE_PREFIX = "veh_import_file:" +VEHL_IMPORT_META_PREFIX = "veh_import_meta:" +VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:" +VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:" +VEHL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import") + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + try: + total_rows = common_csv_reader.count_csv_rows(file_path) + except Exception as e: + return {"status": "failed", "error": str(e)} + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_vehicle_keys: Set[str] = set() + if actualizar: + try: + from api.v1.modules.a76.transportation.vehicles.models import Vehicle + with CoreSessionLocal() as session: + for v in ( + session.query(Vehicle.vehicle_key) + .filter( + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, + ) + .all() + ): + if v[0] and (v[0] or "").strip(): + existing_vehicle_keys.add((v[0] or "").strip()) + except Exception as e: + logger.warning("Vehicles import: could not load existing vehicle_keys for actualizar: %s", e) + + ( + valid_transport_codes, + valid_country_ame, + state_descriptions_upper, + state_country_set, + ) = load_vehicles_fk_sets(tenant_id, company_id) + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + + try: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, common_normalize.normalize_header) + # Desfase: advertencia no bloqueante (no se añade a error_lines) + _ = validate_row_vehicle_desfase(row_norm, i) + err = validate_row_vehicle( + row_norm, + i, + actualizar=actualizar, + existing_vehicle_keys=existing_vehicle_keys, + valid_transport_codes=valid_transport_codes, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.error("Vehicles import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Vehicles import: failed to store scan status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info("Vehicles import: starting scan for job %s", job_id) + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Vehicles import: failed to store scan status in Redis: %s", e) + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import") + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} + file_path = alt_path + else: + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import") + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) or {} + actualizar = meta.get("actualizar", False) + existing_vehicle_keys: Set[str] = set() + if actualizar: + try: + from api.v1.modules.a76.transportation.vehicles.models import Vehicle + with CoreSessionLocal() as session: + for v in ( + session.query(Vehicle.vehicle_key) + .filter( + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, + ) + .all() + ): + if v[0] and (v[0] or "").strip(): + existing_vehicle_keys.add((v[0] or "").strip()) + except Exception as e: + logger.warning("Vehicles import: could not load existing vehicle_keys for actualizar: %s", e) + + ( + valid_transport_codes, + valid_country_ame, + state_descriptions_upper, + state_country_set, + ) = load_vehicles_fk_sets(tenant_id, company_id) + + from api.v1.modules.a76.transportation.vehicles.services import VehicleService + from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) + + try: + with CoreSessionLocal() as session: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_vehicle( + row_norm, + i, + actualizar=actualizar, + existing_vehicle_keys=existing_vehicle_keys, + valid_transport_codes=valid_transport_codes, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + if err: + skipped_invalid += 1 + vk = (row_norm.get("CLAVE") or "").strip()[:14] or "-" + skipped_details.append({ + "line": i, + "vehicle_key": vk, + "invoice": vk, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + vk = (row_norm.get("CLAVE") or "").strip()[:14] or "" + if not vk: + skipped_invalid += 1 + continue + if vk in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append({ + "line": i, + "vehicle_key": vk, + "invoice": vk, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) + continue + seen_keys_in_file[vk] = i + + existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) + try: + if existing: + if actualizar: + data = row_to_vehicle_data_for_update(row_norm, existing) + else: + data = row_to_vehicle_data(row_norm) + if not data or not data.get("vehicle_key"): + skipped_invalid += 1 + continue + update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) + VehicleService.update(session, vk, tenant_id, update_data, company_id) + updated_count += 1 + else: + data = row_to_vehicle_data(row_norm) + if not data or not data.get("vehicle_key"): + skipped_invalid += 1 + continue + create_data = VehicleCreateDTO(**data) + VehicleService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append({ + "line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err), + }) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error("Vehicles import DB error: %s", db_err) + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.exception("Vehicles import task failed") + return {"status": "failed", "error": str(e)} + + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + try: + r = _get_redis() + r.delete(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}") + except Exception as e: + logger.warning("Vehicles import: failed to delete status key: %s", e) + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Vehicles import: failed to store commit status in Redis: %s", e) + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Vehicles import: starting commit for job %s", job_id) + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning("Vehicles import: failed to store commit status in Redis: %s", e) + return result diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/template_config.py similarity index 97% rename from backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py rename to backend/api/v1/modules/a76/layouts_csv/vehicles/template_config.py index 3474ae50..0f67ef69 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/template_config.py @@ -24,6 +24,7 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]}, {"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]}, {"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]}, + {"canonical": "COL_EXTRA", "aliases": ["COLUMNA R", "COL R"]}, ], } diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py new file mode 100644 index 00000000..b6a725f1 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_vehicle, validate_row_vehicle_desfase + +__all__ = ["validate_row_vehicle", "validate_row_vehicle_desfase"] diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py new file mode 100644 index 00000000..30a50c66 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py @@ -0,0 +1,146 @@ +""" +Validaciones comunes de fila para import CSV de vehículos. +Paridad Clarion: VALIDACIONES_TRANSPORTE, VALIDA_TODA_TRANSPORTE, VALIDA_PARCIAL_TRANSPORTE. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from ..common.common_validators import ( + MAX_LEN, + check_required, + check_max_length, + check_codigo_entidad_valores, + check_codigo_entidad_required, + check_transport_type_catalog, + check_pais_catalog_vehicles, + check_estado_catalog, + check_estado_pais_consistency, +) + + +def validate_row_vehicle_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_required(row, "CLAVE", MAX_LEN["vehicle_key"], line_num) + + +def validate_row_vehicle_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("CLAVE ACE", MAX_LEN["ace_vehicle_key"]), + ("CLAVE TRANSPORTE", MAX_LEN["transporter_key"]), + ("VIN", MAX_LEN["series"]), + ("TIPO TRANSPORTE", MAX_LEN["transport_type"]), + ("CODIGO DE ENTIDAD", MAX_LEN["entity_code"]), + ("TRANSPONDEDOR", MAX_LEN["transponder_number"]), + ("NUMERO DOT", MAX_LEN["dot_number"]), + ("PLACAS", MAX_LEN["plate_number"]), + ("CIUDAD", MAX_LEN["city"]), + ("ESTADO", MAX_LEN["state"]), + ("PAIS", MAX_LEN["country"]), + ("PRECINTO", MAX_LEN["seal"]), + ("EMPRESA ASEGURADORA", MAX_LEN["insurance_company_name"]), + ("NUM. ASEGURADORA", MAX_LEN["insurance_number"]), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_vehicle_amount(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + from ..common.common_validators import parse_decimal + monto = row.get("MONTO ASEGURADO") or row.get("MONTO") + if monto is None or not str(monto).strip(): + return None + if parse_decimal(monto) is None: + return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"} + return None + + +def validate_row_vehicle_insurance_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + from ..common.common_validators import parse_insurance_date + fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA") + if fecha is None or not str(fecha).strip(): + return None + if parse_insurance_date(fecha) is None: + return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} + return None + + +def validaciones_transporte( + row: Dict[str, Any], + line_num: int, + valid_transport_codes: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """ + VALIDACIONES_TRANSPORTE: reglas compartidas (longitudes, tipo transporte, código entidad dominio, + país, estado, estado-país). No exige CODIGO DE ENTIDAD obligatorio (eso es solo VALIDA_TODA). + """ + err = validate_row_vehicle_required(row, line_num) + if err: + return err + err = validate_row_vehicle_lengths(row, line_num) + if err: + return err + err = validate_row_vehicle_amount(row, line_num) + if err: + return err + err = validate_row_vehicle_insurance_date(row, line_num) + if err: + return err + err = check_codigo_entidad_valores(row, line_num) + if err: + return err + err = check_transport_type_catalog(row, line_num, valid_transport_codes) + if err: + return err + err = check_pais_catalog_vehicles(row, line_num, valid_country_ame) + if err: + return err + err = check_estado_catalog(row, line_num, state_descriptions_upper) + if err: + return err + err = check_estado_pais_consistency(row, line_num, state_country_set) + if err: + return err + return None + + +def valida_toda_transporte( + row: Dict[str, Any], + line_num: int, + valid_transport_codes: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_TODA_TRANSPORTE: CODIGO DE ENTIDAD obligatorio + VALIDACIONES_TRANSPORTE (registro nuevo).""" + err = check_codigo_entidad_required(row, line_num) + if err: + return err + return validaciones_transporte( + row, line_num, + valid_transport_codes=valid_transport_codes, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + + +def valida_parcial_transporte( + row: Dict[str, Any], + line_num: int, + valid_transport_codes: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """VALIDA_PARCIAL_TRANSPORTE: solo VALIDACIONES_TRANSPORTE (actualizar registro existente).""" + return validaciones_transporte( + row, line_num, + valid_transport_codes=valid_transport_codes, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py new file mode 100644 index 00000000..15eaf052 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py @@ -0,0 +1,63 @@ +""" +Punto de entrada de validación para import de una fila vehículo. +Paridad Clarion: desfase (advertencia), CLAVE vacía, VALIDA_TODA vs VALIDA_PARCIAL según actualizar y clave existente. +""" +from typing import Dict, Any, Optional, Set, Tuple + +from .common import ( + validate_row_vehicle_required, + valida_toda_transporte, + valida_parcial_transporte, +) +from ..common.common_validators import check_desfase_vehicles + + +def validate_row_vehicle( + row: Dict[str, Any], + line_num: int, + actualizar: bool = False, + existing_vehicle_keys: Optional[Set[str]] = None, + valid_transport_codes: Optional[Set[str]] = None, + valid_country_ame: Optional[Set[str]] = None, + state_descriptions_upper: Optional[Set[str]] = None, + state_country_set: Optional[Set[Tuple[str, str]]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de vehículos. + 1. CLAVE vacía → error. + 2. Si actualizar y CLAVE en existing_vehicle_keys → valida_parcial_transporte (sin exigir CODIGO DE ENTIDAD). + 3. Si no actualizar o CLAVE no existe → valida_toda_transporte (CODIGO DE ENTIDAD obligatorio). + Desfase (COL_EXTRA) no se valida aquí; el caller puede llamar validate_row_vehicle_desfase para advertencias no bloqueantes. + """ + err = validate_row_vehicle_required(row, line_num) + if err: + return err + + existing = existing_vehicle_keys or set() + clave = (row.get("CLAVE") or "").strip() + use_partial = actualizar and bool(clave and clave in existing) + + if use_partial: + err = valida_parcial_transporte( + row, line_num, + valid_transport_codes=valid_transport_codes, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + else: + err = valida_toda_transporte( + row, line_num, + valid_transport_codes=valid_transport_codes, + valid_country_ame=valid_country_ame, + state_descriptions_upper=state_descriptions_upper, + state_country_set=state_country_set, + ) + return err + + +def validate_row_vehicle_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Advertencia de desfase si COL_EXTRA (Col R) tiene valor. No bloqueante; el caller puede acumular en warnings. + """ + return check_desfase_vehicles(row, line_num) diff --git a/backend/api/v1/modules/a76/parts/imports/tasks.py b/backend/api/v1/modules/a76/parts/imports/tasks.py deleted file mode 100644 index c87bca7c..00000000 --- a/backend/api/v1/modules/a76/parts/imports/tasks.py +++ /dev/null @@ -1,639 +0,0 @@ -""" -Tareas Celery para importación CSV de Números de Parte. -Flujo: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from decimal import Decimal, InvalidOperation -from typing import Dict, Any, Optional, List, Set - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -PART_IMPORT_FILE_PREFIX = "part_import_file:" -PART_IMPORT_META_PREFIX = "part_import_meta:" -PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:" -PART_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Parts import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"part_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Parts import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{PART_IMPORT_FILE_PREFIX}{job_id}", - f"{PART_IMPORT_META_PREFIX}{job_id}", - f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Parts import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_part( - row: Dict[str, Any], - line_num: int, - valid_class_codes: Optional[Set[str]] = None, - valid_uom_codes: Optional[Set[str]] = None, - valid_currency_codes: Optional[Set[str]] = None, -) -> Optional[Dict[str, Any]]: - part_number = (row.get("NUMPARTE") or "").strip() - if not part_number: - return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} - if len(part_number) > 70: - return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"} - - commercial = (row.get("NUMPARTECOM") or "").strip() - if commercial and len(commercial) > 70: - return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"} - - desc_es = (row.get("DESCRIPCIONE") or "").strip() - if desc_es and len(desc_es) > 500: - return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} - desc_en = (row.get("DESCRIPCIONI") or "").strip() - if desc_en and len(desc_en) > 500: - return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} - - part_class = (row.get("CLASE") or "").strip() - if part_class and len(part_class) > 8: - return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} - # Si CLASE no existe en catálogo se guardará null (no se rechaza la fila) - - uom = (row.get("UNIMED") or "").strip() - if uom and len(uom) > 5: - return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} - # Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila) - - currency_key = (row.get("MONEDA") or "").strip() - if currency_key and len(currency_key) > 3: - return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"} - # Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila) - - unit_cost = row.get("COSTOUNIT") - if unit_cost is not None and unit_cost != "": - try: - Decimal(str(unit_cost)) - except (InvalidOperation, ValueError, TypeError): - return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"} - - unit_weight = row.get("PESOUNIT") - if unit_weight is not None and unit_weight != "": - try: - Decimal(str(unit_weight)) - except (InvalidOperation, ValueError, TypeError): - return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"} - - fraction = (row.get("FRACCION") or "").strip() - if fraction and len(fraction) > 10: - return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"} - us_fraction = (row.get("FRACCIONAME") or "").strip() - if us_fraction and len(us_fraction) > 16: - return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} - fda_key = (row.get("FDAKEY") or "").strip() - if fda_key and len(fda_key) > 20: - return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"} - fcc_key = (row.get("FCCKEY") or "").strip() - if fcc_key and len(fcc_key) > 30: - return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"} - license_code = (row.get("LICENCIA") or "").strip() - if license_code and len(license_code) > 3: - return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"} - eccn = (row.get("ECCN") or "").strip() - if eccn and len(eccn) > 20: - return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"} - export_code = (row.get("EXPORTCODE") or "").strip() - if export_code and len(export_code) > 2: - return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"} - exclusion = (row.get("EXCLUSION") or "").strip() - if exclusion and len(exclusion) > 19: - return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"} - weight_type = (row.get("TIPOPESO") or "").strip() - if weight_type and len(weight_type) > 6: - return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - logger.info(f"Parts import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Parts import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_class_codes: Set[str] = set() - valid_uom_codes: Set[str] = set() - valid_currency_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.classes.models import Class - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - for c in ( - session.query(Class.class_code) - .filter( - Class.tenant_id == tenant_id, - Class.company_id == company_id, - ) - .all() - ): - valid_class_codes.add(c[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - for cur in session.query(CurrencyType.code).all(): - valid_currency_codes.add(cur[0]) - except Exception as e: - logger.warning(f"Parts import: could not load FK sets: {e}") - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_part( - row_norm, - i, - valid_class_codes=valid_class_codes, - valid_uom_codes=valid_uom_codes, - valid_currency_codes=valid_currency_codes, - ) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"Parts import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=PART_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Parts import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _int_or_none(val: Any) -> Optional[int]: - if val is None or val == "": - return None - try: - return int(val) - except (ValueError, TypeError): - return None - - -def _decimal_or_none(val: Any) -> Optional[Decimal]: - if val is None or val == "": - return None - try: - return Decimal(str(val)) - except (InvalidOperation, ValueError, TypeError): - return None - - -def _bool_from_row(val: Any) -> bool: - if val is None or val == "": - return True - s = str(val).strip().upper() - if s in ("0", "F", "FALSE", "NO", "N"): - return False - return True - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - logger.info(f"Parts import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Parts import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.parts.models import Part - - valid_class_codes: Set[str] = set() - valid_uom_codes: Set[str] = set() - valid_currency_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.classes.models import Class - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - for c in ( - session.query(Class.class_code) - .filter( - Class.tenant_id == tenant_id, - Class.company_id == company_id, - ) - .all() - ): - valid_class_codes.add(c[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - for cur in session.query(CurrencyType.code).all(): - valid_currency_codes.add(cur[0]) - except Exception as e: - logger.warning(f"Parts import: could not load FK sets: {e}") - - inserted_count = 0 - skipped_invalid = 0 - skipped_missing_fk = 0 - skipped_duplicate = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - existing_by_part_number: Dict[str, Part] = {} - for p in ( - session.query(Part) - .filter( - Part.tenant_id == tenant_id, - Part.company_id == company_id, - ) - .all() - ): - existing_by_part_number[p.part_number] = p - - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_part( - row_norm, - i, - valid_class_codes=valid_class_codes, - valid_uom_codes=valid_uom_codes, - valid_currency_codes=valid_currency_codes, - ) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue - - part_number = _str_or_none(row_norm.get("NUMPARTE"), 70) - if not part_number: - skipped_invalid += 1 - continue - - commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70) - desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) - desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) - part_class = _str_or_none(row_norm.get("CLASE"), 8) - if part_class and part_class not in valid_class_codes: - part_class = None - unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) - if unit_of_measure and unit_of_measure not in valid_uom_codes: - unit_of_measure = None - currency_key = _str_or_none(row_norm.get("MONEDA"), 3) - if currency_key and currency_key not in valid_currency_codes: - currency_key = None - - unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT")) - currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None - unit_weight = _decimal_or_none(row_norm.get("PESOUNIT")) - weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6) - fraction = _str_or_none(row_norm.get("FRACCION"), 10) - us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) - fda_key = _str_or_none(row_norm.get("FDAKEY"), 20) - fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30) - license_code = _str_or_none(row_norm.get("LICENCIA"), 3) - eccn = _str_or_none(row_norm.get("ECCN"), 20) - export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2) - exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19) - is_active = _bool_from_row(row_norm.get("ACTIVO")) - - existing = existing_by_part_number.get(part_number) - if existing: - existing.commercial_part_number = commercial - existing.description_spanish = desc_es - existing.description_english = desc_en - existing.part_class = part_class - existing.unit_of_measure = unit_of_measure - existing.unit_cost = unit_cost - existing.currency_type = currency_type - existing.currency_key = currency_key - existing.unit_weight = unit_weight - existing.weight_type = weight_type - existing.fraction = fraction - existing.us_fraction = us_fraction - existing.fda_key = fda_key - existing.fcc_key = fcc_key - existing.license_code = license_code - existing.eccn = eccn - existing.export_code = export_code - existing.exclusion_symbol = exclusion_symbol - existing.is_active = is_active - session.add(existing) - inserted_count += 1 - else: - new_part = Part( - tenant_id=tenant_id, - company_id=company_id, - client_id=company_id, - part_number=part_number, - commercial_part_number=commercial, - description_spanish=desc_es, - description_english=desc_en, - part_class=part_class, - unit_of_measure=unit_of_measure, - unit_cost=unit_cost, - currency_type=currency_type, - currency_key=currency_key, - unit_weight=unit_weight, - weight_type=weight_type, - fraction=fraction, - us_fraction=us_fraction, - fda_key=fda_key, - fcc_key=fcc_key, - license_code=license_code, - eccn=eccn, - export_code=export_code, - exclusion_symbol=exclusion_symbol, - is_active=is_active, - ) - session.add(new_part) - existing_by_part_number[part_number] = new_part - inserted_count += 1 - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"Parts import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"Parts import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Parts import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/parts/imports/template_config.py b/backend/api/v1/modules/a76/parts/imports/template_config.py deleted file mode 100644 index 7cc240c4..00000000 --- a/backend/api/v1/modules/a76/parts/imports/template_config.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Configuración de plantilla CSV para Números de Parte (EstructuraCatPartesAF.xls). -""" - -from typing import Dict, List, Any - -TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { - "part_numbers": [ - {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE", "PART NUMBER", "NUM PARTE"]}, - {"canonical": "NUMPARTECOM", "aliases": ["NUMERO PARTE COMERCIAL", "COMMERCIAL PART", "PARTE COMERCIAL"]}, - {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES", "DESC ESPANOL"]}, - {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION", "DESC INGLES"]}, - {"canonical": "CLASE", "aliases": ["CLASS", "CLASE MATERIAL", "PART CLASS"]}, - {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM", "UNIT OF MEASURE"]}, - {"canonical": "COSTOUNIT", "aliases": ["COSTO UNITARIO", "UNIT COST", "COSTO"]}, - {"canonical": "MONEDA", "aliases": ["CURRENCY", "MONEDA CLAVE", "CURRENCY KEY"]}, - {"canonical": "PESOUNIT", "aliases": ["PESO UNITARIO", "UNIT WEIGHT", "PESO"]}, - {"canonical": "TIPOPESO", "aliases": ["WEIGHT TYPE", "TIPO PESO"]}, - {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, - {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, - {"canonical": "FDAKEY", "aliases": ["FDA", "FDA KEY"]}, - {"canonical": "FCCKEY", "aliases": ["FCC", "FCC KEY"]}, - {"canonical": "LICENCIA", "aliases": ["LICENSE CODE", "LICENSE"]}, - {"canonical": "ECCN", "aliases": ["ECCN CODE"]}, - {"canonical": "EXPORTCODE", "aliases": ["EXPORT CODE", "CODIGO EXPORT"]}, - {"canonical": "EXCLUSION", "aliases": ["EXCLUSION SYMBOL", "SIMBOLO EXCLUSION"]}, - {"canonical": "ACTIVO", "aliases": ["IS ACTIVE", "ACTIVE", "ACTIVO"]}, - ], -} - - -def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: - cols = TEMPLATE_COLUMNS.get("part_numbers") - if not cols: - return {} - lookup: Dict[str, str] = {} - for item in cols: - canonical = item["canonical"] - lookup[normalize_header_fn(canonical)] = canonical - for alias in item.get("aliases") or []: - lookup[normalize_header_fn(alias)] = canonical - return lookup - - -def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: - lookup = build_normalized_lookup(normalize_header_fn) - if not lookup: - return {normalize_header_fn(k): v for k, v in row.items()} - out: Dict[str, Any] = {} - for csv_header, value in row.items(): - key_norm = normalize_header_fn(csv_header) - if key_norm in lookup: - out[lookup[key_norm]] = value - return out diff --git a/backend/api/v1/modules/a76/parts/routes.py b/backend/api/v1/modules/a76/parts/routes.py index 59c25a58..7b92de23 100644 --- a/backend/api/v1/modules/a76/parts/routes.py +++ b/backend/api/v1/modules/a76/parts/routes.py @@ -8,7 +8,7 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import PartCreateDTO, PartResponseDTO, PartUpdateDTO from .service import PartService -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.parts.routes import router as imports_router router = APIRouter() diff --git a/backend/api/v1/modules/a76/pedmientos/dtos/pedimentos.py b/backend/api/v1/modules/a76/pedmientos/dtos/pedimentos.py index 4767f77d..ffde30df 100644 --- a/backend/api/v1/modules/a76/pedmientos/dtos/pedimentos.py +++ b/backend/api/v1/modules/a76/pedmientos/dtos/pedimentos.py @@ -107,7 +107,7 @@ class PedimentosCreate(PedimentosBase): customs_office: str = Field(..., max_length=3, description="Customs office") license: str = Field(..., max_length=4, description="License") pedimento_number: str = Field(..., max_length=7, description="Pedimento number") - client_id: int = Field(..., description="Client ID") + client_id: Optional[int] = Field(None, description="Client ID (opcional)") # operation_type, pedimento_type, status son opcionales - se pueden llenar después pedimento_code: str = Field(..., max_length=2, description="Pedimento key") regime: str = Field(..., max_length=3, description="Regime") diff --git a/backend/api/v1/modules/a76/pedmientos/imports/tasks.py b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py deleted file mode 100644 index 82b630d2..00000000 --- a/backend/api/v1/modules/a76/pedmientos/imports/tasks.py +++ /dev/null @@ -1,563 +0,0 @@ -""" -Tareas Celery para importación CSV de Pedimentos. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from decimal import Decimal, InvalidOperation -from datetime import datetime -from typing import Dict, Any, Optional, List, Set - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -# Redis keys (prefijo propio para no colisionar con otros imports) -PED_IMPORT_FILE_PREFIX = "ped_import_file:" -PED_IMPORT_META_PREFIX = "ped_import_meta:" -PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" -PED_IMPORT_REDIS_TTL = 3600 # 1 hour - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"ped_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{PED_IMPORT_FILE_PREFIX}{job_id}", - f"{PED_IMPORT_META_PREFIX}{job_id}", - f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_pedimento( - row: Dict[str, Any], - line_num: int, - valid_client_ids: Set[int], - valid_regimes: Set[str], - valid_pedimento_codes: Set[str], -) -> Optional[Dict[str, Any]]: - """Valida una fila para Pedimento. Retorna error dict o None.""" - # Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN - year = (row.get("AÑO") or "").strip() - if not year: - return {"line": line_num, "col": "AÑO", "msg": "Requerido"} - if len(year) > 2: - return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"} - - customs_office = (row.get("ADUANA") or "").strip() - if not customs_office: - return {"line": line_num, "col": "ADUANA", "msg": "Requerido"} - if len(customs_office) > 3: - return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"} - - license_val = (row.get("PATENTE") or "").strip() - if not license_val: - return {"line": line_num, "col": "PATENTE", "msg": "Requerido"} - if len(license_val) > 4: - return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"} - - pedimento_number = (row.get("NUMERO") or "").strip() - if not pedimento_number: - return {"line": line_num, "col": "NUMERO", "msg": "Requerido"} - if len(pedimento_number) > 7: - return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"} - - client_id_str = (row.get("CLIENTE_ID") or "").strip() - if not client_id_str: - return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"} - try: - client_id = int(client_id_str) - except ValueError: - return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"} - if client_id not in valid_client_ids: - return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"} - - pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip() - if not pedimento_code: - return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"} - if len(pedimento_code) > 2: - return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"} - if pedimento_code not in valid_pedimento_codes: - return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"} - - regime = (row.get("REGIMEN") or "").strip() - if not regime: - return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"} - if len(regime) > 3: - return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"} - if regime not in valid_regimes: - return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"} - - # Optional numeric/string fields - validate format if present - status = (row.get("ESTATUS") or "").strip() - if status and len(status) > 30: - return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"} - - for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]: - val = (row.get(col) or "").strip() - if not val: - continue - try: - Decimal(val.replace(",", ".")) - except (InvalidOperation, ValueError): - return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} - - operation_type = (row.get("TIPO_OPERACION") or "").strip().lower() - if operation_type and operation_type not in ("imp", "exp", ""): - return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"} - - pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower() - if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""): - return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"} - - return None - - -def _parse_decimal(val: Any) -> Optional[Decimal]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - try: - return Decimal(str(val).strip().replace(",", ".")) - except (InvalidOperation, ValueError): - return None - - -def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]: - """Build dict for PedimentosCreate from normalized CSV row (canonical names).""" - year = (row.get("AÑO") or "").strip()[:2] - customs_office = (row.get("ADUANA") or "").strip()[:3] - license_val = (row.get("PATENTE") or "").strip()[:4] - pedimento_number = (row.get("NUMERO") or "").strip()[:7] - client_id_str = (row.get("CLIENTE_ID") or "").strip() - client_id = int(client_id_str) if client_id_str else None - pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2] - regime = (row.get("REGIMEN") or "").strip()[:3] - - data = { - "year": year, - "customs_office": customs_office, - "license": license_val, - "pedimento_number": pedimento_number, - "client_id": client_id, - "pedimento_code": pedimento_code, - "regime": regime, - } - - op = (row.get("TIPO_OPERACION") or "").strip().lower() - if op in ("imp", "exp"): - data["operation_type"] = op - - ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower() - if ptype in ("normal", "consolidated", "complementary", "automobile"): - data["pedimento_type"] = ptype - - status = (row.get("ESTATUS") or "").strip() - if status: - data["status"] = status[:30] - - data["usd_value"] = _parse_decimal(row.get("VALOR_USD")) - data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO")) - data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO")) - data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO")) - - obs = (row.get("OBSERVACIONES") or "").strip() - if obs: - data["observations"] = obs - - return data - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"Pedimentos import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Pedimentos import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - # Load valid FK sets for validation - valid_client_ids: Set[int] = set() - valid_regimes: Set[str] = set() - valid_pedimento_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.clients_and_providers.models import ClientProvider - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode - - for cp in session.query(ClientProvider).filter( - ClientProvider.tenant_id == tenant_id, - ClientProvider.company_id == company_id, - ).all(): - valid_client_ids.add(cp.id) - for r in session.query(RegimenPedimento).all(): - valid_regimes.add(r.code) - for pc in session.query(PedimentoCode).all(): - valid_pedimento_codes.add(pc.code) - except Exception as e: - logger.error(f"Pedimentos import: failed to load FK sets: {e}") - return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header, "pedimentos") - err = _validate_row_pedimento( - row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes - ) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"Pedimentos import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=PED_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create. - """ - logger.info(f"Pedimentos import: starting commit for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - # Reload FK sets for commit-time validation - valid_client_ids: Set[int] = set() - valid_regimes: Set[str] = set() - valid_pedimento_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.clients_and_providers.models import ClientProvider - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode - - for cp in session.query(ClientProvider).filter( - ClientProvider.tenant_id == tenant_id, - ClientProvider.company_id == company_id, - ).all(): - valid_client_ids.add(cp.id) - for r in session.query(RegimenPedimento).all(): - valid_regimes.add(r.code) - for pc in session.query(PedimentoCode).all(): - valid_pedimento_codes.add(pc.code) - except Exception as e: - logger.error(f"Pedimentos import: failed to load FK sets: {e}") - return {"status": "failed", "error": str(e)} - - from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate - from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate - from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService - - inserted_count = 0 - skipped_invalid = 0 - skipped_missing_fk = 0 - skipped_duplicate = 0 - skipped_details: List[Dict[str, Any]] = [] - response = None - - try: - with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header, "pedimentos") - err = _validate_row_pedimento( - row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes - ) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue - - try: - data = _row_to_pedimentos_create(row_norm) - # Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL - if "pedimento_dates" not in data or data.get("pedimento_dates") is None: - data["pedimento_dates"] = PedimentoDatesCreate( - entry_date=datetime.now(), - end_date=datetime.now(), - ) - create_data = PedimentosCreate(**data) - PedimentosService.create(session, create_data, tenant_id, company_id) - inserted_count += 1 - except ValueError as ve: - if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): - skipped_duplicate += 1 - skipped_details.append({"line": i, "reason": str(ve)}) - else: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": str(ve)}) - except Exception as e: - logger.warning(f"Pedimentos import line {i}: {e}") - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": str(e)}) - - total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - - except Exception as e: - logger.error(f"Pedimentos import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}") - - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - return response diff --git a/backend/api/v1/modules/a76/pedmientos/imports/template_config.py b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py deleted file mode 100644 index 06fea167..00000000 --- a/backend/api/v1/modules/a76/pedmientos/imports/template_config.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls). -Solo se leen columnas definidas aquí; el resto se ignora. -""" - -from typing import Dict, List, Any, Optional - -TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { - "pedimentos": [ - {"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]}, - {"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]}, - {"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]}, - {"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]}, - {"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]}, - {"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]}, - {"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]}, - {"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]}, - {"canonical": "REGIMEN", "aliases": ["REGIME"]}, - {"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]}, - {"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]}, - {"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]}, - {"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]}, - {"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]}, - {"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]}, - ], -} - - -def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]: - """normalized_header -> canonical_name para plantilla pedimentos.""" - cols = TEMPLATE_COLUMNS.get(template_id) - if not cols: - return {} - lookup: Dict[str, str] = {} - for item in cols: - canonical = item["canonical"] - lookup[normalize_header_fn(canonical)] = canonical - for alias in item.get("aliases") or []: - lookup[normalize_header_fn(alias)] = canonical - return lookup - - -def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]: - """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" - lookup = build_normalized_lookup(normalize_header_fn, template_id) - if not lookup: - return {normalize_header_fn(k): v for k, v in row.items()} - out: Dict[str, Any] = {} - for csv_header, value in row.items(): - key_norm = normalize_header_fn(csv_header) - if key_norm in lookup: - out[lookup[key_norm]] = value - return out diff --git a/backend/api/v1/modules/a76/pedmientos/models/pedimentos.py b/backend/api/v1/modules/a76/pedmientos/models/pedimentos.py index f97600dc..fe104ac7 100644 --- a/backend/api/v1/modules/a76/pedmientos/models/pedimentos.py +++ b/backend/api/v1/modules/a76/pedmientos/models/pedimentos.py @@ -128,7 +128,9 @@ class Pedimentos(Base, TenantScopedMixin, TimestampMixin): customs_office: Mapped[str] = mapped_column(String(3)) license: Mapped[str] = mapped_column(String(4)) pedimento_number: Mapped[str] = mapped_column(String(7)) - client_id: Mapped[int] = mapped_column(Integer) + client_id: Mapped[Optional[int]] = mapped_column( + Integer, nullable=True + ) # Opcional: no requerido para el pedimento operation_type: Mapped[OperationType] = mapped_column(String(3)) pedimento_type: Mapped[PedimentoType] = mapped_column(String(20)) pedimento_code: Mapped[str] = mapped_column(String(2)) diff --git a/backend/api/v1/modules/a76/pedmientos/router.py b/backend/api/v1/modules/a76/pedmientos/router.py index 2b987b46..b9444c04 100644 --- a/backend/api/v1/modules/a76/pedmientos/router.py +++ b/backend/api/v1/modules/a76/pedmientos/router.py @@ -31,7 +31,7 @@ from .routes.pedimento_rectification_origin import ( from .routes.pedimento_transport_means import router as pedimento_transport_means_router from .routes.pedimento_validation import router as pedimento_validation_router from .routes.pedimentos import router as pedimentos_router -from .imports.routes import router as pedimentos_imports_router +from api.v1.modules.a76.layouts_csv.pedmientos.routes import router as pedimentos_imports_router router = APIRouter() diff --git a/backend/api/v1/modules/a76/router.py b/backend/api/v1/modules/a76/router.py index ae3aa47e..796bf862 100644 --- a/backend/api/v1/modules/a76/router.py +++ b/backend/api/v1/modules/a76/router.py @@ -14,7 +14,9 @@ from .items.routes import router as items_router from .classes import router as classes_router from .clients_and_providers import router as client_and_provider_router -from .imports.routes import router as imports_router +from .layouts_csv.facturas.routes import router as imports_router +from .layouts_csv.exportacion.routes import router as exportacion_imports_router +from .layouts_csv.cambio_regimen_regularizacion.routes import router as cambio_regimen_regularizacion_imports_router from .csv_templates.routes import router as csv_templates_router from .invoice_settings.routes import router as invoice_settings_router from .item_presets.routes import router as item_presets_router @@ -57,6 +59,8 @@ router.include_router(general_catalogs_router, prefix="/a76", tags=["a76 / gener router.include_router(invoices_router, prefix="/a76", tags=["a76 / invoices"]) router.include_router(items_router, prefix="/a76", tags=["a76 / items"]) router.include_router(imports_router, prefix="/a76/imports", tags=["a76 / imports"]) +router.include_router(exportacion_imports_router, prefix="/a76/imports/exportacion", tags=["a76 / imports / exportacion"]) +router.include_router(cambio_regimen_regularizacion_imports_router, prefix="/a76/imports/cambio-regimen-regularizacion", tags=["a76 / imports / cambio_regimen_regularizacion"]) router.include_router(csv_templates_router, prefix="/a76/csv-templates", tags=["a76 / csv_templates"]) router.include_router(invoice_settings_router) router.include_router(item_presets_router, prefix="/a76/item-presets", tags=["a76 / item_presets"]) diff --git a/backend/api/v1/modules/a76/transportation/drivers/dto.py b/backend/api/v1/modules/a76/transportation/drivers/dto.py index bfb3d8fc..02420ea8 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/dto.py +++ b/backend/api/v1/modules/a76/transportation/drivers/dto.py @@ -36,6 +36,33 @@ class DriverCreateDTO(DriverBaseDTO): pass +class DriverUpdateDTO(BaseModel): + """All fields optional; transporter_key, line, company_id, tenant_id are not updated.""" + + driver_name: Optional[str] = None + license_number: Optional[str] = None + express_line_id: Optional[str] = None + ace_id: Optional[str] = None + birth_date: Optional[int] = None + gender: Optional[str] = None + birth_country: Optional[str] = None + hazardous_material_auth: Optional[str] = None + hazardous_material_state: Optional[str] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + id_key1: Optional[str] = None + id_number1: Optional[str] = None + id_state1: Optional[str] = None + id_country1: Optional[str] = None + id_key2: Optional[str] = None + id_number2: Optional[str] = None + id_state2: Optional[str] = None + id_country2: Optional[str] = None + badge_number: Optional[str] = None + class_type: Optional[str] = None + unique_badge_number: Optional[str] = None + + class DriverResponseDTO(DriverBaseDTO): class Config: from_attributes = True diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py b/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py deleted file mode 100644 index 9234776e..00000000 --- a/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py +++ /dev/null @@ -1,640 +0,0 @@ -""" -Tareas Celery para importacion CSV de Conductores. -Flujo: scan_file (validacion) -> insert_valid_rows (commit). -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -DRV_IMPORT_FILE_PREFIX = "drv_import_file:" -DRV_IMPORT_META_PREFIX = "drv_import_meta:" -DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:" -DRV_IMPORT_STATUS_PREFIX = "drv_import_status:" -DRV_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Drivers import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"drv_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Drivers import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{DRV_IMPORT_FILE_PREFIX}{job_id}", - f"{DRV_IMPORT_META_PREFIX}{job_id}", - f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", - f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Drivers import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -_MAX = { - "transporter_key": 5, - "driver_name": 80, - "license_number": 29, - "express_line_id": 17, - "ace_id": 20, - "gender": 1, - "birth_country": 3, - "hazardous_material_auth": 2, - "hazardous_material_state": 30, - "first_name": 20, - "last_name": 20, - "id_key1": 40, - "id_number1": 20, - "id_state1": 30, - "id_country1": 3, - "id_key2": 40, - "id_number2": 20, - "id_state2": 30, - "id_country2": 3, - "badge_number": 20, - "class_type": 1, - "unique_badge_number": 100, -} - - -def _parse_int(val: Any) -> Optional[int]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - s = str(val).strip() - if re.match(r"^\d+$", s): - return int(s) - if re.match(r"^\d+\.0+$", s): - try: - return int(float(s)) - except ValueError: - return None - return None - - -def _parse_birth_date(val: Any) -> Optional[int]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - s = str(val).strip() - if re.match(r"^\d{8}$", s): - try: - return int(s) - except ValueError: - return None - if re.match(r"^\d+\.0+$", s): - try: - return int(float(s)) - except ValueError: - return None - for sep in ["/", "-", "."]: - if sep in s: - parts = s.split(sep) - if len(parts) == 3: - try: - a, b, c = [p.strip() for p in parts] - if len(c) == 4 and len(a) <= 2 and len(b) <= 2: - return int(c) * 10000 + int(b) * 100 + int(a) - if len(a) == 4 and len(b) <= 2 and len(c) <= 2: - return int(a) * 10000 + int(b) * 100 + int(c) - except (ValueError, TypeError): - return None - break - return None - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _dedupe_headers(headers: List[str]) -> List[str]: - counts: Dict[str, int] = {} - unique: List[str] = [] - for header in headers: - name = str(header or "").strip() or "COL" - count = counts.get(name, 0) + 1 - counts[name] = count - if count == 1: - unique.append(name) - else: - unique.append(f"{name} {count}") - return unique - - -def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - transporter_key = (row.get("TRANSPORTISTA") or "").strip() - if not transporter_key: - return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"} - if len(transporter_key) > _MAX["transporter_key"]: - return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"} - - line_val = _parse_int(row.get("LINEA")) - if line_val is None: - return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"} - if line_val <= 0: - return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"} - - driver_name = (row.get("CLAVE CONDUCTOR") or "").strip() - if driver_name and len(driver_name) > _MAX["driver_name"]: - return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"} - - for col, max_len in [ - ("LICENCIA", _MAX["license_number"]), - ("PERMISO LINEA EXPRESS", _MAX["express_line_id"]), - ("IDENTIFICACION ACE", _MAX["ace_id"]), - ("SEXO", _MAX["gender"]), - ("PAIS NACIMIENTO", _MAX["birth_country"]), - ("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]), - ("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]), - ("NOMBRE(S)", _MAX["first_name"]), - ("APELLIDO PATERNO", _MAX["last_name"]), - ("FORMA IDENTIFICACION 1", _MAX["id_key1"]), - ("NUM. IDENTIFICACION 1", _MAX["id_number1"]), - ("ESTADO", _MAX["id_state1"]), - ("PAIS", _MAX["id_country1"]), - ("FORMA IDENTIFICACION 2", _MAX["id_key2"]), - ("NUM. IDENTIFICACION 2", _MAX["id_number2"]), - ("ESTADO 2", _MAX["id_state2"]), - ("PAIS 2", _MAX["id_country2"]), - ]: - val = (row.get(col) or "").strip() - if val and len(val) > max_len: - return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"} - - fecha = row.get("FECHA NACIMIENTO") - if fecha is not None and str(fecha).strip(): - if _parse_birth_date(fecha) is None: - return { - "line": line_num, - "col": "FECHA NACIMIENTO", - "msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)", - } - - return None - - -def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: - transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"]) - line = _parse_int(row.get("LINEA")) - if not transporter_key or line is None: - return {} - data = { - "transporter_key": transporter_key, - "line": line, - "driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]), - "license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]), - "express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]), - "ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]), - "birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")), - "gender": _str_or_none(row.get("SEXO"), _MAX["gender"]), - "birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]), - "hazardous_material_auth": _str_or_none( - row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"] - ), - "hazardous_material_state": _str_or_none( - row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"] - ), - "first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]), - "last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]), - "id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]), - "id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]), - "id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]), - "id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]), - "id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]), - "id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]), - "id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]), - "id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]), - "company_id": company_id, - "tenant_id": tenant_id, - } - return data - - -def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Drivers import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.reader(f_in, dialect=dialect) - try: - headers = next(reader) - except StopIteration: - headers = [] - headers = _dedupe_headers(headers) - dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) - - for i, row in enumerate(dict_reader, start=1): - if progress_callback and i % 500 == 0: - progress_callback(i, total_rows, error_count) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_driver(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"Drivers import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Drivers import: failed to store error lines: {e}") - - result = { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - return result - - -def run_scan_sync(job_id: str) -> Dict[str, Any]: - result = _do_scan(job_id, progress_callback=None) - try: - r = _get_redis() - r.set( - f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") - return result - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - logger.info(f"Drivers import: starting scan for job {job_id}") - - def on_progress(current: int, total: int, errors: int) -> None: - self.update_state( - state="PROGRESS", - meta={"current": current, "total": total, "errors": errors}, - ) - - result = _do_scan(job_id, progress_callback=on_progress) - try: - r = _get_redis() - r.set( - f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") - return result - - -def _do_commit(job_id: str) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Drivers import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.transportation.drivers.services import DriverService - from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO - - inserted_count = 0 - updated_count = 0 - skipped_invalid = 0 - skipped_duplicate = 0 - skipped_details: List[Dict[str, Any]] = [] - seen_keys_in_file: Dict[str, int] = {} - - try: - with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.reader(f, dialect=dialect) - try: - headers = next(reader) - except StopIteration: - headers = [] - headers = _dedupe_headers(headers) - dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) - - for i, row in enumerate(dict_reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_driver(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", - "invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - data = _row_to_driver_dto(row_norm, tenant_id, company_id) - if not data or not data.get("transporter_key") or data.get("line") is None: - skipped_invalid += 1 - continue - - key = f"{data['transporter_key']}:{data['line']}" - if key in seen_keys_in_file: - skipped_duplicate += 1 - skipped_details.append( - { - "line": i, - "driver_key": key, - "invoice": key, - "reason": "Clave duplicada en el archivo (se usa la primera)", - } - ) - continue - seen_keys_in_file[key] = i - - existing = DriverService.get_driver_by_key_and_line( - session, data["transporter_key"], data["line"], str(company_id), tenant_id - ) - try: - if existing: - update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")} - for field, value in update_fields.items(): - setattr(existing, field, value) - session.add(existing) - updated_count += 1 - else: - create_data = DriverCreateDTO(**data) - DriverService.create_driver(session, create_data) - inserted_count += 1 - except Exception as db_err: - session.rollback() - skipped_invalid += 1 - skipped_details.append( - {"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)} - ) - continue - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"Drivers import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - except Exception as e: - logger.error(f"Drivers import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Drivers import cleanup failed: {cleanup_err}") - - total_ok = inserted_count + updated_count - if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: - return { - "status": "warning", - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", - } - if total_ok == 0: - return { - "status": "failed", - "error": "No hay registros validos en el archivo CSV", - "inserted": 0, - "updated": 0, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return { - "status": "finished", - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - -def run_commit_sync(job_id: str) -> Dict[str, Any]: - result = _do_commit(job_id) - try: - r = _get_redis() - r.set( - f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") - return result - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - logger.info(f"Drivers import: starting commit for job {job_id}") - result = _do_commit(job_id) - try: - r = _get_redis() - r.set( - f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") - return result diff --git a/backend/api/v1/modules/a76/transportation/drivers/routes.py b/backend/api/v1/modules/a76/transportation/drivers/routes.py index a07b5c55..0a7efa2f 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/routes.py +++ b/backend/api/v1/modules/a76/transportation/drivers/routes.py @@ -5,10 +5,16 @@ from core.security import get_current_user, validate_access_to_resource from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy.orm import Session -from .dto import DriverCreateDTO, DriverResponseDTO +from .dto import DriverCreateDTO, DriverResponseDTO, DriverUpdateDTO from .models import Driver from .services import DriverService -from .imports.routes import router as imports_router +from api.v1.modules.a76.transportation.transporters.services import TransporterService +from api.v1.modules.a76.transportation.transporters.models import Transporter +from api.v1.modules.a76.layouts_csv.drivers.routes import router as imports_router +from sqlalchemy import func + +import logging +logger = logging.getLogger(__name__) router = APIRouter(prefix="/drivers") @@ -63,9 +69,67 @@ async def create_driver( db: Session = Depends(get_core_db), current_user: dict = Depends(get_current_user), ): + # Validar acceso a la empresa del cuerpo + tenant_id = validate_access_to_resource( + db, driver_data.company_id, current_user + ) + tk = (driver_data.transporter_key or "").strip() + # Buscar transportista: primero exacto, luego ignorando mayúsculas + transporter = TransporterService.get_by_id( + db, tk, tenant_id, driver_data.company_id + ) + if not transporter: + transporter = TransporterService.get_by_id_ignore_case( + db, tk, tenant_id, driver_data.company_id + ) + if not transporter: + # Diagnóstico: ¿existe ese transportista con otra empresa/tenant? + any_with_key = ( + db.query(Transporter) + .filter(func.upper(Transporter.transporter_key) == tk.upper()) + .limit(1) + .first() + ) + logger.warning( + "Driver create: transporter not found. key=%r tenant_id=%s company_id=%s; " + "any_transporter_with_key=%s (other_tenant=%s other_company=%s)", + tk, tenant_id, driver_data.company_id, + getattr(any_with_key, "transporter_key", None) if any_with_key else None, + getattr(any_with_key, "tenant_id", None) if any_with_key else None, + getattr(any_with_key, "company_id", None) if any_with_key else None, + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"El transportista {tk} no existe en el catálogo de esta empresa. Crea primero el transportista o elige uno existente.", + ) + # Usar la clave tal como está en BD (mismo caso) + driver_data.transporter_key = transporter.transporter_key return DriverService.create_driver(db, driver_data) +@router.put("/{transporter_key}/{line}", response_model=DriverResponseDTO) +async def update_driver( + transporter_key: str, + line: int, + driver_data: DriverUpdateDTO, + company_id: int = Query(..., description="Company ID for filtering"), + db: Session = Depends(get_core_db), + current_user: dict = Depends(get_current_user), +): + tenant_id = validate_access_to_resource(db, company_id, current_user) + driver = DriverService.update_driver( + db, + transporter_key, + line, + str(company_id), + tenant_id, + driver_data, + ) + if not driver: + raise HTTPException(status_code=404, detail="Driver not found") + return driver + + @router.delete("/{transporter_key}/{line}", status_code=status.HTTP_204_NO_CONTENT) async def delete_driver( transporter_key: str, diff --git a/backend/api/v1/modules/a76/transportation/drivers/services.py b/backend/api/v1/modules/a76/transportation/drivers/services.py index c22a49c0..d9d9d261 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/services.py +++ b/backend/api/v1/modules/a76/transportation/drivers/services.py @@ -40,6 +40,27 @@ class DriverService: db.refresh(new_driver) return new_driver + @staticmethod + def update_driver( + db: Session, + transporter_key: str, + line: int, + company_id: str, + tenant_id: Optional[str], + data: dto.DriverUpdateDTO, + ) -> Optional[models.Driver]: + driver = DriverService.get_driver_by_key_and_line( + db, transporter_key, line, company_id, tenant_id + ) + if not driver: + return None + update_data = data.model_dump(exclude_unset=True) + for key, value in update_data.items(): + setattr(driver, key, value) + db.commit() + db.refresh(driver) + return driver + @staticmethod def delete_driver( db: Session, diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py b/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py deleted file mode 100644 index c43c93cf..00000000 --- a/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py +++ /dev/null @@ -1,535 +0,0 @@ -""" -Tareas Celery para importación CSV de Trailers y Cajas. -Flujo: scan_file (validación) → insert_valid_rows (commit). -Upsert por trailer_number usando TrailerService. -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -TRL_IMPORT_FILE_PREFIX = "trl_import_file:" -TRL_IMPORT_META_PREFIX = "trl_import_meta:" -TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:" -TRL_IMPORT_STATUS_PREFIX = "trl_import_status:" -TRL_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Trailers import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"trl_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Trailers import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{TRL_IMPORT_FILE_PREFIX}{job_id}", - f"{TRL_IMPORT_META_PREFIX}{job_id}", - f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Trailers import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -# Max lengths from Trailer model (a76.trailer) -_MAX = { - "trailer_number": 20, - "ace_trailer_number": 10, - "trailer_type_key": 2, - "seal": 15, - "entity_code": 1, - "plate_number": 17, - "state": 30, - "country": 3, - "container_key": 3, -} - - -def _dedupe_headers(headers: List[str]) -> List[str]: - counts: Dict[str, int] = {} - unique: List[str] = [] - for header in headers: - name = str(header or "").strip() or "COL" - count = counts.get(name, 0) + 1 - counts[name] = count - if count == 1: - unique.append(name) - else: - unique.append(f"{name} {count}") - return unique - - -def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Trailer. Retorna error dict o None.""" - trailer_number = (row.get("NUMERO TRAILER") or "").strip() - if not trailer_number: - return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"} - if len(trailer_number) > _MAX["trailer_number"]: - return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"} - - for col, max_len in [ - ("CLAVE ACE", _MAX["ace_trailer_number"]), - ("TIPO TRAILER", _MAX["trailer_type_key"]), - ("PRECINTO", _MAX["seal"]), - ("CODIGO ENTIDAD", _MAX["entity_code"]), - ("PLACAS", _MAX["plate_number"]), - ("ESTADO", _MAX["state"]), - ("PAIS", _MAX["country"]), - ("CLAVE CONTENEDOR", _MAX["container_key"]), - ]: - val = (row.get(col) or "").strip() - if val and len(val) > max_len: - return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} - - return None - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: - """Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row.""" - trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"]) - if not trailer_number: - return {} - return { - "trailer_number": trailer_number, - "ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]), - "trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]), - "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), - "entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]), - "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), - "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), - "country": _str_or_none(row.get("PAIS"), _MAX["country"]), - "container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]), - } - - -def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Trailers import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.reader(f_in, dialect=dialect) - try: - headers = next(reader) - except StopIteration: - headers = [] - headers = _dedupe_headers(headers) - dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) - - for i, row in enumerate(dict_reader, start=1): - if progress_callback and i % 500 == 0: - progress_callback(i, total_rows, error_count) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_trailer(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"Trailers import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Trailers import: failed to store error lines: {e}") - - result = { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - return result - - -def run_scan_sync(job_id: str) -> Dict[str, Any]: - result = _do_scan(job_id, progress_callback=None) - try: - r = _get_redis() - r.set( - f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") - return result - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - logger.info(f"Trailers import: starting scan for job {job_id}") - - def on_progress(current: int, total: int, errors: int) -> None: - self.update_state( - state="PROGRESS", - meta={"current": current, "total": total, "errors": errors}, - ) - - result = _do_scan(job_id, progress_callback=on_progress) - try: - r = _get_redis() - r.set( - f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") - return result - - -def _do_commit(job_id: str) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Trailers import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.transportation.trailers.services import TrailerService - from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO - - inserted_count = 0 - updated_count = 0 - skipped_invalid = 0 - skipped_duplicate = 0 - skipped_details: List[Dict[str, Any]] = [] - seen_keys_in_file: Dict[str, int] = {} - - try: - with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.reader(f, dialect=dialect) - try: - headers = next(reader) - except StopIteration: - headers = [] - headers = _dedupe_headers(headers) - dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) - - for i, row in enumerate(dict_reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_trailer(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", - "invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - data = _row_to_trailer_dto(row_norm, tenant_id, company_id) - if not data or not data.get("trailer_number"): - skipped_invalid += 1 - continue - - tn = data["trailer_number"] - if tn in seen_keys_in_file: - skipped_duplicate += 1 - skipped_details.append( - { - "line": i, - "trailer_number": tn, - "invoice": tn, - "reason": "Clave duplicada en el archivo (se usa la primera)", - } - ) - continue - seen_keys_in_file[tn] = i - - existing = TrailerService.get_by_id(session, tn, tenant_id, company_id) - try: - if existing: - update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"}) - TrailerService.update(session, tn, tenant_id, update_data, company_id) - updated_count += 1 - else: - create_data = TrailerCreateDTO(**data) - TrailerService.create(session, create_data, tenant_id, company_id) - inserted_count += 1 - except Exception as db_err: - session.rollback() - skipped_invalid += 1 - skipped_details.append( - {"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)} - ) - continue - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"Trailers import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - except Exception as e: - logger.error(f"Trailers import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Trailers import cleanup failed: {cleanup_err}") - - total_ok = inserted_count + updated_count - if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: - return { - "status": "warning", - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", - } - if total_ok == 0: - return { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "updated": 0, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return { - "status": "finished", - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - -def run_commit_sync(job_id: str) -> Dict[str, Any]: - result = _do_commit(job_id) - try: - r = _get_redis() - r.set( - f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") - return result - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - logger.info(f"Trailers import: starting commit for job {job_id}") - result = _do_commit(job_id) - try: - r = _get_redis() - r.set( - f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") - return result diff --git a/backend/api/v1/modules/a76/transportation/trailers/routes.py b/backend/api/v1/modules/a76/transportation/trailers/routes.py index 4dcea2e3..81c7fbb7 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/routes.py +++ b/backend/api/v1/modules/a76/transportation/trailers/routes.py @@ -4,7 +4,7 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO from .services import TrailerService -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.trailers.routes import router as imports_router # Main router: trailers CRUD + CSV imports router = APIRouter() diff --git a/backend/api/v1/modules/a76/transportation/transporters/routes.py b/backend/api/v1/modules/a76/transportation/transporters/routes.py index 11796e47..84eb5f74 100644 --- a/backend/api/v1/modules/a76/transportation/transporters/routes.py +++ b/backend/api/v1/modules/a76/transportation/transporters/routes.py @@ -1,11 +1,19 @@ +from fastapi import APIRouter + from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import TransporterCreateDTO, TransporterResponseDTO, TransporterUpdateDTO from .services import TransporterService +from api.v1.modules.a76.layouts_csv.transportistas.routes import router as imports_router -# Create router using TenantCRUDRoutes factory -# Note: transporter_key is a string (not int) and is used as the primary key -router = TenantCRUDRoutes( +# Main router: transporters CRUD + CSV imports +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/transporters/imports", tags=["a76 / transporters / csv_import"]) + +# CRUD routes +crud_router = TenantCRUDRoutes( service=TransporterService, create_schema=TransporterCreateDTO, update_schema=TransporterUpdateDTO, @@ -20,3 +28,4 @@ router = TenantCRUDRoutes( default_page_size=50, max_page_size=100, ).router +router.include_router(crud_router) diff --git a/backend/api/v1/modules/a76/transportation/transporters/services.py b/backend/api/v1/modules/a76/transportation/transporters/services.py index bbf76e69..d92880a9 100644 --- a/backend/api/v1/modules/a76/transportation/transporters/services.py +++ b/backend/api/v1/modules/a76/transportation/transporters/services.py @@ -1,9 +1,13 @@ from typing import Optional, Tuple, List, Dict, Any +import logging from sqlalchemy.orm import Session +from sqlalchemy import func from . import dto, models +logger = logging.getLogger(__name__) + class TransporterService: """Service for Transporter CRUD operations with tenant support""" @@ -47,11 +51,32 @@ class TransporterService: def get_by_id( db: Session, transporter_key: str, tenant_id: int, company_id: int ) -> Optional[models.Transporter]: - """Get transporter by transporter_key""" + """Get transporter by transporter_key (exact match after strip).""" + key = (transporter_key or "").strip() + if not key: + return None return ( db.query(models.Transporter) .filter( - models.Transporter.transporter_key == transporter_key, + models.Transporter.transporter_key == key, + models.Transporter.tenant_id == tenant_id, + models.Transporter.company_id == company_id, + ) + .first() + ) + + @staticmethod + def get_by_id_ignore_case( + db: Session, transporter_key: str, tenant_id: int, company_id: int + ) -> Optional[models.Transporter]: + """Get transporter by transporter_key (case-insensitive) for given tenant/company.""" + key = (transporter_key or "").strip() + if not key: + return None + return ( + db.query(models.Transporter) + .filter( + func.upper(models.Transporter.transporter_key) == key.upper(), models.Transporter.tenant_id == tenant_id, models.Transporter.company_id == company_id, ) diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py deleted file mode 100644 index 7e9df06c..00000000 --- a/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py +++ /dev/null @@ -1,607 +0,0 @@ -""" -Tareas Celery para importación CSV de Vehículos (Transportes). -Flujo: scan_file (validación) → insert_valid_rows (commit). -Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key. -""" -import os -import base64 -import csv -import json -import logging -import re -import unicodedata -from decimal import Decimal -from typing import Dict, Any, Optional, List - -from core.celery_app import celery_app -from core.database import CoreSessionLocal - -from .template_config import row_from_template - -logger = logging.getLogger(__name__) - -VEHL_IMPORT_FILE_PREFIX = "veh_import_file:" -VEHL_IMPORT_META_PREFIX = "veh_import_meta:" -VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:" -VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:" -VEHL_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return os.path.join(os.getcwd(), "uploads", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Vehicles import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"veh_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", - f"{VEHL_IMPORT_META_PREFIX}{job_id}", - f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -# Max lengths from Vehicle model (a76.vehicle) -_MAX = { - "vehicle_key": 14, - "ace_vehicle_key": 10, - "transporter_key": 23, - "transport_identifier": 30, - "transport_type": 2, - "entity_code": 1, - "transponder_number": 16, - "dot_number": 8, - "plate_number": 17, - "city": 30, - "state": 30, - "country": 3, - "seal": 49, - "insurance_company_name": 30, - "insurance_number": 20, - "series": 30, -} - - -def _parse_decimal(val: Any) -> Optional[Decimal]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - try: - s = str(val).strip().replace(",", "") - return Decimal(s) - except Exception: - return None - - -def _parse_insurance_date(val: Any) -> Optional[int]: - """Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD.""" - if val is None or (isinstance(val, str) and not val.strip()): - return None - s = str(val).strip() - if not s: - return None - # Already integer-like - if re.match(r"^\d{8}$", s): - try: - return int(s) - except ValueError: - pass - # Try DD/MM/YYYY or similar - for sep in ["/", "-", "."]: - if sep in s: - parts = s.split(sep) - if len(parts) == 3: - try: - a, b, c = [p.strip() for p in parts] - if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year - return int(c) * 10000 + int(b) * 100 + int(a) - if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year - return int(a) * 10000 + int(b) * 100 + int(c) - except (ValueError, TypeError): - pass - break - return None - - -def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Vehículo. Retorna error dict o None.""" - clave = (row.get("CLAVE") or "").strip() - if not clave: - return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} - if len(clave) > _MAX["vehicle_key"]: - return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"} - - # Optional fields: max lengths only - for col, max_len in [ - ("CLAVE ACE", _MAX["ace_vehicle_key"]), - ("CLAVE TRANSPORTE", _MAX["transporter_key"]), - ("VIN", _MAX["series"]), - ("TIPO TRANSPORTE", _MAX["transport_type"]), - ("CODIGO DE ENTIDAD", _MAX["entity_code"]), - ("TRANSPONDEDOR", _MAX["transponder_number"]), - ("NUMERO DOT", _MAX["dot_number"]), - ("PLACAS", _MAX["plate_number"]), - ("CIUDAD", _MAX["city"]), - ("ESTADO", _MAX["state"]), - ("PAIS", _MAX["country"]), - ("PRECINTO", _MAX["seal"]), - ("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]), - ("NUM. ASEGURADORA", _MAX["insurance_number"]), - ]: - val = (row.get(col) or "").strip() - if val and len(val) > max_len: - return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} - - # MONTO ASEGURADO: must be numeric if present - monto = row.get("MONTO ASEGURADO") or row.get("MONTO") - if monto is not None and str(monto).strip(): - if _parse_decimal(monto) is None: - return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"} - - # FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None) - # Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either - # reject or set None. We reject invalid date to keep data quality. - fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA") - if fecha is not None and str(fecha).strip(): - if _parse_insurance_date(fecha) is None: - return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} - - return None - - -def _do_scan( - job_id: str, - progress_callback: Optional[Any] = None, -) -> Dict[str, Any]: - """ - Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync. - progress_callback(current, total, error_count) opcional. - """ - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") - - total_rows = 0 - try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 - except Exception as e: - return {"status": "failed", "error": str(e)} - - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Vehicles import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - error_count = 0 - processed_rows = 0 - errors_detail: List[Dict[str, Any]] = [] - - try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if progress_callback and i % 500 == 0: - progress_callback(i, total_rows, error_count) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_vehicle(row_norm, i) - if err: - error_count += 1 - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) - processed_rows += 1 - - except Exception as e: - logger.error(f"Vehicles import scan failed: {e}") - return {"status": "failed", "error": str(e)} - - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}") - - result = { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - return result - - -def run_scan_sync(job_id: str) -> Dict[str, Any]: - """ - Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis. - Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery. - """ - result = _do_scan(job_id, progress_callback=None) - try: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") - return result - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"Vehicles import: starting scan for job {job_id}") - - def on_progress(current: int, total: int, errors: int) -> None: - self.update_state( - state="PROGRESS", - meta={"current": current, "total": total, "errors": errors}, - ) - - result = _do_scan(job_id, progress_callback=on_progress) - try: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") - return result - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]: - """Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row.""" - vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"]) - if not vehicle_key: - return {} - data = { - "vehicle_key": vehicle_key, - "ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]), - "transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]), - "series": _str_or_none(row.get("VIN"), _MAX["series"]), - "transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]), - "entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]), - "transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]), - "dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]), - "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), - "city": _str_or_none(row.get("CIUDAD"), _MAX["city"]), - "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), - "country": _str_or_none(row.get("PAIS"), _MAX["country"]), - "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), - "insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]), - "insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]), - "insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")), - "insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")), - } - return data - - -def _do_commit(job_id: str) -> Dict[str, Any]: - """ - Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync. - """ - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv") - if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - base_dir = os.path.dirname(file_path) - error_dir = base_dir.replace("temp", "errors") - error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") - - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Vehicles import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - from api.v1.modules.a76.transportation.vehicles.services import VehicleService - from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO - - inserted_count = 0 - updated_count = 0 - skipped_invalid = 0 - skipped_duplicate = 0 - skipped_details: List[Dict[str, Any]] = [] - seen_keys_in_file: Dict[str, int] = {} - - try: - with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_vehicle(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-", - "invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-", - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - data = _row_to_vehicle_dto(row_norm) - if not data or not data.get("vehicle_key"): - skipped_invalid += 1 - continue - - vk = data["vehicle_key"] - if vk in seen_keys_in_file: - skipped_duplicate += 1 - skipped_details.append( - {"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"} - ) - continue - seen_keys_in_file[vk] = i - - existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) - try: - if existing: - update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) - VehicleService.update(session, vk, tenant_id, update_data, company_id) - updated_count += 1 - else: - create_data = VehicleCreateDTO(**data) - VehicleService.create(session, create_data, tenant_id, company_id) - inserted_count += 1 - except Exception as db_err: - session.rollback() - skipped_invalid += 1 - skipped_details.append( - {"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)} - ) - continue - - try: - session.commit() - except Exception as db_err: - session.rollback() - logger.error(f"Vehicles import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} - - except Exception as e: - logger.error(f"Vehicles import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Vehicles import cleanup failed: {cleanup_err}") - - total_ok = inserted_count + updated_count - if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: - return { - "status": "warning", - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", - } - if total_ok == 0: - return { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "updated": 0, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - return { - "status": "finished", - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_duplicate": skipped_duplicate, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - - -def run_commit_sync(job_id: str) -> Dict[str, Any]: - """ - Ejecuta el commit en el proceso actual y guarda el resultado en Redis. - Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery. - """ - result = _do_commit(job_id) - try: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") - return result - - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService. - """ - logger.info(f"Vehicles import: starting commit for job {job_id}") - result = _do_commit(job_id) - try: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", - json.dumps(result).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") - return result diff --git a/backend/api/v1/modules/a76/transportation/vehicles/routes.py b/backend/api/v1/modules/a76/transportation/vehicles/routes.py index ce823259..1df5b21c 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/routes.py +++ b/backend/api/v1/modules/a76/transportation/vehicles/routes.py @@ -4,7 +4,7 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO from .services import VehicleService -from .imports.routes import router as imports_router +from api.v1.modules.a76.layouts_csv.vehicles.routes import router as imports_router # Main router: vehicles CRUD + CSV imports router = APIRouter() diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 8ebbca7c..4e739e77 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -33,16 +33,21 @@ celery_app.conf.update( "api.v1.modules.a76.reports.movements.invoices.tasks", "api.v1.modules.a76.reports.movements.saldos.tasks", "api.v1.modules.a76.reports.exportacion.descargo.task", - "api.v1.modules.a76.imports.tasks", - "api.v1.modules.a76.customs_brokers.imports.tasks", - "api.v1.modules.a76.clients_and_providers.imports.tasks", - "api.v1.modules.a76.pedmientos.imports.tasks", - "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", - "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", - "api.v1.modules.a76.classes.imports.tasks", - "api.v1.modules.a76.parts.imports.tasks", - "api.v1.modules.a76.boms.imports.tasks", - "api.v1.modules.a76.transportation.vehicles.imports.tasks", + "api.v1.modules.a76.layouts_csv.facturas.tasks", + "api.v1.modules.a76.layouts_csv.exportacion.tasks", + "api.v1.modules.a76.layouts_csv.cambio_regimen_regularizacion.tasks", + "api.v1.modules.a76.layouts_csv.customs_brokers.tasks", + "api.v1.modules.a76.layouts_csv.clients_and_providers.tasks", + "api.v1.modules.a76.layouts_csv.pedmientos.tasks", + "api.v1.modules.a76.layouts_csv.exchange_rate.tasks", + "api.v1.modules.a76.layouts_csv.us_tariff_fractions.tasks", + "api.v1.modules.a76.layouts_csv.classes.tasks", + "api.v1.modules.a76.layouts_csv.parts.tasks", + "api.v1.modules.a76.layouts_csv.boms.tasks", + "api.v1.modules.a76.layouts_csv.vehicles.tasks", + "api.v1.modules.a76.layouts_csv.drivers.tasks", + "api.v1.modules.a76.layouts_csv.trailers.tasks", + "api.v1.modules.a76.layouts_csv.transportistas.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/backend/core/paths.py b/backend/core/paths.py new file mode 100644 index 00000000..05e33b34 --- /dev/null +++ b/backend/core/paths.py @@ -0,0 +1,15 @@ +""" +Rutas base y resolución de paths para layouts (importación CSV, temp, errors). +""" +from pathlib import Path + +# Raíz del backend (directorio que contiene api/, core/, etc.) +BASE_DIR = Path(__file__).resolve().parent.parent + + +def layout_path(*parts: str) -> str: + """Construye una ruta absoluta bajo backend/layouts/.""" + p = BASE_DIR / "layouts" + for part in parts: + p = p / part + return str(p) diff --git a/backend/main.py b/backend/main.py index 6d6e3ddb..e158475c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -89,6 +89,7 @@ import core.celery_app # Initialize Celery App from api.v1.router import router as api_v1_router from core.config import settings from core.database import init_db +from core.paths import layout_path from core.error_handlers import register_exception_handlers from core.middleware import ( LicenseValidationMiddleware, @@ -334,6 +335,9 @@ uploads_dir = Path("uploads").resolve() uploads_dir.mkdir(parents=True, exist_ok=True) app.mount("/api/uploads", StaticFiles(directory=str(uploads_dir)), name="uploads") +# Directorios para importación CSV (layouts: temp y errors) +Path(layout_path("imports", "temp")).mkdir(parents=True, exist_ok=True) +Path(layout_path("imports", "errors")).mkdir(parents=True, exist_ok=True) # Registrar routers app.include_router(api_v1_router, prefix="/api/v1") diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index a525d200..fadc28b4 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -325,6 +325,8 @@ export const api = { validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`) }, + // CSV import for Operaciones de Importación/Exportación (facturas: encabezados y partidas). + // Backend: api/v1/modules/a76/layouts_csv/facturas (rutas bajo /v1/a76/imports/). imports: { upload: ( file: File, @@ -358,6 +360,40 @@ export const api = { api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget }) }, + // CSV import for Operaciones de Exportación (encabezado y partidas). + // Backend: layouts_csv/exportacion — rutas /v1/a76/imports/exportacion/ + exportacionImports: { + upload: ( + file: File, + modelTarget: string, + footerConfig: any, + companyId: number, + templateId?: string + ) => { + const formData = new FormData(); + formData.append('file', file); + if (footerConfig) { + formData.append('footer_config', JSON.stringify(footerConfig)); + } + if (templateId) { + formData.append('template_id', templateId); + } + + const queryParams = new URLSearchParams({ + company_id: String(companyId), + operation_type: 'exp' + }).toString(); + + return fetchApi(`/v1/a76/imports/exportacion/upload/${modelTarget}?${queryParams}`, { + method: 'POST', + body: formData + }); + }, + status: (jobId: string) => api.get(`/v1/a76/imports/exportacion/${jobId}/status`), + commit: (jobId: string, modelTarget: string) => + api.post(`/v1/a76/imports/exportacion/${jobId}/commit`, { model_target: modelTarget }) + }, + // CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports) customsBrokerImports: { upload: (file: File, companyId: number) => { @@ -390,11 +426,20 @@ export const api = { // CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports) exchangeRateImports: { - upload: (file: File, companyId: number) => { + upload: ( + file: File, + companyId: number, + params?: { reemplazar_sin_preguntar?: boolean; date_format?: string } + ) => { const formData = new FormData(); formData.append('file', file); + const search = new URLSearchParams({ company_id: String(companyId) }); + if (params?.reemplazar_sin_preguntar !== undefined) + search.set('reemplazar_sin_preguntar', String(!!params.reemplazar_sin_preguntar)); + if (params?.date_format != null && params.date_format !== '') + search.set('date_format', params.date_format); return fetchApi( - `/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`, + `/v1/a76/exchange-rate/imports/upload?${search.toString()}`, { method: 'POST', body: formData } ); }, @@ -420,11 +465,18 @@ export const api = { // CSV import for Pedimentos (pedimentos/imports) pedimentosImports: { - upload: (file: File, companyId: number) => { + upload: ( + file: File, + companyId: number, + params?: { actualizar?: boolean; dateFormat?: string } + ) => { const formData = new FormData(); formData.append('file', file); + const search = new URLSearchParams({ company_id: String(companyId) }); + if (params?.actualizar !== undefined) search.set('actualizar', String(!!params.actualizar)); + if (params?.dateFormat != null) search.set('dateFormat', params.dateFormat); return fetchApi( - `/v1/a76/pedimentos/imports/upload?company_id=${companyId}`, + `/v1/a76/pedimentos/imports/upload?${search.toString()}`, { method: 'POST', body: formData } ); }, @@ -435,11 +487,18 @@ export const api = { // CSV import for Clases de Materiales (classes/imports) materialClassImports: { - upload: (file: File, companyId: number) => { + upload: ( + file: File, + companyId: number, + params?: { actualizar?: boolean; siempre_toda?: boolean } + ) => { const formData = new FormData(); formData.append('file', file); + const search = new URLSearchParams({ company_id: String(companyId) }); + if (params?.actualizar !== undefined) search.set('actualizar', String(!!params.actualizar)); + if (params?.siempre_toda !== undefined) search.set('siempre_toda', String(!!params.siempre_toda)); return fetchApi( - `/v1/a76/classes/imports/upload?company_id=${companyId}`, + `/v1/a76/classes/imports/upload?${search.toString()}`, { method: 'POST', body: formData } ); }, @@ -450,11 +509,13 @@ export const api = { // CSV import for Vehículos / Transportes (transportation/vehicles/imports) vehicleImports: { - upload: (file: File, companyId: number) => { + upload: (file: File, companyId: number, options?: { actualizar?: boolean }) => { const formData = new FormData(); formData.append('file', file); + const params = new URLSearchParams({ company_id: String(companyId) }); + if (options?.actualizar !== undefined) params.set('actualizar', String(options.actualizar)); return fetchApi( - `/v1/a76/transportation/vehicles/imports/upload?company_id=${companyId}`, + `/v1/a76/transportation/vehicles/imports/upload?${params.toString()}`, { method: 'POST', body: formData } ); }, @@ -479,11 +540,12 @@ export const api = { // CSV import for Trailers y Cajas (transportation/trailers/imports) trailerImports: { - upload: (file: File, companyId: number) => { + upload: (file: File, companyId: number, params?: { actualizar?: boolean }) => { const formData = new FormData(); formData.append('file', file); + const actualizar = params?.actualizar ?? false; return fetchApi( - `/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}`, + `/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}&actualizar=${actualizar}`, { method: 'POST', body: formData } ); }, @@ -494,11 +556,12 @@ export const api = { // CSV import for Transportistas (transporters/imports) transporterImports: { - upload: (file: File, companyId: number) => { + upload: (file: File, companyId: number, params?: { actualizar?: boolean }) => { const formData = new FormData(); formData.append('file', file); + const actualizar = params?.actualizar ?? false; return fetchApi( - `/v1/a76/transporters/imports/upload?company_id=${companyId}`, + `/v1/a76/transporters/imports/upload?company_id=${companyId}&actualizar=${actualizar}`, { method: 'POST', body: formData } ); }, @@ -508,11 +571,14 @@ export const api = { // CSV import for Números de parte (parts/imports) partNumberImports: { - upload: (file: File, companyId: number) => { + upload: (file: File, companyId: number, options?: { actualizar?: boolean; reemplazar_sin_preguntar?: boolean }) => { const formData = new FormData(); formData.append('file', file); + const params = new URLSearchParams({ company_id: String(companyId) }); + if (options?.actualizar !== undefined) params.set('actualizar', String(options.actualizar)); + if (options?.reemplazar_sin_preguntar !== undefined) params.set('reemplazar_sin_preguntar', String(options.reemplazar_sin_preguntar)); return fetchApi( - `/v1/a76/parts/imports/upload?company_id=${companyId}`, + `/v1/a76/parts/imports/upload?${params.toString()}`, { method: 'POST', body: formData } ); }, diff --git a/frontend/src/lib/api/dashboard/a76/drivers.ts b/frontend/src/lib/api/dashboard/a76/drivers.ts index cb0725b1..3ebc7354 100644 --- a/frontend/src/lib/api/dashboard/a76/drivers.ts +++ b/frontend/src/lib/api/dashboard/a76/drivers.ts @@ -1,57 +1,105 @@ import { api, type ApiResponse } from '$lib/api'; export interface Driver { - transporter_key: string; - line: number; - driver_name?: string; - license_number?: string; - express_line_id?: string; - ace_id?: string; - birth_date?: number; - gender?: string; - birth_country?: string; - hazardous_material_auth?: string; - hazardous_material_state?: string; - first_name?: string; - last_name?: string; - badge_number?: string; - class_type?: string; + transporter_key: string; + line: number; + driver_name?: string; + license_number?: string; + express_line_id?: string; + ace_id?: string; + birth_date?: number; + gender?: string; + birth_country?: string; + hazardous_material_auth?: string; + hazardous_material_state?: string; + first_name?: string; + last_name?: string; + id_key1?: string; + id_number1?: string; + id_state1?: string; + id_country1?: string; + id_key2?: string; + id_number2?: string; + id_state2?: string; + id_country2?: string; + badge_number?: string; + class_type?: string; + unique_badge_number?: string; + company_id?: number; + tenant_id?: number; } export interface DriverResponse { - items: Driver[]; - total: number; - page: number; - page_size: number; + items: Driver[]; + total: number; + page: number; + page_size: number; } class DriversApi { - private baseUrl = '/v1/a76/drivers'; + private baseUrl = '/v1/a76/drivers'; - async list( - companyId: string | number, - page: number = 1, - pageSize: number = 50 - ): Promise> { - const params = new URLSearchParams({ - company_id: companyId.toString(), - page: page.toString(), - page_size: pageSize.toString() - }); + async list( + companyId: string | number, + params?: Record + ): Promise> { + const queryParams = new URLSearchParams({ + company_id: companyId.toString(), + ...(params as Record) + }); + return api.get(`${this.baseUrl}?${queryParams.toString()}`); + } - return api.get(`${this.baseUrl}?${params.toString()}`); - } + async get( + transporterKey: string, + line: number, + companyId: string | number + ): Promise> { + const params = new URLSearchParams({ + company_id: companyId.toString() + }); + return api.get( + `${this.baseUrl}/${transporterKey}/${line}?${params.toString()}` + ); + } - async get( - transporterKey: string, - line: number, - companyId: string | number - ): Promise> { - const params = new URLSearchParams({ - company_id: companyId.toString() - }); - return api.get(`${this.baseUrl}/${transporterKey}/${line}?${params.toString()}`); - } + async create( + data: Omit & { company_id: number; tenant_id: number }, + companyId: string | number + ): Promise> { + const params = new URLSearchParams({ + company_id: companyId.toString() + }); + return api.post(`${this.baseUrl}?${params.toString()}`, data); + } + + async update( + transporterKey: string, + line: number, + data: Partial, + companyId: string | number + ): Promise> { + const params = new URLSearchParams({ + company_id: companyId.toString() + }); + return api.put( + `${this.baseUrl}/${transporterKey}/${line}?${params.toString()}`, + data + ); + } + + async delete( + transporterKey: string, + line: number, + companyId: string | number + ): Promise> { + const params = new URLSearchParams({ + company_id: companyId.toString() + }); + return api.delete( + `${this.baseUrl}/${transporterKey}/${line}?${params.toString()}` + ); + } } export const driversApi = new DriversApi(); diff --git a/frontend/src/lib/api/dashboard/a76/transporters.ts b/frontend/src/lib/api/dashboard/a76/transporters.ts index 70ecbcdd..fa585465 100644 --- a/frontend/src/lib/api/dashboard/a76/transporters.ts +++ b/frontend/src/lib/api/dashboard/a76/transporters.ts @@ -34,16 +34,22 @@ export interface TransporterResponse { class TransportersApi { private baseUrl = '/v1/a76/transporters'; - async list( - companyId: string | number, - params?: Record - ): Promise> { - const queryParams = new URLSearchParams({ - company_id: companyId.toString(), - ...params - }); - return api.get(`${this.baseUrl}?${queryParams.toString()}`); - } + async list( + companyId: string | number, + params?: Record + ): Promise> { + const raw: Record = { + company_id: companyId.toString(), + ...params + }; + const queryParams = new URLSearchParams(); + for (const [k, v] of Object.entries(raw)) { + if (v != null && v !== '') { + queryParams.set(k, String(v)); + } + } + return api.get(`${this.baseUrl}?${queryParams.toString()}`); + } async get(id: string, companyId: string | number): Promise> { const queryParams = new URLSearchParams({ diff --git a/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte b/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte new file mode 100644 index 00000000..45c640e8 --- /dev/null +++ b/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte @@ -0,0 +1,124 @@ + + + +
+
+
+ +
+
+ + Parámetros globales +
+ {#each globalCsvParams as param} +
+ + +
+ {/each} +
+ + + {#if currentTabFields.length > 0} +
+ Configuración: {activeTab} + {#each currentTabFields as field} +
+ {#if field.type !== 'boolean'} + + {/if} + {#if field.type === 'select' && field.options} + + {:else if field.type === 'radio' && field.options} +
+ {#each field.options as opt} + + {/each} +
+ {:else if field.type === 'boolean'} + + {/if} +
+ {/each} +
+ {/if} +
+
+
diff --git a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte index bd7bac46..0e2a0b4e 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte @@ -1,311 +1,312 @@ - - - - - -
-
- {#if isPending} - - {:else if isFinished && !hasErrors} - - {:else} - - {/if} -
- -
- - {#if isPending} - Validación de Importación - {:else if isFinished} - {hasErrors ? 'Importación con Observaciones' : 'Importación Exitosa'} - {/if} - - - {#if isPending} - Revise el análisis preliminar antes de confirmar la carga de datos. - {:else if isFinished} - El proceso de importación ha finalizado. - {/if} - -
-
- - -
- - {#if isPending} -
- -
- Total Filas - {scanResults.total_rows || 0} -
- - -
- Válidos - {scanResults.valid_rows || 0} -
- - -
- Errores - {scanResults.error_count || 0} -
-
- - {#if scanResults.error_count > 0} -
- -
-

Se detectaron problemas en el archivo

-

- Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para - importar solo las filas válidas (las erróneas se omitirán). -

-
-
- {#if scanResults.errors && scanResults.errors.length > 0} -
-
-
- Detalle de errores (para corregir en el CSV) -
- - {scanResults.errors.length} error(es) - -
-
- - - - - - - - - - {#each scanResults.errors as err} - - - - - - {/each} - -
LíneaColumnaMensaje
{err.line}{err.col || '-'}{err.msg || '-'}
-
-
- {/if} - {:else} -
- -
-

Archivo validado correctamente

-

Todos los registros parecen correctos y listos para importar.

-
-
- {/if} - {/if} - - - {#if isFinished} -
- -
- -
-
- - Insertados -
- {commitResults.inserted || 0} -
- - -
-
- - Rechazados -
- {totalSkipped} -
-
- - - {#if commitResults.skipped_details && commitResults.skipped_details.length > 0} -
-
-
- Detalle de Errores -
- - {commitResults.skipped_details.length} filas - -
-
- - - - - - - - - - {#each commitResults.skipped_details as detail} - - - - - - {/each} - -
LíneaReferenciaMotivo
{detail.line}{detail.invoice || '-'}{detail.reason}
-
-
- {/if} -
- {/if} -
- - -
- {#if isPending} - - - {:else if isFinished} - - {/if} -
- - + + + + + +
+
+ {#if isPending} + + {:else if isFinished && !hasErrors} + + {:else} + + {/if} +
+ +
+ + {#if isPending} + Validación de Importación + {:else if isFinished} + {hasErrors ? 'Importación con Observaciones' : 'Importación Exitosa'} + {/if} + + + {#if isPending} + Revise el análisis preliminar antes de confirmar la carga de datos. + {:else if isFinished} + El proceso de importación ha finalizado. + {/if} + +
+
+ + +
+ + {#if isPending} +
+ +
+ Total Filas + {scanResults.total_rows || 0} +
+ + +
+ Válidos + {scanResults.valid_rows || 0} +
+ + +
+ Errores + {scanResults.error_count || 0} +
+
+ + {#if scanResults.error_count > 0} +
+ +
+

Se detectaron problemas en el archivo

+

+ Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para + importar solo las filas válidas (las erróneas se omitirán). +

+
+
+ {#if scanResults.errors && scanResults.errors.length > 0} +
+
+
+ Detalle de errores (para corregir en el CSV) +
+ + {scanResults.errors.length} error(es) + +
+
+ + + + + + + + + + {#each scanResults.errors as err} + + + + + + {/each} + +
LíneaColumnaMensaje
{err.line}{err.col || '-'}{err.msg || '-'}
+
+
+ {/if} + {:else} +
+ +
+

Archivo validado correctamente

+

Todos los registros parecen correctos y listos para importar.

+
+
+ {/if} + {/if} + + + {#if isFinished} +
+ +
+ +
+
+ + Insertados +
+ {commitResults.inserted || 0} +
+ + +
+
+ + Rechazados +
+ {totalSkipped} +
+
+ + + {#if commitResults.skipped_details && commitResults.skipped_details.length > 0} +
+
+
+ Detalle de Errores +
+ + {commitResults.skipped_details.length} filas + +
+
+ + + + + + + + + + {#each commitResults.skipped_details as detail} + + + + + + {/each} + +
LíneaReferenciaMotivo
{detail.line}{detail.invoice || '-'}{detail.reason}
+
+
+ {/if} +
+ {/if} +
+ + +
+ {#if isPending} + + + {:else if isFinished} + + {/if} +
+ + diff --git a/frontend/src/lib/components/dashboard/pedimentos/columns.ts b/frontend/src/lib/components/dashboard/pedimentos/columns.ts index 69341969..6591717a 100644 --- a/frontend/src/lib/components/dashboard/pedimentos/columns.ts +++ b/frontend/src/lib/components/dashboard/pedimentos/columns.ts @@ -138,7 +138,8 @@ export function createColumns(onSuccess?: () => void): ColumnDef[] { header: "Número de Pedimento", cell: ({ row }) => { const pedimento = row.original; - const fullNumber = `${pedimento.year || ''}-${pedimento.customs_office || ''}-${pedimento.license || ''}-${pedimento.pedimento_number || ''}`; + const customs2 = (pedimento.customs_office ?? '').toString().slice(0, 2); + const fullNumber = `${pedimento.year || ''}-${customs2}-${pedimento.license || ''}-${pedimento.pedimento_number || ''}`; const numberSnippet = createRawSnippet<[{ number: string }]>((getNumber) => { const { number } = getNumber(); diff --git a/frontend/src/lib/components/dashboard/pedimentos/edit/general-tab-form.svelte b/frontend/src/lib/components/dashboard/pedimentos/edit/general-tab-form.svelte index aa8d3f8e..88b31a85 100644 --- a/frontend/src/lib/components/dashboard/pedimentos/edit/general-tab-form.svelte +++ b/frontend/src/lib/components/dashboard/pedimentos/edit/general-tab-form.svelte @@ -665,9 +665,9 @@
- +
- + void): ColumnDef[] { + return [ + { + accessorKey: 'transporter_key', + header: 'Clave Transportista', + cell: ({ row }) => row.original.transporter_key + }, + { + accessorKey: 'line', + header: 'Línea', + cell: ({ row }) => row.original.line + }, + { + accessorKey: 'driver_name', + header: 'Nombre del Conductor', + cell: ({ row }) => row.original.driver_name || '-' + }, + { + accessorKey: 'license_number', + header: 'Número de Licencia', + cell: ({ row }) => row.original.license_number || '-' + }, + { + accessorKey: 'first_name', + header: 'Nombre', + cell: ({ row }) => row.original.first_name || '-' + }, + { + accessorKey: 'last_name', + header: 'Apellido', + cell: ({ row }) => row.original.last_name || '-' + }, + { + id: 'actions', + header: 'Acciones', + cell: ({ row }) => { + return renderComponent(DataTableActions, { + item: row.original, + onSuccess + }); + } + } + ]; +} diff --git a/frontend/src/lib/components/dashboard/transportation/drivers/create-edit-dialog.svelte b/frontend/src/lib/components/dashboard/transportation/drivers/create-edit-dialog.svelte new file mode 100644 index 00000000..10304064 --- /dev/null +++ b/frontend/src/lib/components/dashboard/transportation/drivers/create-edit-dialog.svelte @@ -0,0 +1,313 @@ + + + + + + {title} + + {isEdit + ? 'Modifica los datos del conductor' + : 'Completa los datos para crear un nuevo conductor'} + + + +
{ + e.preventDefault(); + handleSubmit(); + }} + class="space-y-6" + > + {#if error} +
+ {error} +
+ {/if} + +
+
+ + {#if isEdit} + + {:else} + + + {transportersLoading + ? 'Cargando transportistas...' + : transporters.length === 0 + ? 'No hay transportistas' + : transporters.find((t) => t.transporter_key === formData.transporter_key) + ? `${formData.transporter_key} - ${transporters.find((t) => t.transporter_key === formData.transporter_key)?.name || ''}` + : 'Seleccionar transportista'} + + + {#each transporters as t} + + {t.transporter_key} — {t.name || t.short_name || 'Sin nombre'} + + {/each} + {#if !transportersLoading && transporters.length === 0} +
+ No hay transportistas. Crea uno en el catálogo Transportistas. +
+ {/if} + + + {/if} +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+ + + + + +
+ + diff --git a/frontend/src/lib/components/dashboard/transportation/drivers/data-table-actions.svelte b/frontend/src/lib/components/dashboard/transportation/drivers/data-table-actions.svelte new file mode 100644 index 00000000..442c66c6 --- /dev/null +++ b/frontend/src/lib/components/dashboard/transportation/drivers/data-table-actions.svelte @@ -0,0 +1,116 @@ + + + + + {#snippet child({ props })} + + {/snippet} + + + Acciones + + + + Editar + + + + {#if loading} + + {:else} + + {/if} + Eliminar + + + + + diff --git a/frontend/src/lib/components/dashboard/transportation/drivers/data-table.svelte b/frontend/src/lib/components/dashboard/transportation/drivers/data-table.svelte new file mode 100644 index 00000000..e0cdf7c1 --- /dev/null +++ b/frontend/src/lib/components/dashboard/transportation/drivers/data-table.svelte @@ -0,0 +1,99 @@ + + +
+ + + {#each table.getHeaderGroups() as headerGroup (headerGroup.id)} + + {#each headerGroup.headers as header (header.id)} + + {#if !header.isPlaceholder} + + {/if} + + {/each} + + {/each} + + + {#each table.getRowModel().rows as row (row.id)} + + {#each row.getVisibleCells() as cell (cell.id)} + + + + {/each} + + {:else} + + + No hay resultados. + + + {/each} + + +
+ +
+
+ Total: {totalItems} +
+
+ + +
+
diff --git a/frontend/src/lib/components/sidebar/modules.ts b/frontend/src/lib/components/sidebar/modules.ts index 812103f5..0566199d 100644 --- a/frontend/src/lib/components/sidebar/modules.ts +++ b/frontend/src/lib/components/sidebar/modules.ts @@ -344,6 +344,10 @@ export function getSidebarData(): SidebarData { title: "Transportistas", url: "/dashboard/general_catalogs/transporters", }, + { + title: "Conductores", + url: "/dashboard/general_catalogs/drivers", + }, { title: "Trailers", url: "/dashboard/general_catalogs/trailers", diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index c51ceaae..3af99c66 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -31,6 +31,8 @@ export interface CsvUploadItem { /** Backend template id for CSV download (e.g. customs_brokers, part_numbers). No physical file. */ templateId?: string; disabled?: boolean; // New property to mark items as "Coming Soon" + /** Backend module path (layouts_csv) for traceability, e.g. "layouts_csv/facturas", "layouts_csv/parts". */ + layoutModule?: string; } export interface CsvUploadField { @@ -42,6 +44,80 @@ export interface CsvUploadField { defaultValue?: any; } +/** Global parameters shown in the CSV upload footer bar (one option or the other via select). */ +export interface GlobalCsvParam { + name: string; + label: string; + type: 'select'; + options: { label: string; value: string }[]; + defaultValue: string; +} + +/** Global parameters for all CSV loads; merged with tab-specific settings when sending footer_config. */ +export const globalCsvParams: GlobalCsvParam[] = [ + { + name: 'mode', + label: 'Modo de Carga', + type: 'select', + options: [ + { label: 'Actualizar', value: 'update' }, + { label: 'Reemplazar', value: 'replace' } + ], + defaultValue: 'update' + }, + { + name: 'dateFormat', + label: 'Formato de Fecha', + type: 'select', + options: [ + { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, + { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, + { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } + ], + defaultValue: 'dd/mm/yyyy' + }, + { + name: 'weight_unit', + label: 'Unidad de Peso', + type: 'select', + options: [ + { label: 'Kilos (Kgs)', value: 'kgs' }, + { label: 'Libras (Lbs)', value: 'lbs' } + ], + defaultValue: 'kgs' + }, + { + name: 'autonumber_series', + label: 'Autonumerar Partidas/Series', + type: 'select', + options: [ + { label: 'Sí', value: 'true' }, + { label: 'No', value: 'false' } + ], + defaultValue: 'false' + }, + { + name: 'load_subpartidas', + label: 'Levantar Subpartidas', + type: 'select', + options: [ + { label: 'Sí', value: 'true' }, + { label: 'No', value: 'false' } + ], + defaultValue: 'false' + }, + { + name: 'recalculate_pedimento_date', + label: 'Recalcular Fecha Pedimento', + type: 'select', + options: [ + { label: 'Sí', value: 'true' }, + { label: 'No', value: 'false' } + ], + defaultValue: 'false' + } +]; + // Map of Tab ID -> Array of Fields export const tabSettings: Record = { catalogos: [ @@ -125,42 +201,47 @@ export const tabSettings: Record = { }; // --- DATA DEFINITIONS (Items only, no config) --- - +// Catálogos: la mayoría usa backend en layouts_csv (customs_brokers, clients_and_providers, etc.). export const catalogosConfig: CsvUploadItem[] = [ { id: 'customs_brokers', title: 'Agentes Aduanales', icon: User, modelTarget: 'CustomsBroker', - templateId: 'customs_brokers' + templateId: 'customs_brokers', + layoutModule: 'layouts_csv/customs_brokers' }, { id: 'clients_providers', title: 'Clientes y Proveedores', icon: Users, modelTarget: 'ClientProvider', - templateId: 'clients_providers' + templateId: 'clients_providers', + layoutModule: 'layouts_csv/clients_and_providers' }, { id: 'exchange_rates', title: 'Tipo de Cambios', icon: DollarSign, modelTarget: 'ExchangeRate', - templateId: 'exchange_rates' + templateId: 'exchange_rates', + layoutModule: 'layouts_csv/exchange_rate' }, { id: 'american_fractions', title: 'Fracc. Ame.', icon: Globe, modelTarget: 'AmericanFraction', - templateId: 'american_fractions' + templateId: 'american_fractions', + layoutModule: 'layouts_csv/us_tariff_fractions' }, { id: 'material_classes', title: 'Clases de Materiales', icon: Package, modelTarget: 'MaterialClass', - templateId: 'material_classes' + templateId: 'material_classes', + layoutModule: 'layouts_csv/classes' }, { id: 'part_numbers', @@ -168,6 +249,7 @@ export const catalogosConfig: CsvUploadItem[] = [ icon: Hash, modelTarget: 'Part', templateId: 'part_numbers', + layoutModule: 'layouts_csv/parts' }, { id: 'boms', @@ -175,6 +257,7 @@ export const catalogosConfig: CsvUploadItem[] = [ icon: Briefcase, modelTarget: 'Bom', templateId: 'boms', + layoutModule: 'layouts_csv/boms' }, { id: 'items', @@ -204,41 +287,49 @@ export const catalogosConfig: CsvUploadItem[] = [ title: 'Pedimentos', icon: FileDigit, modelTarget: 'Pedimento', - templateId: 'pedimentos' + templateId: 'pedimentos', + layoutModule: 'layouts_csv/pedmientos' }, ]; +// Transportes: conductores, trailers y “Transportes” (vehículos) usan layouts_csv; transportistas sin layouts_csv aún. export const transportesConfig: CsvUploadItem[] = [ { id: 'transporters', title: 'Transportistas', icon: Ship, modelTarget: 'Transporter', - // No templateId: backend transporters/imports not implemented yet + templateId: 'transporters', + layoutModule: 'layouts_csv/transportistas' }, { id: 'transports', title: 'Transportes', icon: Truck, modelTarget: 'Transport', - templateId: 'transports' + templateId: 'transports', + layoutModule: 'layouts_csv/vehicles' }, { id: 'drivers', title: 'Conductores', icon: User, modelTarget: 'Driver', - templateId: 'drivers' + templateId: 'drivers', + layoutModule: 'layouts_csv/drivers' }, { id: 'trailers', title: 'Trailers y Cajas', icon: Container, modelTarget: 'Trailer', - templateId: 'trailers' + templateId: 'trailers', + layoutModule: 'layouts_csv/trailers' }, ]; +// --- Operaciones de Importación (facturas: encabezados y partidas) +// Backend: layouts_csv/facturas — rutas /v1/a76/imports/ (upload, status, commit). export const importacionConfig: CsvUploadItem[] = [ // Impo Temp { @@ -247,7 +338,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: FileText, group: 'Impo. Temp.', modelTarget: 'invoice_header', - templateId: 'imp_temp_header' + templateId: 'imp_temp_header', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_temp_details', @@ -255,7 +347,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: Package, group: 'Impo. Temp.', modelTarget: 'invoice_details', - templateId: 'imp_temp_details' + templateId: 'imp_temp_details', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_temp_series', @@ -272,7 +365,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: FileText, group: 'Impo. Def.', modelTarget: 'invoice_header', - templateId: 'imp_def_header' + templateId: 'imp_def_header', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_def_details', @@ -280,7 +374,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: Package, group: 'Impo. Def.', modelTarget: 'invoice_details', - templateId: 'imp_def_details' + templateId: 'imp_def_details', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_def_series', @@ -317,6 +412,8 @@ export const importacionConfig: CsvUploadItem[] = [ }, ]; +// --- Operaciones de Exportación (facturas: encabezados y partidas) +// Backend: layouts_csv/exportacion — rutas /v1/a76/imports/exportacion/ export const exportacionConfig: CsvUploadItem[] = [ // Expo Def / Cam. Reg. { @@ -325,7 +422,8 @@ export const exportacionConfig: CsvUploadItem[] = [ icon: FileText, group: 'Expo. Def./Cam. Reg.', modelTarget: 'invoice_header', - templateId: 'exp_def_header' + templateId: 'exp_def_header', + layoutModule: 'layouts_csv/exportacion' }, { id: 'exp_def_details', @@ -333,7 +431,8 @@ export const exportacionConfig: CsvUploadItem[] = [ icon: Package, group: 'Expo. Def./Cam. Reg.', modelTarget: 'invoice_details', - templateId: 'exp_def_details' + templateId: 'exp_def_details', + layoutModule: 'layouts_csv/exportacion' }, { id: 'exp_def_series', diff --git a/frontend/src/routes/dashboard/+layout.svelte b/frontend/src/routes/dashboard/+layout.svelte index 68d3011f..500e1edd 100644 --- a/frontend/src/routes/dashboard/+layout.svelte +++ b/frontend/src/routes/dashboard/+layout.svelte @@ -106,7 +106,7 @@ -->
-
+
{@render children?.()}
diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 5a1901f1..5f79624d 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -1,7 +1,7 @@ -
- -
+
+ +

Importación Masiva de Datos (CSV)

@@ -449,6 +622,7 @@
+

Catálogos Generales

@@ -456,6 +630,7 @@ +

Logística y Transporte

@@ -463,6 +638,7 @@ +

Operaciones de Importación

@@ -478,15 +654,12 @@
-
+ +
- - {#if allSettings[activeTab]} -
- -
- {/if} + +
{#if scanResults || commitResults} @@ -523,7 +696,9 @@ ? await api.partNumberImports.commit(currentJobId) : useBomImport ? await api.bomImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useExportacionImport + ? await api.exportacionImports.commit(currentJobId, activeModelTarget || '') + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); diff --git a/frontend/src/routes/dashboard/general_catalogs/drivers/+page.svelte b/frontend/src/routes/dashboard/general_catalogs/drivers/+page.svelte new file mode 100644 index 00000000..03a29c1c --- /dev/null +++ b/frontend/src/routes/dashboard/general_catalogs/drivers/+page.svelte @@ -0,0 +1,112 @@ + + +
+
+
+

Conductores

+

Gestión del catálogo de conductores

+
+ +
+ +
+ + +
+ + {#if loading && data.length === 0} +
+ Cargando conductores... +
+ {:else} + + {/if} + + +
diff --git a/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte b/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte index 6fa87ac2..8e342ed2 100644 --- a/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte +++ b/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte @@ -408,14 +408,13 @@ } } - // Validar campos requeridos para creación + // Validar campos requeridos para creación (client_id es opcional) if (data.isCreate && generalFormData) { - const requiredFields = { + const requiredFields: Record = { year: 'Año', customs_office: 'Aduana', license: 'Patente', pedimento_number: 'Número de Pedimento', - client_id: 'ID del Cliente', pedimento_code: 'Clave', regime: 'Régimen' }; @@ -1101,9 +1100,9 @@

{#if generalFormData?.year && generalFormData?.customs_office && generalFormData?.license && generalFormData?.pedimento_number} - Número: {generalFormData.year}-{generalFormData.customs_office}-{generalFormData.license}-{generalFormData.pedimento_number} + Número: {generalFormData.year}-{(generalFormData.customs_office ?? '').toString().slice(0, 2)}-{generalFormData.license}-{generalFormData.pedimento_number} {:else if !data.isCreate && data.pedimento?.pedimento_number} - Número: {data.pedimento?.year ?? ''}-{data.pedimento?.customs_office ?? ''}-{data + Número: {data.pedimento?.year ?? ''}-{(data.pedimento?.customs_office ?? '').toString().slice(0, 2)}-{data .pedimento?.license ?? ''}-{data.pedimento?.pedimento_number ?? ''} {:else} Edita los detalles del pedimento @@ -1151,9 +1150,9 @@ generalFormData?.customs_office && generalFormData?.license && generalFormData?.pedimento_number - ? `${generalFormData.year}-${generalFormData.customs_office}-${generalFormData.license}-${generalFormData.pedimento_number}` + ? `${generalFormData.year}-${(generalFormData.customs_office ?? '').toString().slice(0, 2)}-${generalFormData.license}-${generalFormData.pedimento_number}` : !data.isCreate && data.pedimento?.pedimento_number - ? `${data.pedimento?.year ?? ''}-${data.pedimento?.customs_office ?? ''}-${data.pedimento?.license ?? ''}-${data.pedimento?.pedimento_number ?? ''}` + ? `${data.pedimento?.year ?? ''}-${(data.pedimento?.customs_office ?? '').toString().slice(0, 2)}-${data.pedimento?.license ?? ''}-${data.pedimento?.pedimento_number ?? ''}` : ''} />