From e83205caf93187a261af2aebc9a4f2c5e4de72a0 Mon Sep 17 00:00:00 2001 From: hreyes Date: Wed, 4 Mar 2026 08:12:59 -0700 Subject: [PATCH] feature/abstraccion-de-funcionalidades-y-manejo-por-tareas-en-comun --- .../a76/layouts_csv/boms/common/__init__.py | 1 + .../boms/common/common_validators.py | 79 ++ .../a76/layouts_csv/boms/common/fk_loader.py | 27 + .../a76/layouts_csv/boms/common/mappers.py | 49 ++ .../v1/modules/a76/layouts_csv/boms/tasks.py | 444 +++--------- .../layouts_csv/boms/validators/__init__.py | 3 + .../a76/layouts_csv/boms/validators/common.py | 54 ++ .../a76/layouts_csv/boms/validators/create.py | 43 ++ .../layouts_csv/classes/common/__init__.py | 1 + .../classes/common/common_validators.py | 64 ++ .../layouts_csv/classes/common/fk_loader.py | 37 + .../a76/layouts_csv/classes/common/mappers.py | 62 ++ .../modules/a76/layouts_csv/classes/tasks.py | 571 ++++----------- .../classes/validators/__init__.py | 3 + .../layouts_csv/classes/validators/common.py | 55 ++ .../layouts_csv/classes/validators/create.py | 40 ++ .../clients_and_providers/common/__init__.py | 1 + .../common/common_validators.py | 66 ++ .../clients_and_providers/common/mappers.py | 87 +++ .../clients_and_providers/tasks.py | 550 ++++---------- .../validators/__init__.py | 3 + .../validators/common.py | 52 ++ .../validators/create.py | 8 + .../a76/layouts_csv/common/__init__.py | 1 + .../a76/layouts_csv/common/csv_reader.py | 28 + .../v1/modules/a76/layouts_csv/common/meta.py | 35 + .../a76/layouts_csv/common/normalize.py | 16 + .../a76/layouts_csv/common/responses.py | 47 ++ .../modules/a76/layouts_csv/common/storage.py | 171 +++++ .../customs_brokers/common/__init__.py | 1 + .../common/common_validators.py | 28 + .../customs_brokers/common/mappers.py | 72 ++ .../a76/layouts_csv/customs_brokers/tasks.py | 481 ++++--------- .../customs_brokers/validators/__init__.py | 3 + .../customs_brokers/validators/common.py | 23 + .../customs_brokers/validators/create.py | 6 + .../layouts_csv/drivers/common/__init__.py | 1 + .../drivers/common/common_validators.py | 118 +++ .../a76/layouts_csv/drivers/common/mappers.py | 62 ++ .../modules/a76/layouts_csv/drivers/tasks.py | 470 +++--------- .../drivers/validators/__init__.py | 3 + .../layouts_csv/drivers/validators/common.py | 52 ++ .../layouts_csv/drivers/validators/create.py | 27 + .../exchange_rate/common/__init__.py | 1 + .../exchange_rate/common/common_validators.py | 65 ++ .../exchange_rate/common/mappers.py | 48 ++ .../a76/layouts_csv/exchange_rate/tasks.py | 500 ++++--------- .../exchange_rate/validators/__init__.py | 3 + .../exchange_rate/validators/common.py | 31 + .../exchange_rate/validators/create.py | 6 + .../modules/a76/layouts_csv/facturas/tasks.py | 209 ++---- .../a76/layouts_csv/parts/common/__init__.py | 1 + .../parts/common/common_validators.py | 49 ++ .../a76/layouts_csv/parts/common/fk_loader.py | 48 ++ .../a76/layouts_csv/parts/common/mappers.py | 111 +++ .../v1/modules/a76/layouts_csv/parts/tasks.py | 672 ++++-------------- .../layouts_csv/parts/validators/__init__.py | 4 + .../layouts_csv/parts/validators/common.py | 50 ++ .../layouts_csv/parts/validators/create.py | 39 + .../layouts_csv/pedmientos/common/__init__.py | 1 + .../pedmientos/common/common_validators.py | 82 +++ .../pedmientos/common/fk_loader.py | 38 + .../layouts_csv/pedmientos/common/mappers.py | 59 ++ .../a76/layouts_csv/pedmientos/tasks.py | 617 ++++------------ .../pedmientos/validators/__init__.py | 3 + .../pedmientos/validators/common.py | 78 ++ .../pedmientos/validators/create.py | 37 + .../layouts_csv/trailers/common/__init__.py | 1 + .../trailers/common/common_validators.py | 39 + .../layouts_csv/trailers/common/mappers.py | 35 + .../modules/a76/layouts_csv/trailers/tasks.py | 360 +++------- .../trailers/validators/__init__.py | 3 + .../layouts_csv/trailers/validators/common.py | 32 + .../layouts_csv/trailers/validators/create.py | 22 + .../us_tariff_fractions/common/__init__.py | 1 + .../common/common_validators.py | 62 ++ .../us_tariff_fractions/common/mappers.py | 45 ++ .../layouts_csv/us_tariff_fractions/tasks.py | 528 ++++---------- .../validators/__init__.py | 3 + .../us_tariff_fractions/validators/common.py | 40 ++ .../us_tariff_fractions/validators/create.py | 6 + .../layouts_csv/vehicles/common/__init__.py | 1 + .../vehicles/common/common_validators.py | 104 +++ .../layouts_csv/vehicles/common/mappers.py | 47 ++ .../modules/a76/layouts_csv/vehicles/tasks.py | 545 +++----------- .../vehicles/validators/__init__.py | 3 + .../layouts_csv/vehicles/validators/common.py | 58 ++ .../layouts_csv/vehicles/validators/create.py | 31 + frontend/src/lib/api.ts | 2 + frontend/src/lib/config/csv-upload.ts | 56 +- .../routes/dashboard/csv-upload/+page.svelte | 3 + 91 files changed, 4153 insertions(+), 4571 deletions(-) create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/common/meta.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/common/normalize.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/common/responses.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/common/storage.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py create mode 100644 backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py new file mode 100644 index 00000000..2d458d94 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/__init__.py @@ -0,0 +1 @@ +# common validators, mappers, fk_loader for boms CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py new file mode 100644 index 00000000..7710d9b4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/common_validators.py @@ -0,0 +1,79 @@ +""" +Helpers reutilizables para validación de filas CSV (BOMs). +""" +import re +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional + + +def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_decimal_required_min( + row: Dict[str, Any], col: str, line_num: int, min_val: Decimal +) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or val == "": + return {"line": line_num, "col": col, "msg": "Requerido"} + try: + v = Decimal(str(val)) + if v < min_val: + return {"line": line_num, "col": col, "msg": "Debe ser mayor o igual a cero"} + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser número"} + return None + + +def _optional_number(val: Any) -> bool: + if val is None: + return True + s = re.sub(r"\s+", "", str(val).strip()) + if not s: + return True + try: + float(s.replace(",", ".")) + return True + except (ValueError, TypeError): + return False + + +def check_optional_number(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + if not _optional_number(row.get(col)): + return {"line": line_num, "col": col, "msg": "Debe ser número"} + return None + + +def check_in_set( + row: Dict[str, Any], + col: str, + line_num: int, + allowed: Optional[set], + msg: str, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val or allowed is None or len(allowed) == 0: + return None + if val not in allowed: + return {"line": line_num, "col": col, "msg": msg} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py new file mode 100644 index 00000000..1ee2b81c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/fk_loader.py @@ -0,0 +1,27 @@ +""" +Carga de conjuntos FK para validación de import CSV de BOMs. +""" +from typing import Set + +from core.database import CoreSessionLocal + + +def load_boms_fk_sets(tenant_id: int, company_id: int) -> Set[str]: + """Carga valid_part_numbers (Part.part_number por tenant/company).""" + valid_part_numbers: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.parts.models import Part + for p in ( + session.query(Part.part_number) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + valid_part_numbers.add(p[0]) + except Exception as e: + import logging + logging.getLogger(__name__).warning("BOMs import: could not load parts: %s", e) + return valid_part_numbers diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py new file mode 100644 index 00000000..997723a8 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/common/mappers.py @@ -0,0 +1,49 @@ +""" +Mapeo fila CSV → datos para BOM (para cuando exista tabla BOM). +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, Set + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val).replace(",", ".")) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_bom_data( + row_norm: Dict[str, Any], + valid_part_numbers: Set[str], +) -> Dict[str, Any]: + """ + Mapea una fila normalizada del CSV a un diccionario de datos para BOM. + Para uso futuro cuando exista tabla BOM. + """ + parent = _str_or_none(row_norm.get("NUMPARTE_PADRE"), 70) + component = _str_or_none(row_norm.get("NUMPARTE_COMPONENTE"), 70) + quantity = _decimal_or_none(row_norm.get("CANTIDAD")) + uom = _str_or_none(row_norm.get("UNIMED"), 10) + version_bom = _decimal_or_none(row_norm.get("VERSION_BOM")) + version_bill = _decimal_or_none(row_norm.get("VERSION_BILL")) + return { + "parent_part_number": parent, + "component_part_number": component, + "quantity": quantity, + "uom": uom, + "version_bom": version_bom, + "version_bill": version_bill, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py b/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py index ea544d8d..5e585be5 100644 --- a/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py @@ -1,430 +1,168 @@ """ Tareas Celery para importación CSV de BOMs. Flujo: scan_file (validación) → insert_valid_rows (commit). -Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista. +Sin tabla BOM dedicada aún: insert_valid_rows solo valida y cuenta filas válidas. +Usa layouts_csv.common y common.fk_loader, validators. """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from decimal import Decimal, InvalidOperation -from typing import Dict, Any, Optional, List, Set +import os +from typing import Dict, Any, List from core.celery_app import celery_app -from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import csv_reader as common_csv +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template +from .validators import validate_row_bom +from .common.fk_loader import load_boms_fk_sets logger = logging.getLogger(__name__) +JOB_TYPE = "bom" + +# Para routes.py BOM_IMPORT_FILE_PREFIX = "bom_import_file:" BOM_IMPORT_META_PREFIX = "bom_import_meta:" BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:" -BOM_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"BOMs import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"bom_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"BOMs import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{BOM_IMPORT_FILE_PREFIX}{job_id}", - f"{BOM_IMPORT_META_PREFIX}{job_id}", - f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"BOMs import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_bom( - row: Dict[str, Any], - line_num: int, - valid_part_numbers: Optional[Set[str]] = None, -) -> Optional[Dict[str, Any]]: - """Valida una fila BOM según columnas del template. FK opcional a a76.parts.""" - parent = (row.get("NUMPARTE_PADRE") or "").strip() - if not parent: - return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"} - if len(parent) > 70: - return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"} - - component = (row.get("NUMPARTE_COMPONENTE") or "").strip() - if not component: - return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"} - if len(component) > 70: - return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"} - - qty = row.get("CANTIDAD") - if qty is None or qty == "": - return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"} - try: - val = Decimal(str(qty)) - if val < 0: - return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"} - except (InvalidOperation, ValueError, TypeError): - return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"} - - uom = (row.get("UNIMED") or "").strip() - if uom and len(uom) > 10: - return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"} - - def _optional_number(val: Any) -> bool: - if val is None: - return True - s = re.sub(r"\s+", "", str(val).strip()) - if not s: - return True - try: - float(s.replace(",", ".")) - return True - except (ValueError, TypeError): - return False - - version_bom = row.get("VERSION_BOM") - if not _optional_number(version_bom): - return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"} - - version_bill = row.get("VERSION_BILL") - if not _optional_number(version_bill): - return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"} - - # Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas) - if valid_part_numbers is not None and len(valid_part_numbers) > 0: - if parent not in valid_part_numbers: - return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"} - if component not in valid_part_numbers: - return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"} - - return None +BOM_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): - logger.info(f"BOMs import: starting scan for job {job_id}") + logger.info("BOMs import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"BOMs import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_part_numbers: Set[str] = set() try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.parts.models import Part - for p in ( - session.query(Part.part_number) - .filter( - Part.tenant_id == tenant_id, - Part.company_id == company_id, - ) - .all() - ): - valid_part_numbers.add(p[0]) - except Exception as e: - logger.warning(f"BOMs import: could not load parts for FK validation: {e}") + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + valid_part_numbers = load_boms_fk_sets(tenant_id, company_id) error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv.iter_csv_rows(file_path): if i % 500 == 0: self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"BOMs import scan failed: {e}") + logger.error("BOMs import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=BOM_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"BOMs import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _decimal_or_none(val: Any) -> Optional[Decimal]: - if val is None or val == "": - return None - try: - return Decimal(str(val)) - except (InvalidOperation, ValueError, TypeError): - return None - - -def _int_or_none(val: Any) -> Optional[int]: - if val is None or val == "": - return None - try: - return int(val) - except (ValueError, TypeError): - return None + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail + ) @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): - logger.info(f"BOMs import: starting commit for job {job_id}") + logger.info("BOMs import: starting commit for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"BOMs import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_part_numbers: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.parts.models import Part - for p in ( - session.query(Part.part_number) - .filter( - Part.tenant_id == tenant_id, - Part.company_id == company_id, - ) - .all() - ): - valid_part_numbers.add(p[0]) - except Exception as e: - logger.warning(f"BOMs import: could not load parts: {e}") + valid_part_numbers = load_boms_fk_sets(tenant_id, company_id) inserted_count = 0 skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] valid_count = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + meta_path = common_meta.get_meta_path(file_path) try: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + for i, row in common_csv.iter_csv_rows(file_path): + if i in error_lines: + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - row_norm = row_from_template(row, normalize_header) - err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue + valid_count += 1 + # Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas. - valid_count += 1 - # Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas. - # Cuando exista la tabla de destino, aquí se hará insert/update. - - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_duplicate": 0, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "finished", inserted_count, skipped_invalid, 0, 0, skipped_details, + ) if valid_count > 0 and inserted_count == 0: - response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada." - + response["message"] = ( + f"WIP: {valid_count} filas válidas. " + "La tabla BOM aún no existe en el sistema; no se insertó nada." + ) except Exception as e: - logger.error(f"BOMs import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - response = { - "status": "failed", - "error": str(e), - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_duplicate": 0, - "skipped_details": skipped_details, - } + logger.exception("BOMs import task failed") + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e), + ) - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"BOMs import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + if response is None: + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado", + ) return response diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py new file mode 100644 index 00000000..43a40a2d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_bom + +__all__ = ["validate_row_bom"] diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py new file mode 100644 index 00000000..739690ac --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/validators/common.py @@ -0,0 +1,54 @@ +""" +Validaciones comunes de fila para import CSV de BOMs. +""" +from decimal import Decimal +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + check_max_length, + check_decimal_required_min, + check_optional_number, + check_in_set, +) + + +def validate_row_required_parent(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "NUMPARTE_PADRE", 70, line_num, required=True) + + +def validate_row_required_component(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "NUMPARTE_COMPONENTE", 70, line_num, required=True) + + +def validate_row_quantity(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_decimal_required_min(row, "CANTIDAD", line_num, Decimal("0")) + + +def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "UNIMED", 10, line_num) + + +def validate_row_optional_numbers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + err = check_optional_number(row, "VERSION_BOM", line_num) + if err: + return err + return check_optional_number(row, "VERSION_BILL", line_num) + + +def validate_row_fks( + row: Dict[str, Any], + line_num: int, + valid_part_numbers: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + if not valid_part_numbers or len(valid_part_numbers) == 0: + return None + err = check_in_set( + row, "NUMPARTE_PADRE", line_num, + valid_part_numbers, "Parte padre no existe en catálogo", + ) + if err: + return err + return check_in_set( + row, "NUMPARTE_COMPONENTE", line_num, + valid_part_numbers, "Parte componente no existe en catálogo", + ) diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py new file mode 100644 index 00000000..cad75aa0 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/boms/validators/create.py @@ -0,0 +1,43 @@ +""" +Punto de entrada de validación para import de una fila BOM. +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_required_parent, + validate_row_required_component, + validate_row_quantity, + validate_row_lengths, + validate_row_optional_numbers, + validate_row_fks, +) + + +def validate_row_bom( + row: Dict[str, Any], + line_num: int, + valid_part_numbers: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de BOMs. + Encadena: requeridos (padre, componente, cantidad) → longitudes → opcionales numéricos → FKs. + """ + err = validate_row_required_parent(row, line_num) + if err: + return err + err = validate_row_required_component(row, line_num) + if err: + return err + err = validate_row_quantity(row, line_num) + if err: + return err + err = validate_row_lengths(row, line_num) + if err: + return err + err = validate_row_optional_numbers(row, line_num) + if err: + return err + err = validate_row_fks(row, line_num, valid_part_numbers) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py new file mode 100644 index 00000000..e7e4b5e4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/__init__.py @@ -0,0 +1 @@ +# common validators, mappers, fk_loader for classes CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py new file mode 100644 index 00000000..9a35fd86 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/common_validators.py @@ -0,0 +1,64 @@ +""" +Helpers reutilizables para validación de filas CSV (clases de materiales). +""" +from typing import Dict, Any, Optional + + +def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_int_range( + row: Dict[str, Any], + col: str, + line_num: int, + min_val: int, + max_val: int, +) -> Optional[Dict[str, Any]]: + """Solo valida si hay valor; devuelve error si no es int o está fuera de rango.""" + val = row.get(col) + if val is None or val == "": + return None + try: + v = int(val) + if v < min_val or v > max_val: + return {"line": line_num, "col": col, "msg": "Valor fuera de rango"} + except (ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser número entero"} + return None + + +def check_in_set( + row: Dict[str, Any], + col: str, + line_num: int, + allowed: Optional[set], + msg: str = "No existe en el catálogo", +) -> Optional[Dict[str, Any]]: + """Solo valida si hay valor y allowed no es None.""" + val = (row.get(col) or "").strip() + if not val or allowed is None: + return None + if val not in allowed: + return {"line": line_num, "col": col, "msg": msg} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py new file mode 100644 index 00000000..de4a8f39 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/fk_loader.py @@ -0,0 +1,37 @@ +""" +Carga de conjuntos FK para validación/mapeo de import CSV de clases de materiales. +""" +from typing import Set, Tuple + +from core.database import CoreSessionLocal + + +def load_classes_fk_sets( + tenant_id: int, + company_id: int, +) -> Tuple[Set[str], Set[str]]: + """ + Carga valid_material_keys (MaterialType.key) y valid_uom_codes (UnitOfMeasure.code). + Devuelve (valid_material_keys, valid_uom_codes). + """ + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + import logging + logging.getLogger(__name__).warning("Classes import: could not load FK sets: %s", e) + return valid_material_keys, valid_uom_codes diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py new file mode 100644 index 00000000..5bef32eb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/common/mappers.py @@ -0,0 +1,62 @@ +""" +Mapeo fila CSV → datos para Class (clases de materiales). +""" +from typing import Dict, Any, Optional, Set + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +def row_to_class_data( + row_norm: Dict[str, Any], + valid_material_keys: Set[str], + valid_uom_codes: Set[str], +) -> Dict[str, Any]: + """ + Mapea una fila normalizada del CSV a un diccionario de datos para Class. + Ajusta material_key y unit_of_measure a None si no están en los conjuntos. + """ + class_code = _str_or_none(row_norm.get("CLASE"), 8) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) + if material_key and material_key not in valid_material_keys: + material_key = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + fraction = _str_or_none(row_norm.get("FRACCION"), 20) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) + physical_review = _int_or_none(row_norm.get("REVFISICA")) + iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) + + return { + "class_code": class_code, + "description_es": desc_es, + "description_en": desc_en, + "material_key": material_key, + "unit_of_measure": unit_of_measure, + "fraction": fraction, + "us_fraction": us_fraction, + "sub_key": sub_key, + "physical_review": physical_review, + "iva_exempt_fraction": iva_exempt_fraction, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py b/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py index 1a230911..ba93da56 100644 --- a/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py @@ -1,528 +1,237 @@ """ Tareas Celery para importación CSV de Clases de Materiales. Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers. """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List, Set +import os +from typing import Dict, Any, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import csv_reader as common_csv +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template +from .validators import validate_row_class +from .common.mappers import row_to_class_data +from .common.fk_loader import load_classes_fk_sets logger = logging.getLogger(__name__) +JOB_TYPE = "cls" + +# Para routes.py CLS_IMPORT_FILE_PREFIX = "cls_import_file:" CLS_IMPORT_META_PREFIX = "cls_import_meta:" CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" -CLS_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Classes import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"cls_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Classes import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{CLS_IMPORT_FILE_PREFIX}{job_id}", - f"{CLS_IMPORT_META_PREFIX}{job_id}", - f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Classes import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_class( - row: Dict[str, Any], - line_num: int, - valid_material_keys: Optional[Set[str]] = None, - valid_uom_codes: Optional[Set[str]] = None, -) -> Optional[Dict[str, Any]]: - class_code = (row.get("CLASE") or "").strip() - if not class_code: - return {"line": line_num, "col": "CLASE", "msg": "Requerido"} - if len(class_code) > 8: - return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} - - desc_es = (row.get("DESCRIPCIONE") or "").strip() - if desc_es and len(desc_es) > 500: - return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} - desc_en = (row.get("DESCRIPCIONI") or "").strip() - if desc_en and len(desc_en) > 500: - return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} - - material_key = (row.get("CLAVEMAT") or "").strip() - if material_key: - if len(material_key) > 10: - return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"} - if valid_material_keys is not None and material_key not in valid_material_keys: - return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"} - - uom = (row.get("UNIMED") or "").strip() - if uom: - if len(uom) > 5: - return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} - if valid_uom_codes is not None and uom not in valid_uom_codes: - return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"} - - fraction = (row.get("FRACCION") or "").strip() - if fraction and len(fraction) > 20: - return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"} - us_fraction = (row.get("FRACCIONAME") or "").strip() - if us_fraction and len(us_fraction) > 16: - return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} - sub_key = (row.get("CLAVESUB") or "").strip() - if sub_key and len(sub_key) > 5: - return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"} - iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip() - if iva_exempt and len(iva_exempt) > 4: - return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"} - - rev_fisica = row.get("REVFISICA") - if rev_fisica is not None and rev_fisica != "": - try: - v = int(rev_fisica) - if v < -32768 or v > 32767: - return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"} - except (ValueError, TypeError): - return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"} - - return None +CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): - logger.info(f"Classes import: starting scan for job {job_id}") + logger.info("Classes import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Classes import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_material_keys: Set[str] = set() - valid_uom_codes: Set[str] = set() try: - with CoreSessionLocal() as session: - from api.v1.modules.public.reference_data.material_types.models import MaterialType - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - for m in session.query(MaterialType.key).all(): - valid_material_keys.add(m[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - except Exception as e: - logger.warning(f"Classes import: could not load FK sets: {e}") + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + valid_material_keys, valid_uom_codes = load_classes_fk_sets(tenant_id, company_id) error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv.iter_csv_rows(file_path): if i % 500 == 0: self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_class( - row_norm, i, + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_class( + row_norm, + i, valid_material_keys=valid_material_keys, valid_uom_codes=valid_uom_codes, ) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"Classes import scan failed: {e}") + logger.error("Classes import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=CLS_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Classes import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _int_or_none(val: Any) -> Optional[int]: - if val is None or val == "": - return None - try: - return int(val) - except (ValueError, TypeError): - return None + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail + ) @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): - logger.info(f"Classes import: starting commit for job {job_id}") + logger.info("Classes import: starting commit for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Classes import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.classes.models import Class - valid_material_keys: Set[str] = set() - valid_uom_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.public.reference_data.material_types.models import MaterialType - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - for m in session.query(MaterialType.key).all(): - valid_material_keys.add(m[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - except Exception as e: - logger.warning(f"Classes import: could not load FK sets: {e}") + valid_material_keys, valid_uom_codes = load_classes_fk_sets(tenant_id, company_id) inserted_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: - existing_by_code: Dict[str, Class] = {} - for c in ( - session.query(Class) - .filter( + existing_by_code = { + c.class_code: c + for c in session.query(Class).filter( Class.tenant_id == tenant_id, Class.company_id == company_id, + ).all() + } + + for i, row in common_csv.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_class( + row_norm, + i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, ) - .all() - ): - existing_by_code[c.class_code] = c + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes) + class_code = data.get("class_code") + if not class_code: + skipped_invalid += 1 + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_class( - row_norm, i, - valid_material_keys=valid_material_keys, - valid_uom_codes=valid_uom_codes, + existing = existing_by_code.get(class_code) + if existing: + existing.description_es = data["description_es"] + existing.description_en = data["description_en"] + existing.material_key = data["material_key"] + existing.unit_of_measure = data["unit_of_measure"] + existing.fraction = data["fraction"] + existing.us_fraction = data["us_fraction"] + existing.sub_key = data["sub_key"] + existing.physical_review = data["physical_review"] + existing.iva_exempt_fraction = data["iva_exempt_fraction"] + session.add(existing) + inserted_count += 1 + else: + new_class = Class( + tenant_id=tenant_id, + company_id=company_id, + **data, ) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue - - class_code = _str_or_none(row_norm.get("CLASE"), 8) - if not class_code: - skipped_invalid += 1 - continue - - existing = existing_by_code.get(class_code) - desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) - desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) - material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) - if material_key and material_key not in valid_material_keys: - material_key = None - unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) - if unit_of_measure and unit_of_measure not in valid_uom_codes: - unit_of_measure = None - fraction = _str_or_none(row_norm.get("FRACCION"), 20) - us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) - sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) - physical_review = _int_or_none(row_norm.get("REVFISICA")) - iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) - - if existing: - existing.description_es = desc_es - existing.description_en = desc_en - existing.material_key = material_key - existing.unit_of_measure = unit_of_measure - existing.fraction = fraction - existing.us_fraction = us_fraction - existing.sub_key = sub_key - existing.physical_review = physical_review - existing.iva_exempt_fraction = iva_exempt_fraction - session.add(existing) - inserted_count += 1 - else: - new_class = Class( - tenant_id=tenant_id, - company_id=company_id, - class_code=class_code, - description_es=desc_es, - description_en=desc_en, - material_key=material_key, - unit_of_measure=unit_of_measure, - fraction=fraction, - us_fraction=us_fraction, - sub_key=sub_key, - physical_review=physical_review, - iva_exempt_fraction=iva_exempt_fraction, - ) - session.add(new_class) - existing_by_code[class_code] = new_class - inserted_count += 1 + session.add(new_class) + existing_by_code[class_code] = new_class + inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"Classes import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} + logger.error("Classes import DB error: %s", db_err) + return common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, error=str(db_err), + ) - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } + if inserted_count == 0 and skipped_invalid > 0: + response = common_responses.commit_result( + "warning", 0, skipped_invalid, 0, 0, + skipped_details, + message=f"No se insertaron registros. {skipped_invalid} rechazados.", + ) elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, + error="No hay registros válidos en el archivo CSV", + ) else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "finished", inserted_count, skipped_invalid, 0, 0, + skipped_details, + ) except Exception as e: - logger.error(f"Classes import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} + logger.exception("Classes import task failed") + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, error=str(e), + ) - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Classes import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "failed", 0, skipped_invalid, 0, 0, + skipped_details, error="Error inesperado", + ) return response diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py new file mode 100644 index 00000000..83e82ef1 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_class + +__all__ = ["validate_row_class"] diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py new file mode 100644 index 00000000..3bbc8cd9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/validators/common.py @@ -0,0 +1,55 @@ +""" +Validaciones comunes de fila para import CSV de clases de materiales. +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + check_max_length, + check_int_range, + check_in_set, +) + + +def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_max_length(row, "CLASE", 8, line_num, required=True) + + +def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("DESCRIPCIONE", 500), + ("DESCRIPCIONI", 500), + ("CLAVEMAT", 10), + ("UNIMED", 5), + ("FRACCION", 20), + ("FRACCIONAME", 16), + ("CLAVESUB", 5), + ("FRACCIONEXENTAIVA", 4), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_int_range(row, "REVFISICA", line_num, -32768, 32767) + + +def validate_row_fks( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]], + valid_uom_codes: Optional[Set[str]], +) -> Optional[Dict[str, Any]]: + err = check_in_set( + row, "CLAVEMAT", line_num, valid_material_keys, "Tipo de material no existe" + ) + if err: + return err + err = check_in_set( + row, "UNIMED", line_num, valid_uom_codes, "Unidad de medida no existe" + ) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py new file mode 100644 index 00000000..50c6b922 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/classes/validators/create.py @@ -0,0 +1,40 @@ +""" +Punto de entrada de validación para import de una fila de clase de material. +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_required, + validate_row_lengths, + validate_row_types, + validate_row_fks, +) + + +def validate_row_class( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de clases de materiales. + Encadena: requeridos → longitudes → tipos → FKs. + """ + err = validate_row_required(row, line_num) + if err: + return err + + err = validate_row_lengths(row, line_num) + if err: + return err + + err = validate_row_types(row, line_num) + if err: + return err + + err = validate_row_fks(row, line_num, valid_material_keys, valid_uom_codes) + if err: + return err + + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py new file mode 100644 index 00000000..36641126 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for clients_and_providers) diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py new file mode 100644 index 00000000..801c24ab --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/common_validators.py @@ -0,0 +1,66 @@ +""" +Validadores reutilizables para import CSV de clientes y proveedores. +""" +from typing import Dict, Any, Optional + +from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum + +RFC_MAX = 30 +NAME_MAX = 256 +SHORT_NAME_MAX = 10 +CURP_MAX = 19 + + +def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: + if not val or not str(val).strip(): + return None + v = str(val).strip().lower() + if v in ("client", "cliente", "c"): + return ClientOrProviderEnum.CLIENT + if v in ("provider", "proveedor", "p"): + return ClientOrProviderEnum.PROVIDER + if v in ("both", "ambos", "b", "cliente y proveedor"): + return ClientOrProviderEnum.BOTH + return None + + +def check_tipo_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + tipo_raw = (row.get("TIPO") or "").strip() + if not tipo_raw: + return None + if parse_client_or_provider(tipo_raw) is None: + return { + "line": line_num, + "col": "TIPO", + "msg": "Valor no válido. Use Cliente, Proveedor o Ambos.", + } + return None + + +def parse_active(val: Optional[str]) -> bool: + if not val or not str(val).strip(): + return True + v = str(val).strip().lower() + if v in ("1", "true", "si", "sí", "yes", "s", "x"): + return True + if v in ("0", "false", "no", "n"): + return False + return True diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py new file mode 100644 index 00000000..271aa04c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/common/mappers.py @@ -0,0 +1,87 @@ +""" +Mapeo fila CSV → datos para ClientProvider y opcional ClientProviderAddress. +""" +from typing import Dict, Any, Optional, Tuple + +from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum + +from .common_validators import ( + parse_client_or_provider, + parse_active, +) + +MAX_LEN = { + "rfc": 30, + "name": 256, + "short_name": 10, + "curp": 19, + "responsible": 80, + "position": 30, + "incoterm": 19, + "email": 100, + "phone": 30, + "address": 100, + "postal_code": 15, + "city": 30, + "state": 30, + "country": 3, + "contact": 50, +} + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_client_provider_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int +) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]: + """ + Mapea fila normalizada a datos para ClientProvider y opcional ClientProviderAddress. + Devuelve (cp_data, address_data_or_none). address_data es para crear después del flush (necesita client_id). + """ + rfc = _str_or_none(row_norm.get("RFC"), MAX_LEN["rfc"]) + if not rfc: + return ({}, None) + + client_or_provider = parse_client_or_provider(row_norm.get("TIPO")) or ClientOrProviderEnum.BOTH + + cp_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "rfc": rfc, + "name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]), + "short_name": _str_or_none(row_norm.get("SHORT_NAME"), MAX_LEN["short_name"]), + "curp": _str_or_none(row_norm.get("CURP"), MAX_LEN["curp"]), + "client_or_provider": client_or_provider, + "responsible": _str_or_none(row_norm.get("RESPONSABLE"), MAX_LEN["responsible"]), + "position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]), + "incoterm": _str_or_none(row_norm.get("INCOTERM"), MAX_LEN["incoterm"]), + "is_active": parse_active(row_norm.get("ACTIVO")), + } + + email = _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]) + phone = _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]) + address_str = _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]) + if email or phone or address_str: + address_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "streets": address_str, + "postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "phone": phone, + "email": email, + "contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]), + } + return (cp_data, address_data) + return (cp_data, None) diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py index a133918c..521244b3 100644 --- a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py @@ -1,340 +1,128 @@ """ Tareas Celery para importación CSV de Clientes y Proveedores. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). """ -import os -import base64 -import csv import json import logging -import re -import unicodedata +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader from .template_config import row_from_template -from api.v1.modules.a76.clients_and_providers.models import ( - ClientProvider, - ClientProviderAddress, - ClientOrProviderEnum, -) +from .validators import validate_row_client_provider +from .common.mappers import row_to_client_provider_data logger = logging.getLogger(__name__) -# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports) +JOB_TYPE = "cp" + +# Para routes.py CP_IMPORT_FILE_PREFIX = "cp_import_file:" CP_IMPORT_META_PREFIX = "cp_import_meta:" CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:" -CP_IMPORT_REDIS_TTL = 3600 # 1 hour +CP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"CP import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"cp_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"CP import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{CP_IMPORT_FILE_PREFIX}{job_id}", - f"{CP_IMPORT_META_PREFIX}{job_id}", - f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"CP import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: - """Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido.""" - if not val or not str(val).strip(): - return None - v = str(val).strip().lower() - if v in ("client", "cliente", "c"): - return ClientOrProviderEnum.CLIENT - if v in ("provider", "proveedor", "p"): - return ClientOrProviderEnum.PROVIDER - if v in ("both", "ambos", "b", "cliente y proveedor"): - return ClientOrProviderEnum.BOTH - return None - - -def _parse_active(val: Optional[str]) -> bool: - """Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True.""" - if not val or not str(val).strip(): - return True - v = str(val).strip().lower() - if v in ("1", "true", "si", "sí", "yes", "s", "x"): - return True - if v in ("0", "false", "no", "n"): - return False - return True - - -def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Cliente/Proveedor. Retorna error dict o None.""" - rfc = (row.get("RFC") or "").strip() - if not rfc: - return {"line": line_num, "col": "RFC", "msg": "Requerido"} - if len(rfc) > 30: - return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"} - - tipo_raw = (row.get("TIPO") or "").strip() - if tipo_raw and _parse_client_or_provider(tipo_raw) is None: - return { - "line": line_num, - "col": "TIPO", - "msg": "Valor no válido. Use Cliente, Proveedor o Ambos.", - } - - name = (row.get("NOMBRE") or "").strip() - if len(name) > 256: - return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"} - - short_name = (row.get("SHORT_NAME") or "").strip() - if short_name and len(short_name) > 10: - return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"} - - curp = (row.get("CURP") or "").strip() - if curp and len(curp) > 19: - return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"CP import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"CP import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_client_provider(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_client_provider(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"CP import scan failed: {e}") + logger.error("CP import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=CP_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"CP import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider. - Upsert por (tenant_id, company_id, rfc). - """ - logger.info(f"CP import: starting commit for job {job_id}") +def scan_file(self, job_id: str, config: str = None): + logger.info("CP import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"CP import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + from api.v1.modules.a76.clients_and_providers.models import ( + ClientProvider, + ClientProviderAddress, + ) inserted_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] - response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: - # Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key existing_by_rfc: Dict[str, ClientProvider] = {} for cp in ( session.query(ClientProvider) @@ -347,154 +135,108 @@ def insert_valid_rows(self, job_id: str): key = (cp.rfc or "").strip() existing_by_rfc[key] = cp - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_client_provider(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - row_norm = row_from_template(row, normalize_header) - err = _validate_row_client_provider(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue + cp_data, address_data = row_to_client_provider_data(row_norm, tenant_id, company_id) + if not cp_data or not cp_data.get("rfc"): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "RFC requerido"}) + continue - rfc = _str_or_none(row_norm.get("RFC"), 30) - if not rfc: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "RFC requerido"}) - continue - - client_or_provider = _parse_client_or_provider(row_norm.get("TIPO")) - if client_or_provider is None: - client_or_provider = ClientOrProviderEnum.BOTH - - is_active = _parse_active(row_norm.get("ACTIVO")) - - existing = existing_by_rfc.get(rfc) - if existing: - existing.name = _str_or_none(row_norm.get("NOMBRE"), 256) - existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10) - existing.curp = _str_or_none(row_norm.get("CURP"), 19) - existing.client_or_provider = client_or_provider - existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80) - existing.position = _str_or_none(row_norm.get("POSICION"), 30) - existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19) - existing.is_active = is_active - session.add(existing) - inserted_count += 1 - else: - new_cp = ClientProvider( + rfc = cp_data["rfc"] + existing = existing_by_rfc.get(rfc) + if existing: + for k, v in cp_data.items(): + if k not in ("tenant_id", "company_id", "rfc"): + setattr(existing, k, v) + session.add(existing) + inserted_count += 1 + else: + new_cp = ClientProvider(**cp_data) + session.add(new_cp) + session.flush() + existing_by_rfc[rfc] = new_cp + inserted_count += 1 + if address_data: + addr = ClientProviderAddress( + client_id=new_cp.id, tenant_id=tenant_id, company_id=company_id, - rfc=rfc, - name=_str_or_none(row_norm.get("NOMBRE"), 256), - short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10), - curp=_str_or_none(row_norm.get("CURP"), 19), - client_or_provider=client_or_provider, - responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80), - position=_str_or_none(row_norm.get("POSICION"), 30), - incoterm=_str_or_none(row_norm.get("INCOTERM"), 19), - is_active=is_active, + streets=address_data.get("streets"), + postal_code=address_data.get("postal_code"), + city=address_data.get("city"), + state=address_data.get("state"), + country=address_data.get("country"), + phone=address_data.get("phone"), + email=address_data.get("email"), + contact=address_data.get("contact"), ) - session.add(new_cp) - session.flush() - existing_by_rfc[rfc] = new_cp - inserted_count += 1 - - # Opcional: crear dirección si hay email/teléfono/dirección - email = _str_or_none(row_norm.get("EMAIL"), 100) - phone = _str_or_none(row_norm.get("TELEFONO"), 30) - address_str = _str_or_none(row_norm.get("DIRECCION"), 100) - if email or phone or address_str: - addr = ClientProviderAddress( - client_id=new_cp.id, - tenant_id=tenant_id, - company_id=company_id, - streets=address_str, - postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), - city=_str_or_none(row_norm.get("CIUDAD"), 30), - state=_str_or_none(row_norm.get("ESTADO"), 30), - country=_str_or_none(row_norm.get("PAIS"), 3), - phone=phone, - email=email, - contact=_str_or_none(row_norm.get("CONTACTO"), 50), - ) - session.add(addr) + session.add(addr) try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"CP import DB error: {db_err}") + logger.error("CP import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - except Exception as e: - logger.error(f"CP import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("CP import task failed") return {"status": "failed", "error": str(e)} - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"CP import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", + if inserted_count == 0 and skipped_invalid > 0: + return { + "status": "warning", "inserted": 0, + "updated": 0, "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, "skipped_missing_fk": 0, "skipped_details": skipped_details, } - return response + return { + "status": "finished", + "inserted": inserted_count, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("CP import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py new file mode 100644 index 00000000..895b3327 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_client_provider + +__all__ = ["validate_row_client_provider"] diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py new file mode 100644 index 00000000..91c8c3dd --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/common.py @@ -0,0 +1,52 @@ +""" +Validaciones comunes de fila para import CSV de clientes y proveedores. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + RFC_MAX, + NAME_MAX, + SHORT_NAME_MAX, + CURP_MAX, + check_required_max, + check_max_length, + check_tipo_client_provider, +) + + +def validate_row_client_provider_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_required_max(row, "RFC", RFC_MAX, line_num) + + +def validate_row_client_provider_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + err = check_max_length(row, "NOMBRE", NAME_MAX, line_num) + if err: + return err + err = check_max_length(row, "SHORT_NAME", SHORT_NAME_MAX, line_num) + if err: + return err + err = check_max_length(row, "CURP", CURP_MAX, line_num) + if err: + return err + return None + + +def validate_row_client_provider_tipo(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_tipo_client_provider(row, line_num) + + +def validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de clientes y proveedores. + RFC requerido (max 30); TIPO opcional pero debe ser Cliente/Proveedor/Ambos; NOMBRE/SHORT_NAME/CURP longitudes. + """ + err = validate_row_client_provider_required(row, line_num) + if err: + return err + err = validate_row_client_provider_tipo(row, line_num) + if err: + return err + err = validate_row_client_provider_lengths(row, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py new file mode 100644 index 00000000..734e7f8b --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/validators/create.py @@ -0,0 +1,8 @@ +""" +Punto de entrada de validación para import de una fila cliente/proveedor. +""" +from typing import Dict, Any, Optional + +from .common import validate_row_client_provider + +__all__ = ["validate_row_client_provider"] diff --git a/backend/api/v1/modules/a76/layouts_csv/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/common/__init__.py new file mode 100644 index 00000000..d5f02bc1 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/__init__.py @@ -0,0 +1 @@ +# Shared utilities for layouts_csv imports (storage, normalize, csv, meta, responses) diff --git a/backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py b/backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py new file mode 100644 index 00000000..3db621db --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py @@ -0,0 +1,28 @@ +""" +Lectura de CSV con detección de delimitador (compartida por layouts_csv). +""" +import csv +from typing import Iterator, Tuple, Dict, Any + + +def iter_csv_rows(file_path: str) -> Iterator[Tuple[int, Dict[str, Any]]]: + """ + Abre el CSV, detecta dialecto y devuelve (line_num, row_dict) por cada fila. + line_num empieza en 1 (primera fila de datos). + """ + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + for i, row in enumerate(reader, start=1): + yield i, row + + +def count_csv_rows(file_path: str) -> int: + """Cuenta filas del CSV (sin contar cabecera).""" + with open(file_path, "r", encoding="utf-8-sig") as f: + return sum(1 for _ in f) - 1 diff --git a/backend/api/v1/modules/a76/layouts_csv/common/meta.py b/backend/api/v1/modules/a76/layouts_csv/common/meta.py new file mode 100644 index 00000000..fc68d96f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/meta.py @@ -0,0 +1,35 @@ +""" +Carga y guardado de meta (tenant_id, company_id) para imports CSV. +""" +import json +import os +from typing import Dict, Any, Tuple + + +def load_meta(file_path: str) -> Dict[str, Any]: + """Carga meta desde archivo .meta.json asociado al CSV. Devuelve dict vacío si no existe o falla.""" + meta_path = file_path.replace(".csv", ".meta.json") + if not os.path.exists(meta_path): + return {} + try: + with open(meta_path, "r", encoding="utf-8") as f: + return json.load(f) or {} + except Exception: + return {} + + +def require_tenant_context(file_path: str) -> Tuple[int, int]: + """ + Obtiene tenant_id y company_id del meta. Lanza ValueError si faltan. + """ + meta = load_meta(file_path) + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + raise ValueError("Falta contexto (tenant/company)") + return int(tenant_id), int(company_id) + + +def get_meta_path(file_path: str) -> str: + """Ruta del archivo .meta.json para un CSV.""" + return file_path.replace(".csv", ".meta.json") diff --git a/backend/api/v1/modules/a76/layouts_csv/common/normalize.py b/backend/api/v1/modules/a76/layouts_csv/common/normalize.py new file mode 100644 index 00000000..6af81dbb --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/normalize.py @@ -0,0 +1,16 @@ +""" +Normalización de cabeceras CSV (compartida por todos los módulos layouts_csv). +""" +import re +import unicodedata +from typing import Optional + + +def normalize_header(name: Optional[str]) -> str: + """Normaliza nombre de columna: NFKD, mayúsculas, sin acentos, espacios colapsados.""" + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() diff --git a/backend/api/v1/modules/a76/layouts_csv/common/responses.py b/backend/api/v1/modules/a76/layouts_csv/common/responses.py new file mode 100644 index 00000000..f122c9cf --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/responses.py @@ -0,0 +1,47 @@ +""" +Helpers para construir respuestas de scan y commit (formato unificado). +""" +from typing import Dict, Any, List, Optional + + +def scan_result( + job_id: str, + processed_rows: int, + error_count: int, + errors_detail: List[Dict[str, Any]], +) -> Dict[str, Any]: + """Respuesta de scan_file (waiting_confirmation).""" + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def commit_result( + status: str, + inserted: int, + skipped_invalid: int, + skipped_missing_fk: int, + skipped_duplicate: int, + skipped_details: List[Dict[str, Any]], + message: Optional[str] = None, + error: Optional[str] = None, +) -> Dict[str, Any]: + """Respuesta de insert_valid_rows (finished / warning / failed).""" + out = { + "status": status, + "inserted": inserted, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + if message: + out["message"] = message + if error: + out["error"] = error + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/common/storage.py b/backend/api/v1/modules/a76/layouts_csv/common/storage.py new file mode 100644 index 00000000..f2875d5c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/common/storage.py @@ -0,0 +1,171 @@ +""" +Redis y rutas de archivos para imports CSV (compartido por módulos layouts_csv). +Cada módulo usa un job_type (ej. "part", "cls", "bom") para prefijos y nombres de archivo. +""" +import os +import base64 +import json +import logging +from typing import Optional, Set, List + +from core.paths import layout_path + +logger = logging.getLogger(__name__) + +IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def upload_dir() -> str: + """Directorio temporal para CSV en el worker.""" + return layout_path("imports", "temp") + + +def error_dir() -> str: + """Directorio de archivos JSONL de errores.""" + return layout_path("imports", "errors") + + +def storage_keys(job_type: str, job_id: str) -> tuple: + """Prefijos Redis para un job_type y job_id. Devuelve (file_key, meta_key, error_lines_key).""" + # Facturas usa prefijo "import_" sin tipo (compatibilidad con rutas existentes) + if job_type == "" or job_type == "invoice": + prefix = "import_" + else: + prefix = f"{job_type}_import_" + return ( + f"{prefix}file:{job_id}", + f"{prefix}meta:{job_id}", + f"{prefix}error_lines:{job_id}", + ) + + +def file_path_for_job(job_type: str, job_id: str) -> str: + """Ruta local del archivo CSV para un job.""" + if job_type == "" or job_type == "invoice": + return os.path.join(upload_dir(), f"{job_id}.csv") + return os.path.join(upload_dir(), f"{job_type}_{job_id}.csv") + + +def error_path_for_job(job_type: str, job_id: str) -> str: + """Ruta del archivo JSONL de errores para un job.""" + os.makedirs(error_dir(), exist_ok=True) + if job_type == "" or job_type == "invoice": + return os.path.join(error_dir(), f"{job_id}.jsonl") + return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl") + + +def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]: + """ + Descarga contenido del CSV desde Redis y lo escribe en disco. + Devuelve la ruta del archivo o None si no hay datos o falla. + """ + file_key, _, _ = storage_keys(job_type, job_id) + r = _get_redis() + data = r.get(file_key) + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e) + return None + path = file_path_for_job(job_type, job_id) + os.makedirs(upload_dir(), exist_ok=True) + with open(path, "wb") as f: + f.write(raw) + return path + + +def ensure_meta_from_redis(job_type: str, job_id: str, file_path: str, log_prefix: str = "") -> bool: + """Descarga meta desde Redis y la escribe en .meta.json. Devuelve True si hubo datos.""" + _, meta_key, _ = storage_keys(job_type, job_id) + r = _get_redis() + data = r.get(meta_key) + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning("%s failed to decode meta from Redis: %s", log_prefix or job_type, e) + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def store_error_lines(job_type: str, job_id: str, line_numbers: List[int]) -> None: + """Guarda la lista de números de línea con error en Redis.""" + _, _, error_key = storage_keys(job_type, job_id) + try: + r = _get_redis() + r.set(error_key, json.dumps(line_numbers).encode("utf-8"), ex=IMPORT_REDIS_TTL) + except Exception as e: + logger.warning("Failed to store error lines in Redis: %s", e) + + +def get_error_lines(job_type: str, job_id: str, error_path: str) -> Set[int]: + """ + Obtiene el conjunto de líneas con error: primero desde Redis, si está vacío desde el JSONL. + """ + _, _, error_key = storage_keys(job_type, job_id) + lines = set() + try: + r = _get_redis() + raw = r.get(error_key) + if raw: + lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug("Could not load error lines from Redis: %s", e) + if not lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + lines.add(err["line"]) + except Exception: + pass + return lines + + +def delete_import_from_redis(job_type: str, job_id: str) -> None: + """Borra claves Redis del import (file, meta, error_lines).""" + file_key, meta_key, error_key = storage_keys(job_type, job_id) + try: + r = _get_redis() + r.delete(file_key, meta_key, error_key) + except Exception as e: + logger.warning("Failed to delete import keys from Redis: %s", e) + + +def cleanup_import_job( + job_type: str, + job_id: str, + file_path: Optional[str] = None, + error_path: Optional[str] = None, + meta_path: Optional[str] = None, +) -> None: + """Elimina archivos locales y claves Redis del job.""" + if file_path and os.path.exists(file_path): + try: + os.remove(file_path) + except Exception as e: + logger.warning("Cleanup: failed to remove file %s: %s", file_path, e) + if error_path and os.path.exists(error_path): + try: + os.remove(error_path) + except Exception as e: + logger.warning("Cleanup: failed to remove error file %s: %s", error_path, e) + if meta_path and os.path.exists(meta_path): + try: + os.remove(meta_path) + except Exception as e: + logger.warning("Cleanup: failed to remove meta %s: %s", meta_path, e) + delete_import_from_redis(job_type, job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py new file mode 100644 index 00000000..9b248794 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for customs_brokers) diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py new file mode 100644 index 00000000..1902dad2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/common_validators.py @@ -0,0 +1,28 @@ +""" +Validadores reutilizables para import CSV de agentes aduanales (clave, licencia). +""" +import re +from typing import Dict, Any, Optional + +BROKER_KEY_MAX = 5 +LICENSE_MAX = 4 + + +def check_required_broker_key(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + clave = (row.get("CLAVE") or "").strip() + if not clave: + return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} + if len(clave) > BROKER_KEY_MAX: + return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {BROKER_KEY_MAX} caracteres"} + if not re.match(r"^[a-zA-Z0-9]+$", clave): + return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"} + return None + + +def check_optional_license(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + licencia = (row.get("LICENCIA") or "").strip() + if not licencia: + return None + if len(licencia) > LICENSE_MAX or not licencia.isdigit(): + return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py new file mode 100644 index 00000000..5dee87dc --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/common/mappers.py @@ -0,0 +1,72 @@ +""" +Mapeo fila CSV → datos para CustomsBroker. +""" +from typing import Dict, Any, Optional + +MAX_LEN = { + "broker_key": 5, + "type": 9, + "name": 80, + "address": 1500, + "postal_code": 15, + "city": 30, + "state": 30, + "phone": 30, + "fax": 30, + "email": 100, + "country": 3, + "tax_id": 30, + "personal_id": 20, + "position": 30, + "license": 4, + "company": 200, + "contact": 80, +} + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _license_value(row_norm: Dict[str, Any]) -> Optional[str]: + lic = (row_norm.get("LICENCIA") or "").strip() + if not lic or not lic.isdigit(): + return None + return lic[:4] + + +def row_to_customs_broker_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int +) -> Dict[str, Any]: + """Build dict for CustomsBroker model (create or update).""" + clave = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["broker_key"]) + if not clave: + return {} + return { + "tenant_id": tenant_id, + "company_id": company_id, + "broker_key": clave, + "type": _str_or_none(row_norm.get("TIPO"), MAX_LEN["type"]), + "name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]), + "address": _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]), + "postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "phone": _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]), + "fax": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]), + "email": _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "tax_id": _str_or_none(row_norm.get("RFC"), MAX_LEN["tax_id"]), + "personal_id": _str_or_none(row_norm.get("PERSONAL_ID"), MAX_LEN["personal_id"]), + "position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]), + "license": _license_value(row_norm), + "company": _str_or_none(row_norm.get("EMPRESA"), MAX_LEN["company"]), + "contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]), + } diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py index 80a8df78..987bd736 100644 --- a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py @@ -1,293 +1,122 @@ """ Tareas Celery para importación CSV de Agentes Aduanales. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). """ -import os -import base64 -import csv import json import logging -import re -import unicodedata +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader from .template_config import row_from_template +from .validators import validate_row_customs_broker +from .common.mappers import row_to_customs_broker_data logger = logging.getLogger(__name__) -# Redis keys (prefijo propio para no colisionar con a76.imports) +JOB_TYPE = "cb" + +# Para routes.py CB_IMPORT_FILE_PREFIX = "cb_import_file:" CB_IMPORT_META_PREFIX = "cb_import_meta:" CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:" -CB_IMPORT_REDIS_TTL = 3600 # 1 hour +CB_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"CB import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"cb_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"CB import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{CB_IMPORT_FILE_PREFIX}{job_id}", - f"{CB_IMPORT_META_PREFIX}{job_id}", - f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"CB import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Agente Aduanal. Retorna error dict o None.""" - clave = (row.get("CLAVE") or "").strip() - if not clave: - return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} - if len(clave) > 5: - return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"} - if not re.match(r"^[a-zA-Z0-9]+$", clave): - return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"} - - licencia = (row.get("LICENCIA") or "").strip() - if licencia and (len(licencia) > 4 or not licencia.isdigit()): - return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"CB import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"CB import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_customs_broker(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_customs_broker(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"CB import scan failed: {e}") + logger.error("CB import scan failed: %s", e) return {"status": "failed", "error": str(e)} - # Guardar números de línea con error en Redis para insert_valid_rows - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=CB_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"CB import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker. - """ - logger.info(f"CB import: starting commit for job {job_id}") +def scan_file(self, job_id: str, config: str = None): + logger.info("CB import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"CB import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.customs_brokers.models import CustomsBroker inserted_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] - response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: @@ -302,146 +131,90 @@ def insert_valid_rows(self, job_id: str): ): existing_by_key[b.broker_key] = b - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_customs_broker(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - row_norm = row_from_template(row, normalize_header) - err = _validate_row_customs_broker(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue + data = row_to_customs_broker_data(row_norm, tenant_id, company_id) + if not data or not data.get("broker_key"): + skipped_invalid += 1 + continue - clave = (row_norm.get("CLAVE") or "").strip()[:5] - if not clave: - skipped_invalid += 1 - continue - - existing = existing_by_key.get(clave) - if existing: - existing.type = _str_or_none(row_norm.get("TIPO"), 9) - existing.name = _str_or_none(row_norm.get("NOMBRE"), 80) - existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500) - existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15) - existing.city = _str_or_none(row_norm.get("CIUDAD"), 30) - existing.state = _str_or_none(row_norm.get("ESTADO"), 30) - existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30) - existing.fax = _str_or_none(row_norm.get("FAX"), 30) - existing.email = _str_or_none(row_norm.get("EMAIL"), 100) - existing.country = _str_or_none(row_norm.get("PAIS"), 3) - existing.tax_id = _str_or_none(row_norm.get("RFC"), 30) - existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20) - existing.position = _str_or_none(row_norm.get("POSICION"), 30) - lic = (row_norm.get("LICENCIA") or "").strip() - existing.license = lic[:4] if lic and lic.isdigit() else None - existing.company = _str_or_none(row_norm.get("EMPRESA"), 200) - existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80) - session.add(existing) - inserted_count += 1 - else: - lic = (row_norm.get("LICENCIA") or "").strip() - license_val = lic[:4] if lic and lic.isdigit() else None - new_broker = CustomsBroker( - tenant_id=tenant_id, - company_id=company_id, - broker_key=clave, - type=_str_or_none(row_norm.get("TIPO"), 9), - name=_str_or_none(row_norm.get("NOMBRE"), 80), - address=_str_or_none(row_norm.get("DIRECCION"), 1500), - postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), - city=_str_or_none(row_norm.get("CIUDAD"), 30), - state=_str_or_none(row_norm.get("ESTADO"), 30), - phone=_str_or_none(row_norm.get("TELEFONO"), 30), - fax=_str_or_none(row_norm.get("FAX"), 30), - email=_str_or_none(row_norm.get("EMAIL"), 100), - country=_str_or_none(row_norm.get("PAIS"), 3), - tax_id=_str_or_none(row_norm.get("RFC"), 30), - personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20), - position=_str_or_none(row_norm.get("POSICION"), 30), - license=license_val, - company=_str_or_none(row_norm.get("EMPRESA"), 200), - contact=_str_or_none(row_norm.get("CONTACTO"), 80), - ) - session.add(new_broker) - existing_by_key[clave] = new_broker - inserted_count += 1 + clave = data["broker_key"] + existing = existing_by_key.get(clave) + if existing: + for k, v in data.items(): + if k not in ("tenant_id", "company_id", "broker_key"): + setattr(existing, k, v) + session.add(existing) + else: + new_broker = CustomsBroker(**data) + session.add(new_broker) + existing_by_key[clave] = new_broker + inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"CB import DB error: {db_err}") + logger.error("CB import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - except Exception as e: - logger.error(f"CB import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("CB import task failed") return {"status": "failed", "error": str(e)} - # Limpieza - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"CB import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", + if inserted_count == 0 and skipped_invalid > 0: + return { + "status": "warning", "inserted": 0, + "updated": 0, "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, "skipped_missing_fk": 0, "skipped_details": skipped_details, } - return response + return { + "status": "finished", + "inserted": inserted_count, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("CB import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py new file mode 100644 index 00000000..ecfaa325 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_customs_broker + +__all__ = ["validate_row_customs_broker"] diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py new file mode 100644 index 00000000..492414a1 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/common.py @@ -0,0 +1,23 @@ +""" +Validaciones comunes de fila para import CSV de agentes aduanales. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + check_required_broker_key, + check_optional_license, +) + + +def validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de agentes aduanales. + CLAVE requerida (max 5, alfanumérica); LICENCIA opcional (max 4 dígitos). + """ + err = check_required_broker_key(row, line_num) + if err: + return err + err = check_optional_license(row, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py new file mode 100644 index 00000000..6336bbb2 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/validators/create.py @@ -0,0 +1,6 @@ +""" +Punto de entrada de validación para import de una fila agente aduanal. +""" +from .common import validate_row_customs_broker + +__all__ = ["validate_row_customs_broker"] diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py new file mode 100644 index 00000000..f79c5802 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/__init__.py @@ -0,0 +1 @@ +# common validators, mappers for drivers CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py new file mode 100644 index 00000000..4ce6dcf9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/common_validators.py @@ -0,0 +1,118 @@ +""" +Helpers reutilizables para validación de filas CSV (conductores). +""" +import re +from typing import Dict, Any, Optional + + +MAX_LEN = { + "transporter_key": 5, + "driver_name": 80, + "license_number": 29, + "express_line_id": 17, + "ace_id": 20, + "gender": 1, + "birth_country": 3, + "hazardous_material_auth": 2, + "hazardous_material_state": 30, + "first_name": 20, + "last_name": 20, + "id_key1": 40, + "id_number1": 20, + "id_state1": 30, + "id_country1": 3, + "id_key2": 40, + "id_number2": 20, + "id_state2": 30, + "id_country2": 3, +} + + +def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"} + return None + + +def parse_int(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d+$", s): + return int(s) + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + return None + + +def check_int_positive(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + v = parse_int(row.get(col)) + if v is None: + return {"line": line_num, "col": col, "msg": "Debe ser numerico"} + if v <= 0: + return {"line": line_num, "col": col, "msg": "Debe ser mayor a 0"} + return None + + +def parse_birth_date(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + return None + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + return None + break + return None + + +def check_optional_birth_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or not str(val).strip(): + return None + if parse_birth_date(val) is None: + return { + "line": line_num, + "col": col, + "msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)", + } + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py new file mode 100644 index 00000000..50ef319a --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/common/mappers.py @@ -0,0 +1,62 @@ +""" +Mapeo fila CSV → datos para Driver (conductores). +""" +from typing import Dict, Any, Optional + +from .common_validators import ( + MAX_LEN, + parse_int, + parse_birth_date, +) + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_driver_data( + row_norm: Dict[str, Any], + tenant_id: int, + company_id: int, +) -> Dict[str, Any]: + """Mapea una fila normalizada del CSV a un diccionario para DriverCreateDTO.""" + transporter_key = _str_or_none(row_norm.get("TRANSPORTISTA"), MAX_LEN["transporter_key"]) + line = parse_int(row_norm.get("LINEA")) + if not transporter_key or line is None: + return {} + return { + "transporter_key": transporter_key, + "line": line, + "driver_name": _str_or_none(row_norm.get("CLAVE CONDUCTOR"), MAX_LEN["driver_name"]), + "license_number": _str_or_none(row_norm.get("LICENCIA"), MAX_LEN["license_number"]), + "express_line_id": _str_or_none(row_norm.get("PERMISO LINEA EXPRESS"), MAX_LEN["express_line_id"]), + "ace_id": _str_or_none(row_norm.get("IDENTIFICACION ACE"), MAX_LEN["ace_id"]), + "birth_date": parse_birth_date(row_norm.get("FECHA NACIMIENTO")), + "gender": _str_or_none(row_norm.get("SEXO"), MAX_LEN["gender"]), + "birth_country": _str_or_none(row_norm.get("PAIS NACIMIENTO"), MAX_LEN["birth_country"]), + "hazardous_material_auth": _str_or_none( + row_norm.get("TRANSPORTA MAT. PELIGROSO?"), MAX_LEN["hazardous_material_auth"] + ), + "hazardous_material_state": _str_or_none( + row_norm.get("PERMISO MAT. PELIGROSO"), MAX_LEN["hazardous_material_state"] + ), + "first_name": _str_or_none(row_norm.get("NOMBRE(S)"), MAX_LEN["first_name"]), + "last_name": _str_or_none(row_norm.get("APELLIDO PATERNO"), MAX_LEN["last_name"]), + "id_key1": _str_or_none(row_norm.get("FORMA IDENTIFICACION 1"), MAX_LEN["id_key1"]), + "id_number1": _str_or_none(row_norm.get("NUM. IDENTIFICACION 1"), MAX_LEN["id_number1"]), + "id_state1": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["id_state1"]), + "id_country1": _str_or_none(row_norm.get("PAIS"), MAX_LEN["id_country1"]), + "id_key2": _str_or_none(row_norm.get("FORMA IDENTIFICACION 2"), MAX_LEN["id_key2"]), + "id_number2": _str_or_none(row_norm.get("NUM. IDENTIFICACION 2"), MAX_LEN["id_number2"]), + "id_state2": _str_or_none(row_norm.get("ESTADO 2"), MAX_LEN["id_state2"]), + "id_country2": _str_or_none(row_norm.get("PAIS 2"), MAX_LEN["id_country2"]), + "company_id": company_id, + "tenant_id": tenant_id, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py index e0dfb784..83e1ee2e 100644 --- a/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py @@ -1,29 +1,35 @@ """ -Tareas Celery para importacion CSV de Conductores. -Flujo: scan_file (validacion) -> insert_valid_rows (commit). +Tareas Celery para importación CSV de Conductores. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe) y clave de estado en Redis. """ -import os -import base64 import csv import json import logging -import re -import unicodedata +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template +from .validators import validate_row_driver +from .common.mappers import row_to_driver_data logger = logging.getLogger(__name__) +JOB_TYPE = "drv" + +# Para routes.py DRV_IMPORT_FILE_PREFIX = "drv_import_file:" DRV_IMPORT_META_PREFIX = "drv_import_meta:" DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:" DRV_IMPORT_STATUS_PREFIX = "drv_import_status:" -DRV_IMPORT_REDIS_TTL = 3600 +DRV_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _get_redis(): @@ -32,147 +38,6 @@ def _get_redis(): return redis.Redis.from_url(url, decode_responses=False) -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Drivers import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"drv_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Drivers import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{DRV_IMPORT_FILE_PREFIX}{job_id}", - f"{DRV_IMPORT_META_PREFIX}{job_id}", - f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", - f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Drivers import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -_MAX = { - "transporter_key": 5, - "driver_name": 80, - "license_number": 29, - "express_line_id": 17, - "ace_id": 20, - "gender": 1, - "birth_country": 3, - "hazardous_material_auth": 2, - "hazardous_material_state": 30, - "first_name": 20, - "last_name": 20, - "id_key1": 40, - "id_number1": 20, - "id_state1": 30, - "id_country1": 3, - "id_key2": 40, - "id_number2": 20, - "id_state2": 30, - "id_country2": 3, - "badge_number": 20, - "class_type": 1, - "unique_badge_number": 100, -} - - -def _parse_int(val: Any) -> Optional[int]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - s = str(val).strip() - if re.match(r"^\d+$", s): - return int(s) - if re.match(r"^\d+\.0+$", s): - try: - return int(float(s)) - except ValueError: - return None - return None - - -def _parse_birth_date(val: Any) -> Optional[int]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - s = str(val).strip() - if re.match(r"^\d{8}$", s): - try: - return int(s) - except ValueError: - return None - if re.match(r"^\d+\.0+$", s): - try: - return int(float(s)) - except ValueError: - return None - for sep in ["/", "-", "."]: - if sep in s: - parts = s.split(sep) - if len(parts) == 3: - try: - a, b, c = [p.strip() for p in parts] - if len(c) == 4 and len(a) <= 2 and len(b) <= 2: - return int(c) * 10000 + int(b) * 100 + int(a) - if len(a) == 4 and len(b) <= 2 and len(c) <= 2: - return int(a) * 10000 + int(b) * 100 + int(c) - except (ValueError, TypeError): - return None - break - return None - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - def _dedupe_headers(headers: List[str]) -> List[str]: counts: Dict[str, int] = {} unique: List[str] = [] @@ -180,136 +45,33 @@ def _dedupe_headers(headers: List[str]) -> List[str]: name = str(header or "").strip() or "COL" count = counts.get(name, 0) + 1 counts[name] = count - if count == 1: - unique.append(name) - else: - unique.append(f"{name} {count}") + unique.append(name if count == 1 else f"{name} {count}") return unique -def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - transporter_key = (row.get("TRANSPORTISTA") or "").strip() - if not transporter_key: - return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"} - if len(transporter_key) > _MAX["transporter_key"]: - return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"} - - line_val = _parse_int(row.get("LINEA")) - if line_val is None: - return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"} - if line_val <= 0: - return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"} - - driver_name = (row.get("CLAVE CONDUCTOR") or "").strip() - if driver_name and len(driver_name) > _MAX["driver_name"]: - return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"} - - for col, max_len in [ - ("LICENCIA", _MAX["license_number"]), - ("PERMISO LINEA EXPRESS", _MAX["express_line_id"]), - ("IDENTIFICACION ACE", _MAX["ace_id"]), - ("SEXO", _MAX["gender"]), - ("PAIS NACIMIENTO", _MAX["birth_country"]), - ("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]), - ("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]), - ("NOMBRE(S)", _MAX["first_name"]), - ("APELLIDO PATERNO", _MAX["last_name"]), - ("FORMA IDENTIFICACION 1", _MAX["id_key1"]), - ("NUM. IDENTIFICACION 1", _MAX["id_number1"]), - ("ESTADO", _MAX["id_state1"]), - ("PAIS", _MAX["id_country1"]), - ("FORMA IDENTIFICACION 2", _MAX["id_key2"]), - ("NUM. IDENTIFICACION 2", _MAX["id_number2"]), - ("ESTADO 2", _MAX["id_state2"]), - ("PAIS 2", _MAX["id_country2"]), - ]: - val = (row.get(col) or "").strip() - if val and len(val) > max_len: - return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"} - - fecha = row.get("FECHA NACIMIENTO") - if fecha is not None and str(fecha).strip(): - if _parse_birth_date(fecha) is None: - return { - "line": line_num, - "col": "FECHA NACIMIENTO", - "msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)", - } - - return None - - -def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: - transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"]) - line = _parse_int(row.get("LINEA")) - if not transporter_key or line is None: - return {} - data = { - "transporter_key": transporter_key, - "line": line, - "driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]), - "license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]), - "express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]), - "ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]), - "birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")), - "gender": _str_or_none(row.get("SEXO"), _MAX["gender"]), - "birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]), - "hazardous_material_auth": _str_or_none( - row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"] - ), - "hazardous_material_state": _str_or_none( - row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"] - ), - "first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]), - "last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]), - "id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]), - "id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]), - "id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]), - "id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]), - "id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]), - "id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]), - "id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]), - "id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]), - "company_id": company_id, - "tenant_id": tenant_id, - } - return data - - def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: with open(file_path, "r", encoding="utf-8-sig") as f: total_rows = sum(1 for _ in f) - 1 except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Drivers import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: with open(file_path, "r", encoding="utf-8-sig") as f_in, open( @@ -333,51 +95,27 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, if progress_callback and i % 500 == 0: progress_callback(i, total_rows, error_count) - row_norm = row_from_template(row, normalize_header) - err = _validate_row_driver(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_driver(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"Drivers import scan failed: {e}") + logger.error("Drivers import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Drivers import: failed to store error lines: {e}") - - result = { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - return result + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def run_scan_sync(job_id: str) -> Dict[str, Any]: @@ -390,19 +128,16 @@ def run_scan_sync(job_id: str) -> Dict[str, Any]: ex=DRV_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") + logger.warning("Drivers import: failed to store scan status in Redis: %s", e) return result @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): - logger.info(f"Drivers import: starting scan for job {job_id}") + logger.info("Drivers import: starting scan for job %s", job_id) def on_progress(current: int, total: int, errors: int) -> None: - self.update_state( - state="PROGRESS", - meta={"current": current, "total": total, "errors": errors}, - ) + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) result = _do_scan(job_id, progress_callback=on_progress) try: @@ -413,58 +148,27 @@ def scan_file(self, job_id: str, config: str = None): ex=DRV_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") + logger.warning("Drivers import: failed to store scan status in Redis: %s", e) return result def _do_commit(job_id: str) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Drivers import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.transportation.drivers.services import DriverService from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO @@ -475,6 +179,7 @@ def _do_commit(job_id: str) -> Dict[str, Any]: skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: @@ -497,21 +202,20 @@ def _do_commit(job_id: str) -> Dict[str, Any]: if i in error_lines: continue - row_norm = row_from_template(row, normalize_header) - err = _validate_row_driver(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_driver(row_norm, i) if err: skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", - "invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) + driver_key = (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-" + skipped_details.append({ + "line": i, + "driver_key": driver_key, + "invoice": driver_key, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) continue - data = _row_to_driver_dto(row_norm, tenant_id, company_id) + data = row_to_driver_data(row_norm, tenant_id, company_id) if not data or not data.get("transporter_key") or data.get("line") is None: skipped_invalid += 1 continue @@ -519,14 +223,12 @@ def _do_commit(job_id: str) -> Dict[str, Any]: key = f"{data['transporter_key']}:{data['line']}" if key in seen_keys_in_file: skipped_duplicate += 1 - skipped_details.append( - { - "line": i, - "driver_key": key, - "invoice": key, - "reason": "Clave duplicada en el archivo (se usa la primera)", - } - ) + skipped_details.append({ + "line": i, + "driver_key": key, + "invoice": key, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) continue seen_keys_in_file[key] = i @@ -535,7 +237,10 @@ def _do_commit(job_id: str) -> Dict[str, Any]: ) try: if existing: - update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")} + update_fields = { + k: v for k, v in data.items() + if k not in ("transporter_key", "line", "company_id", "tenant_id") + } for field, value in update_fields.items(): setattr(existing, field, value) session.add(existing) @@ -547,34 +252,33 @@ def _do_commit(job_id: str) -> Dict[str, Any]: except Exception as db_err: session.rollback() skipped_invalid += 1 - skipped_details.append( - {"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)} - ) + skipped_details.append({ + "line": i, "driver_key": key, "invoice": key, "reason": str(db_err), + }) continue try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"Drivers import DB error: {db_err}") + logger.error("Drivers import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} except Exception as e: - logger.error(f"Drivers import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("Drivers import task failed") return {"status": "failed", "error": str(e)} + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Drivers import cleanup failed: {cleanup_err}") + r = _get_redis() + r.delete(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}") + except Exception as e: + logger.warning("Drivers import: failed to delete status key: %s", e) total_ok = inserted_count + updated_count if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: @@ -620,13 +324,13 @@ def run_commit_sync(job_id: str) -> Dict[str, Any]: ex=DRV_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") + logger.warning("Drivers import: failed to store commit status in Redis: %s", e) return result @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): - logger.info(f"Drivers import: starting commit for job {job_id}") + logger.info("Drivers import: starting commit for job %s", job_id) result = _do_commit(job_id) try: r = _get_redis() @@ -636,5 +340,5 @@ def insert_valid_rows(self, job_id: str): ex=DRV_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") + logger.warning("Drivers import: failed to store commit status in Redis: %s", e) return result diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py new file mode 100644 index 00000000..99b8d524 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_driver + +__all__ = ["validate_row_driver"] diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py new file mode 100644 index 00000000..d5714b13 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/common.py @@ -0,0 +1,52 @@ +""" +Validaciones comunes de fila para import CSV de conductores. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + MAX_LEN, + check_max_length, + check_int_positive, + check_optional_birth_date, +) + + +def validate_row_driver_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + err = check_max_length( + row, "TRANSPORTISTA", MAX_LEN["transporter_key"], line_num, required=True + ) + if err: + return err + return check_int_positive(row, "LINEA", line_num) + + +def validate_row_driver_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("CLAVE CONDUCTOR", MAX_LEN["driver_name"]), + ("LICENCIA", MAX_LEN["license_number"]), + ("PERMISO LINEA EXPRESS", MAX_LEN["express_line_id"]), + ("IDENTIFICACION ACE", MAX_LEN["ace_id"]), + ("SEXO", MAX_LEN["gender"]), + ("PAIS NACIMIENTO", MAX_LEN["birth_country"]), + ("TRANSPORTA MAT. PELIGROSO?", MAX_LEN["hazardous_material_auth"]), + ("PERMISO MAT. PELIGROSO", MAX_LEN["hazardous_material_state"]), + ("NOMBRE(S)", MAX_LEN["first_name"]), + ("APELLIDO PATERNO", MAX_LEN["last_name"]), + ("FORMA IDENTIFICACION 1", MAX_LEN["id_key1"]), + ("NUM. IDENTIFICACION 1", MAX_LEN["id_number1"]), + ("ESTADO", MAX_LEN["id_state1"]), + ("PAIS", MAX_LEN["id_country1"]), + ("FORMA IDENTIFICACION 2", MAX_LEN["id_key2"]), + ("NUM. IDENTIFICACION 2", MAX_LEN["id_number2"]), + ("ESTADO 2", MAX_LEN["id_state2"]), + ("PAIS 2", MAX_LEN["id_country2"]), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_driver_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_optional_birth_date(row, "FECHA NACIMIENTO", line_num) diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py new file mode 100644 index 00000000..06a66d25 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/validators/create.py @@ -0,0 +1,27 @@ +""" +Punto de entrada de validación para import de una fila conductor. +""" +from typing import Dict, Any, Optional + +from .common import ( + validate_row_driver_required, + validate_row_driver_lengths, + validate_row_driver_date, +) + + +def validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de conductores. + Encadena: requeridos (TRANSPORTISTA, LINEA) → longitudes → fecha opcional. + """ + err = validate_row_driver_required(row, line_num) + if err: + return err + err = validate_row_driver_lengths(row, line_num) + if err: + return err + err = validate_row_driver_date(row, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py new file mode 100644 index 00000000..ab0f8ea6 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for exchange_rate) diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py new file mode 100644 index 00000000..472ab721 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/common_validators.py @@ -0,0 +1,65 @@ +""" +Validadores reutilizables para import CSV de tipos de cambio. +""" +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Any, Optional, List + +DATE_FORMATS: List[str] = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] +CURRENCY_MAX = 7 + + +def parse_date(val: Optional[str]) -> Optional[datetime]: + if not val or not str(val).strip(): + return None + raw = str(val).strip() + for fmt in DATE_FORMATS: + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + continue + return None + + +def parse_decimal_positive(val: Optional[str]) -> Optional[Decimal]: + if not val or not str(val).strip(): + return None + try: + v = float(str(val).strip().replace(",", ".")) + if v <= 0: + return None + return Decimal(str(round(v, 6))) + except (ValueError, TypeError): + return None + + +def check_required_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + fecha_raw = (row.get("FECHA") or "").strip() + if not fecha_raw: + return {"line": line_num, "col": "FECHA", "msg": "Requerido"} + if parse_date(fecha_raw) is None: + return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} + return None + + +def check_required_value_positive(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + valor_raw = (row.get("VALOR") or "").strip() + if not valor_raw: + return {"line": line_num, "col": "VALOR", "msg": "Requerido"} + try: + v = float(valor_raw.replace(",", ".")) + if v <= 0: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} + except ValueError: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} + return None + + +def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py new file mode 100644 index 00000000..42436530 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/common/mappers.py @@ -0,0 +1,48 @@ +""" +Mapeo fila CSV → datos para ExchangeRate. +""" +from datetime import datetime +from decimal import Decimal +from typing import Dict, Any, Optional + +from .common_validators import ( + parse_date, + parse_decimal_positive, +) + +CURRENCY_MAX = 7 + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_exchange_rate_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int +) -> Dict[str, Any]: + """Build dict for ExchangeRate model. Returns {} if FECHA or VALOR invalid.""" + parsed_date = parse_date(row_norm.get("FECHA")) + value_decimal = parse_decimal_positive(row_norm.get("VALOR")) + if not parsed_date or value_decimal is None: + return {} + local = _str_or_none(row_norm.get("MONEDA_LOCAL"), CURRENCY_MAX) + if local: + local = local.upper() + foreign = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), CURRENCY_MAX) + if foreign: + foreign = foreign.upper() + return { + "tenant_id": tenant_id, + "company_id": company_id, + "date": parsed_date, + "value": value_decimal, + "local_currency": local, + "foreign_currency": foreign, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py index 5ab2657e..9be2b566 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py @@ -1,322 +1,127 @@ """ Tareas Celery para importación CSV de Tipos de Cambio. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from datetime import datetime, time -from decimal import Decimal +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader from .template_config import row_from_template +from .validators import validate_row_exchange_rate +from .common.mappers import row_to_exchange_rate_data logger = logging.getLogger(__name__) +JOB_TYPE = "er" +TEMPLATE_ID = "exchange_rates" + +# Para routes.py ER_IMPORT_FILE_PREFIX = "er_import_file:" ER_IMPORT_META_PREFIX = "er_import_meta:" ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:" -ER_IMPORT_REDIS_TTL = 3600 # 1 hour - -DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] -TEMPLATE_ID = "exchange_rates" +ER_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) +def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: + return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"ER import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"er_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"ER import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{ER_IMPORT_FILE_PREFIX}{job_id}", - f"{ER_IMPORT_META_PREFIX}{job_id}", - f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"ER import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _parse_date(val: Optional[str]) -> Optional[datetime]: - """Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc.""" - if not val or not str(val).strip(): - return None - raw = str(val).strip() - for fmt in DATE_FORMATS: - try: - parsed = datetime.strptime(raw, fmt) - return datetime.combine(parsed.date(), time.min) - except ValueError: - continue - return None - - -def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Tipo de Cambio. Retorna error dict o None.""" - fecha_raw = (row.get("FECHA") or "").strip() - if not fecha_raw: - return {"line": line_num, "col": "FECHA", "msg": "Requerido"} - if _parse_date(fecha_raw) is None: - return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} - - valor_raw = (row.get("VALOR") or "").strip() - if not valor_raw: - return {"line": line_num, "col": "VALOR", "msg": "Requerido"} - try: - v = float(valor_raw.replace(",", ".")) - if v <= 0: - return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} - except ValueError: - return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} - - local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper() - if local_raw and len(local_raw) > 7: - return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"} - - foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper() - if foreign_raw and len(foreign_raw) > 7: - return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"ER import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"ER import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_exchange_rate(row_norm, i) + row_norm = _norm_row(row) + err = validate_row_exchange_rate(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"ER import scan failed: {e}") + logger.error("ER import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=ER_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"ER import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha). - """ - logger.info(f"ER import: starting commit for job {job_id}") +def scan_file(self, job_id: str, config: str = None): + logger.info("ER import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"ER import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate inserted_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] - response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: @@ -332,134 +137,91 @@ def insert_valid_rows(self, job_id: str): d = er.date.date() if hasattr(er.date, "date") else er.date existing_by_date[(tenant_id, company_id, d)] = er - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue + row_norm = _norm_row(row) + err = validate_row_exchange_rate(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_exchange_rate(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue + data = row_to_exchange_rate_data(row_norm, tenant_id, company_id) + if not data or not data.get("date"): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FECHA o VALOR no válidos"}) + continue - parsed_date = _parse_date(row_norm.get("FECHA")) - if not parsed_date: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "FECHA: no parseable"}) - continue - - try: - v = float((row_norm.get("VALOR") or "").strip().replace(",", ".")) - value_decimal = Decimal(str(round(v, 6))) - except (ValueError, TypeError): - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "VALOR: no numérico"}) - continue - - local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7) - if local_currency: - local_currency = local_currency.upper() - foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7) - if foreign_currency: - foreign_currency = foreign_currency.upper() - - key_date = parsed_date.date() - existing = existing_by_date.get((tenant_id, company_id, key_date)) - - if existing: - existing.value = value_decimal - existing.local_currency = local_currency or None - existing.foreign_currency = foreign_currency or None - session.add(existing) - inserted_count += 1 - else: - new_er = ExchangeRate( - tenant_id=tenant_id, - company_id=company_id, - date=parsed_date, - value=value_decimal, - local_currency=local_currency, - foreign_currency=foreign_currency, - ) - session.add(new_er) - existing_by_date[(tenant_id, company_id, key_date)] = new_er - inserted_count += 1 + key_date = data["date"].date() if hasattr(data["date"], "date") else data["date"] + existing = existing_by_date.get((tenant_id, company_id, key_date)) + if existing: + existing.value = data["value"] + existing.local_currency = data.get("local_currency") + existing.foreign_currency = data.get("foreign_currency") + session.add(existing) + else: + new_er = ExchangeRate(**data) + session.add(new_er) + existing_by_date[(tenant_id, company_id, key_date)] = new_er + inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"ER import DB error: {db_err}") + logger.error("ER import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - except Exception as e: - logger.error(f"ER import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("ER import task failed") return {"status": "failed", "error": str(e)} - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"ER import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", + if inserted_count == 0 and skipped_invalid > 0: + return { + "status": "warning", "inserted": 0, + "updated": 0, "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, "skipped_missing_fk": 0, "skipped_details": skipped_details, } - return response + return { + "status": "finished", + "inserted": inserted_count, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("ER import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py new file mode 100644 index 00000000..cf3b1792 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_exchange_rate + +__all__ = ["validate_row_exchange_rate"] diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py new file mode 100644 index 00000000..46f52d86 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/common.py @@ -0,0 +1,31 @@ +""" +Validaciones comunes de fila para import CSV de tipos de cambio. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + check_required_date, + check_required_value_positive, + check_optional_max_length, + CURRENCY_MAX, +) + + +def validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de tipos de cambio. + FECHA y VALOR requeridos; MONEDA_LOCAL y MONEDA_EXTRANJERA opcionales (max 7). + """ + err = check_required_date(row, line_num) + if err: + return err + err = check_required_value_positive(row, line_num) + if err: + return err + err = check_optional_max_length(row, "MONEDA_LOCAL", CURRENCY_MAX, line_num) + if err: + return err + err = check_optional_max_length(row, "MONEDA_EXTRANJERA", CURRENCY_MAX, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py new file mode 100644 index 00000000..ef5d3d47 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/validators/create.py @@ -0,0 +1,6 @@ +""" +Punto de entrada de validación para import de una fila tipo de cambio. +""" +from .common import validate_row_exchange_rate + +__all__ = ["validate_row_exchange_rate"] diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index e136e618..a3fe37be 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -1,5 +1,4 @@ import os -import base64 from datetime import datetime from decimal import Decimal import csv @@ -13,86 +12,35 @@ from core.celery_app import celery_app from core.database import CoreSessionLocal from core.paths import layout_path +from ..common import storage as common_storage +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process -# We'll need schemas for validation -# from api.v1.modules.a76.invoices.schemas import InvoiceHeaderCreate -# But for Phase 1 we use a lighter check - logger = logging.getLogger(__name__) -# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem) +# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py) +JOB_TYPE = "" + +# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="") IMPORT_FILE_KEY_PREFIX = "import_file:" IMPORT_META_KEY_PREFIX = "import_meta:" IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" -IMPORT_REDIS_TTL = 3600 # 1 hour - - -def _get_redis(): - """Redis client using same URL as Celery broker (worker and API can share data).""" - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - """Directory on the worker for temp CSV and meta (same structure as API, but local to worker).""" - return layout_path("imports", "temp") +IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - """ - Load file content from Redis and write to worker's upload dir. - Returns local file_path if successful, None otherwise. - """ - redis_client = _get_redis() - key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}" - data = redis_client.get(key) - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Failed to decode import file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path + """Usa common storage con job_type vacío (prefijo import_).""" + return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import") def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - """Load meta from Redis and write to worker's meta file. Returns True if meta was found and written.""" - redis_client = _get_redis() - key = f"{IMPORT_META_KEY_PREFIX}{job_id}" - data = redis_client.get(key) - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Failed to decode import meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True + return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import") def _delete_import_from_redis(job_id: str) -> None: - """Remove file, meta and error lines from Redis after commit (cleanup).""" - try: - r = _get_redis() - r.delete( - f"{IMPORT_FILE_KEY_PREFIX}{job_id}", - f"{IMPORT_META_KEY_PREFIX}{job_id}", - f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Failed to delete import keys from Redis: {e}") + common_storage.delete_import_from_redis(JOB_TYPE, job_id) class ForeignKeyValidator: def __init__(self, session, tenant_id, company_id): @@ -189,10 +137,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} _ensure_worker_has_meta_from_redis(job_id, file_path) - # 2. Setup Error Log - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) total_rows = 0 error_count = 0 @@ -213,22 +158,12 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - tenant_id = None - company_id = None - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f_meta: - meta = json.load(f_meta) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception as e: - logger.warning(f"Failed to read meta for job {job_id}: {e}") - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Missing context (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + meta = common_meta.load_meta(file_path) template_id = meta.get("template_id") or ( "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" ) @@ -337,24 +272,11 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): except Exception: pass if error_lines_list: - r = _get_redis() - r.set( - f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=IMPORT_REDIS_TTL, - ) + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.warning(f"Failed to store error lines in Redis: {e}") - # 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV) - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def validate_row_phase_1( row: Dict[str, Any], @@ -772,13 +694,23 @@ def insert_valid_rows(self, job_id: str, model_target: str): # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) file_path = _ensure_worker_has_file_from_redis(job_id) if not file_path: - upload_dir = _worker_upload_dir() - file_path = os.path.join(upload_dir, f"{job_id}.csv") - if not os.path.exists(file_path): + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} + file_path = alt_path else: _ensure_worker_has_meta_from_redis(job_id, file_path) + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + meta_path = common_meta.get_meta_path(file_path) + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + try: from api.v1.modules.a76.invoices.models import ( InvoiceHeader, @@ -800,67 +732,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm - from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.line_financials.models import LineFinancial from api.v1.modules.a76.items.line_quantities.models import LineQuantity from api.v1.modules.a76.items.line_customs.models import LineCustom from api.v1.modules.a76.items.line_descriptions.models import LineDescription from api.v1.modules.a76.parts.models import Part - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"{job_id}.jsonl") + footer_config = parse_footer_config(meta.get("footer_config")) - # 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file) - error_lines = set() - try: - r = _get_redis() - raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, 'r', encoding='utf-8') as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err['line']) - except Exception: - pass - - # Load Metadata (Context) - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - footer_config = {} - meta = {} - - if os.path.exists(meta_path): - try: - with open(meta_path, 'r') as f: - meta = json.load(f) or {} - tenant_id = meta.get('tenant_id') - company_id = meta.get('company_id') - operation_type_raw = meta.get('operation_type', 'imp') - footer_config = parse_footer_config(meta.get('footer_config')) - except: pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Missing context (tenant/company)"} - - # 2. Re-read and Map - # Initialize counters outside the session block so they're accessible later - headers_to_insert = [] - details_to_insert = [] - skipped_invalid = 0 - skipped_missing_invoice = 0 - skipped_missing_fk = 0 - skipped_fk_details = [] - inserted_count = 0 - response = None # Will be set inside the session block - date_format = footer_config.get("dateFormat") - # Validate and set default date_format if not provided if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format @@ -874,6 +755,15 @@ def insert_valid_rows(self, job_id: str, model_target: str): logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") + headers_to_insert = [] + details_to_insert = [] + skipped_invalid = 0 + skipped_missing_invoice = 0 + skipped_missing_fk = 0 + skipped_fk_details = [] + inserted_count = 0 + response = None + with CoreSessionLocal() as session: invoice_id_cache = {} cleared_invoices = set() # Track invoices where we've already cleared items in this job @@ -1462,11 +1352,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - _delete_import_from_redis(job_id) + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) except Exception as cleanup_err: logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py new file mode 100644 index 00000000..1efc271d --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/__init__.py @@ -0,0 +1 @@ +# common validators for parts CSV import diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py new file mode 100644 index 00000000..9e77983c --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/common_validators.py @@ -0,0 +1,49 @@ +""" +Helpers reutilizables para validación de filas CSV (partes). +Cada función devuelve Optional[Dict] con keys line, col, msg. +Referencia: a76/invoices (common_validators, validators/common). +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional + + +def check_required( + row: Dict[str, Any], col: str, line_num: int +) -> Optional[Dict[str, Any]]: + """Devuelve error si el campo está vacío.""" + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + + +def check_max_length( + row: Dict[str, Any], + col: str, + max_len: int, + line_num: int, + required: bool = False, +) -> Optional[Dict[str, Any]]: + """Devuelve error si el campo excede max_len. Si required=True, también exige que haya valor.""" + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_decimal( + row: Dict[str, Any], col: str, line_num: int +) -> Optional[Dict[str, Any]]: + """Devuelve error si el valor no es un número decimal válido (solo cuando hay valor).""" + val = row.get(col) + if val is None or val == "": + return None + try: + Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": col, "msg": "Debe ser número"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py new file mode 100644 index 00000000..35d9ba71 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/fk_loader.py @@ -0,0 +1,48 @@ +""" +Carga de conjuntos FK para validación/mapeo de import CSV de partes. +""" +from typing import Set, Tuple + +from core.database import CoreSessionLocal + + +def load_parts_fk_sets( + tenant_id: int, + company_id: int, +) -> Tuple[Set[str], Set[str], Set[str]]: + """ + Carga valid_class_codes, valid_uom_codes, valid_currency_codes desde BD. + Devuelve (valid_class_codes, valid_uom_codes, valid_currency_codes). + """ + valid_class_codes: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_currency_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + for c in ( + session.query(Class.class_code) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + valid_class_codes.add(c[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + for cur in session.query(CurrencyType.code).all(): + valid_currency_codes.add(cur[0]) + except Exception as e: + import logging + logging.getLogger(__name__).warning("Parts import: could not load FK sets: %s", e) + return valid_class_codes, valid_uom_codes, valid_currency_codes diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py new file mode 100644 index 00000000..d0d57713 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py @@ -0,0 +1,111 @@ +""" +Mapeo fila CSV → datos para Part. Helpers de normalización de valores. +Referencia: a76/invoices common/mappers.py +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, Set + + +def clean_dict(data_dict: dict) -> dict: + """Convierte strings vacíos a None; mantiene 0 en campos no-_id. Igual que a76/invoices.""" + cleaned = {} + for key, value in data_dict.items(): + if isinstance(value, str) and not value.strip(): + cleaned[key] = None + elif value == 0 and (key.endswith("_id") or key == "remesa"): + cleaned[key] = None + else: + cleaned[key] = value + return cleaned + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _bool_from_row(val: Any) -> bool: + if val is None or val == "": + return True + s = str(val).strip().upper() + if s in ("0", "F", "FALSE", "NO", "N"): + return False + return True + + +def row_to_part_data( + row_norm: Dict[str, Any], + valid_class_codes: Set[str], + valid_uom_codes: Set[str], + valid_currency_codes: Set[str], +) -> Dict[str, Any]: + """ + Mapea una fila normalizada del CSV a un diccionario de datos para Part. + Ajusta FKs opcionales (part_class, unit_of_measure, currency_key) a None si no están en los conjuntos. + El caller debe añadir tenant_id, company_id, client_id al crear Part. + """ + part_number = _str_or_none(row_norm.get("NUMPARTE"), 70) + commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + part_class = _str_or_none(row_norm.get("CLASE"), 8) + if part_class and part_class not in valid_class_codes: + part_class = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + currency_key = _str_or_none(row_norm.get("MONEDA"), 3) + if currency_key and currency_key not in valid_currency_codes: + currency_key = None + + unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT")) + currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None + unit_weight = _decimal_or_none(row_norm.get("PESOUNIT")) + weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6) + fraction = _str_or_none(row_norm.get("FRACCION"), 10) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + fda_key = _str_or_none(row_norm.get("FDAKEY"), 20) + fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30) + license_code = _str_or_none(row_norm.get("LICENCIA"), 3) + eccn = _str_or_none(row_norm.get("ECCN"), 20) + export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2) + exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19) + is_active = _bool_from_row(row_norm.get("ACTIVO")) + + return { + "part_number": part_number, + "commercial_part_number": commercial, + "description_spanish": desc_es, + "description_english": desc_en, + "part_class": part_class, + "unit_of_measure": unit_of_measure, + "unit_cost": unit_cost, + "currency_type": currency_type, + "currency_key": currency_key, + "unit_weight": unit_weight, + "weight_type": weight_type, + "fraction": fraction, + "us_fraction": us_fraction, + "fda_key": fda_key, + "fcc_key": fcc_key, + "license_code": license_code, + "eccn": eccn, + "export_code": export_code, + "exclusion_symbol": exclusion_symbol, + "is_active": is_active, + } diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py b/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py index 8311b001..7d116e61 100644 --- a/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py @@ -1,270 +1,75 @@ """ Tareas Celery para importación CSV de Números de Parte. Flujo: scan_file (validación) → insert_valid_rows (commit). +Orquestación usa common (storage, normalize, csv_reader, meta, responses) y common.fk_loader. """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from decimal import Decimal, InvalidOperation -from typing import Dict, Any, Optional, List, Set +import os +from typing import Dict, Any, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import csv_reader as common_csv +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template +from .validators import validate_row_part +from .common.mappers import row_to_part_data +from .common.fk_loader import load_parts_fk_sets logger = logging.getLogger(__name__) +JOB_TYPE = "part" + +# Para routes.py (upload guarda con estos prefijos) PART_IMPORT_FILE_PREFIX = "part_import_file:" PART_IMPORT_META_PREFIX = "part_import_meta:" PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:" -PART_IMPORT_REDIS_TTL = 3600 - - -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) - - -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Parts import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"part_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Parts import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{PART_IMPORT_FILE_PREFIX}{job_id}", - f"{PART_IMPORT_META_PREFIX}{job_id}", - f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Parts import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_part( - row: Dict[str, Any], - line_num: int, - valid_class_codes: Optional[Set[str]] = None, - valid_uom_codes: Optional[Set[str]] = None, - valid_currency_codes: Optional[Set[str]] = None, -) -> Optional[Dict[str, Any]]: - part_number = (row.get("NUMPARTE") or "").strip() - if not part_number: - return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} - if len(part_number) > 70: - return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"} - - commercial = (row.get("NUMPARTECOM") or "").strip() - if commercial and len(commercial) > 70: - return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"} - - desc_es = (row.get("DESCRIPCIONE") or "").strip() - if desc_es and len(desc_es) > 500: - return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} - desc_en = (row.get("DESCRIPCIONI") or "").strip() - if desc_en and len(desc_en) > 500: - return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} - - part_class = (row.get("CLASE") or "").strip() - if part_class and len(part_class) > 8: - return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} - # Si CLASE no existe en catálogo se guardará null (no se rechaza la fila) - - uom = (row.get("UNIMED") or "").strip() - if uom and len(uom) > 5: - return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} - # Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila) - - currency_key = (row.get("MONEDA") or "").strip() - if currency_key and len(currency_key) > 3: - return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"} - # Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila) - - unit_cost = row.get("COSTOUNIT") - if unit_cost is not None and unit_cost != "": - try: - Decimal(str(unit_cost)) - except (InvalidOperation, ValueError, TypeError): - return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"} - - unit_weight = row.get("PESOUNIT") - if unit_weight is not None and unit_weight != "": - try: - Decimal(str(unit_weight)) - except (InvalidOperation, ValueError, TypeError): - return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"} - - fraction = (row.get("FRACCION") or "").strip() - if fraction and len(fraction) > 10: - return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"} - us_fraction = (row.get("FRACCIONAME") or "").strip() - if us_fraction and len(us_fraction) > 16: - return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} - fda_key = (row.get("FDAKEY") or "").strip() - if fda_key and len(fda_key) > 20: - return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"} - fcc_key = (row.get("FCCKEY") or "").strip() - if fcc_key and len(fcc_key) > 30: - return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"} - license_code = (row.get("LICENCIA") or "").strip() - if license_code and len(license_code) > 3: - return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"} - eccn = (row.get("ECCN") or "").strip() - if eccn and len(eccn) > 20: - return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"} - export_code = (row.get("EXPORTCODE") or "").strip() - if export_code and len(export_code) > 2: - return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"} - exclusion = (row.get("EXCLUSION") or "").strip() - if exclusion and len(exclusion) > 19: - return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"} - weight_type = (row.get("TIPOPESO") or "").strip() - if weight_type and len(weight_type) > 6: - return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"} - - return None +PART_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): - logger.info(f"Parts import: starting scan for job {job_id}") + logger.info("Parts import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Parts import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - valid_class_codes: Set[str] = set() - valid_uom_codes: Set[str] = set() - valid_currency_codes: Set[str] = set() try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.classes.models import Class - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - for c in ( - session.query(Class.class_code) - .filter( - Class.tenant_id == tenant_id, - Class.company_id == company_id, - ) - .all() - ): - valid_class_codes.add(c[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - for cur in session.query(CurrencyType.code).all(): - valid_currency_codes.add(cur[0]) - except Exception as e: - logger.warning(f"Parts import: could not load FK sets: {e}") + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + valid_class_codes, valid_uom_codes, valid_currency_codes = load_parts_fk_sets(tenant_id, company_id) error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv.iter_csv_rows(file_path): if i % 500 == 0: self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_part( + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_part( row_norm, i, valid_class_codes=valid_class_codes, @@ -273,170 +78,51 @@ def scan_file(self, job_id: str, config: str = None): ) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"Parts import scan failed: {e}") + logger.error("Parts import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=PART_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Parts import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _int_or_none(val: Any) -> Optional[int]: - if val is None or val == "": - return None - try: - return int(val) - except (ValueError, TypeError): - return None - - -def _decimal_or_none(val: Any) -> Optional[Decimal]: - if val is None or val == "": - return None - try: - return Decimal(str(val)) - except (InvalidOperation, ValueError, TypeError): - return None - - -def _bool_from_row(val: Any) -> bool: - if val is None or val == "": - return True - s = str(val).strip().upper() - if s in ("0", "F", "FALSE", "NO", "N"): - return False - return True + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail + ) @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): - logger.info(f"Parts import: starting commit for job {job_id}") + logger.info("Parts import: starting commit for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Parts import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.parts.models import Part - valid_class_codes: Set[str] = set() - valid_uom_codes: Set[str] = set() - valid_currency_codes: Set[str] = set() - try: - with CoreSessionLocal() as session: - from api.v1.modules.a76.classes.models import Class - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - for c in ( - session.query(Class.class_code) - .filter( - Class.tenant_id == tenant_id, - Class.company_id == company_id, - ) - .all() - ): - valid_class_codes.add(c[0]) - for u in ( - session.query(UnitOfMeasure.code) - .filter( - UnitOfMeasure.tenant_id == tenant_id, - UnitOfMeasure.company_id == company_id, - ) - .all() - ): - valid_uom_codes.add(u[0]) - for cur in session.query(CurrencyType.code).all(): - valid_currency_codes.add(cur[0]) - except Exception as e: - logger.warning(f"Parts import: could not load FK sets: {e}") + valid_class_codes, valid_uom_codes, valid_currency_codes = load_parts_fk_sets(tenant_id, company_id) inserted_count = 0 skipped_invalid = 0 @@ -444,196 +130,112 @@ def insert_valid_rows(self, job_id: str): skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: - existing_by_part_number: Dict[str, Part] = {} - for p in ( - session.query(Part) - .filter( + existing_by_part_number = { + p.part_number: p + for p in session.query(Part).filter( Part.tenant_id == tenant_id, Part.company_id == company_id, + ).all() + } + + for i, row in common_csv.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, ) - .all() - ): - existing_by_part_number[p.part_number] = p + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + data = row_to_part_data( + row_norm, + valid_class_codes, + valid_uom_codes, + valid_currency_codes, + ) + part_number = data.get("part_number") + if not part_number: + skipped_invalid += 1 + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_part( - row_norm, - i, - valid_class_codes=valid_class_codes, - valid_uom_codes=valid_uom_codes, - valid_currency_codes=valid_currency_codes, + existing = existing_by_part_number.get(part_number) + if existing: + for key, value in data.items(): + setattr(existing, key, value) + session.add(existing) + inserted_count += 1 + else: + new_part = Part( + tenant_id=tenant_id, + company_id=company_id, + client_id=company_id, + **data, ) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} - ) - continue - - part_number = _str_or_none(row_norm.get("NUMPARTE"), 70) - if not part_number: - skipped_invalid += 1 - continue - - commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70) - desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) - desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) - part_class = _str_or_none(row_norm.get("CLASE"), 8) - if part_class and part_class not in valid_class_codes: - part_class = None - unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) - if unit_of_measure and unit_of_measure not in valid_uom_codes: - unit_of_measure = None - currency_key = _str_or_none(row_norm.get("MONEDA"), 3) - if currency_key and currency_key not in valid_currency_codes: - currency_key = None - - unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT")) - currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None - unit_weight = _decimal_or_none(row_norm.get("PESOUNIT")) - weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6) - fraction = _str_or_none(row_norm.get("FRACCION"), 10) - us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) - fda_key = _str_or_none(row_norm.get("FDAKEY"), 20) - fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30) - license_code = _str_or_none(row_norm.get("LICENCIA"), 3) - eccn = _str_or_none(row_norm.get("ECCN"), 20) - export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2) - exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19) - is_active = _bool_from_row(row_norm.get("ACTIVO")) - - existing = existing_by_part_number.get(part_number) - if existing: - existing.commercial_part_number = commercial - existing.description_spanish = desc_es - existing.description_english = desc_en - existing.part_class = part_class - existing.unit_of_measure = unit_of_measure - existing.unit_cost = unit_cost - existing.currency_type = currency_type - existing.currency_key = currency_key - existing.unit_weight = unit_weight - existing.weight_type = weight_type - existing.fraction = fraction - existing.us_fraction = us_fraction - existing.fda_key = fda_key - existing.fcc_key = fcc_key - existing.license_code = license_code - existing.eccn = eccn - existing.export_code = export_code - existing.exclusion_symbol = exclusion_symbol - existing.is_active = is_active - session.add(existing) - inserted_count += 1 - else: - new_part = Part( - tenant_id=tenant_id, - company_id=company_id, - client_id=company_id, - part_number=part_number, - commercial_part_number=commercial, - description_spanish=desc_es, - description_english=desc_en, - part_class=part_class, - unit_of_measure=unit_of_measure, - unit_cost=unit_cost, - currency_type=currency_type, - currency_key=currency_key, - unit_weight=unit_weight, - weight_type=weight_type, - fraction=fraction, - us_fraction=us_fraction, - fda_key=fda_key, - fcc_key=fcc_key, - license_code=license_code, - eccn=eccn, - export_code=export_code, - exclusion_symbol=exclusion_symbol, - is_active=is_active, - ) - session.add(new_part) - existing_by_part_number[part_number] = new_part - inserted_count += 1 + session.add(new_part) + existing_by_part_number[part_number] = new_part + inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"Parts import DB error: {db_err}") - return {"status": "failed", "error": str(db_err)} + logger.error("Parts import DB error: %s", db_err) + return common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, error=str(db_err), + ) total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } + response = common_responses.commit_result( + "warning", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, + message=f"No se insertaron registros. {total_skipped} rechazados.", + ) elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, + error="No hay registros válidos en el archivo CSV", + ) else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "finished", inserted_count, skipped_invalid, skipped_missing_fk, + skipped_duplicate, skipped_details, + ) except Exception as e: - logger.error(f"Parts import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} + logger.exception("Parts import task failed") + response = common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, error=str(e), + ) - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Parts import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) if response is None: - response = { - "status": "failed", - "error": "Error inesperado", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } + response = common_responses.commit_result( + "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_details, error="Error inesperado", + ) return response diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py new file mode 100644 index 00000000..cf0b1d03 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/validators/__init__.py @@ -0,0 +1,4 @@ +# validators for parts CSV import row validation +from .create import validate_row_part + +__all__ = ["validate_row_part"] diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py new file mode 100644 index 00000000..b21147d9 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/validators/common.py @@ -0,0 +1,50 @@ +""" +Validaciones comunes de fila para import CSV de partes. +Usa helpers de common.common_validators; agrupa por tipo (requeridos, longitudes, tipos). +Referencia: a76/invoices imports/temporary/validators/common.py +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import check_max_length, check_decimal + + +def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida campos obligatorios de una fila de parte.""" + return check_max_length(row, "NUMPARTE", 70, line_num, required=True) + + +def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida longitudes máximas de todos los campos de texto.""" + checks = [ + ("NUMPARTECOM", 70), + ("DESCRIPCIONE", 500), + ("DESCRIPCIONI", 500), + ("CLASE", 8), + ("UNIMED", 5), + ("MONEDA", 3), + ("FRACCION", 10), + ("FRACCIONAME", 16), + ("FDAKEY", 20), + ("FCCKEY", 30), + ("LICENCIA", 3), + ("ECCN", 20), + ("EXPORTCODE", 2), + ("EXCLUSION", 19), + ("TIPOPESO", 6), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida tipos numéricos (decimales) de la fila.""" + err = check_decimal(row, "COSTOUNIT", line_num) + if err: + return err + err = check_decimal(row, "PESOUNIT", line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py new file mode 100644 index 00000000..0154b059 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/parts/validators/create.py @@ -0,0 +1,39 @@ +""" +Punto de entrada de validación para creación/import de una fila de parte. +Encadena validaciones comunes (requeridos, longitudes, tipos). +Referencia: a76/invoices imports/temporary/validators/create.py +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_required, + validate_row_lengths, + validate_row_types, +) + + +def validate_row_part( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_currency_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de partes para import. + Encadena: requeridos → longitudes → tipos. + Los conjuntos valid_* se mantienen en la firma por compatibilidad con el caller. + """ + err = validate_row_required(row, line_num) + if err: + return err + + err = validate_row_lengths(row, line_num) + if err: + return err + + err = validate_row_types(row, line_num) + if err: + return err + + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py new file mode 100644 index 00000000..47ef2168 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers, fk_loader diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py new file mode 100644 index 00000000..64bb9980 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/common_validators.py @@ -0,0 +1,82 @@ +""" +Validadores reutilizables para import CSV de pedimentos. +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, Set + + +def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_int_in_set( + row: Dict[str, Any], col: str, valid_ids: Set[int], line_num: int +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + try: + client_id = int(val) + except ValueError: + return {"line": line_num, "col": col, "msg": "Debe ser número entero"} + if client_id not in valid_ids: + return {"line": line_num, "col": col, "msg": "Cliente no existe en catálogo"} + return None + + +def check_in_set( + row: Dict[str, Any], + col: str, + valid_set: Set[str], + line_num: int, + max_len: int = 10, + catalog_name: str = "catálogo", +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + if valid_set and val not in valid_set: + return {"line": line_num, "col": col, "msg": f"No existe en {catalog_name}"} + return None + + +def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_optional_decimal(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + try: + Decimal(val.replace(",", ".")) + except (InvalidOperation, ValueError): + return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} + return None + + +def check_optional_in_set( + row: Dict[str, Any], col: str, allowed: Set[str], line_num: int, msg: str +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip().lower() + if not val: + return None + if val not in allowed: + return {"line": line_num, "col": col, "msg": msg} + return None + + +PEDIMENTO_CODE_MAX = 2 +REGIMEN_MAX = 3 diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py new file mode 100644 index 00000000..411d289f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/fk_loader.py @@ -0,0 +1,38 @@ +""" +Carga de conjuntos FK para validación de import CSV de pedimentos. +""" +from typing import Set, Tuple + +from sqlalchemy.orm import Session + + +def load_pedimentos_fk_sets( + session: Session, tenant_id: int, company_id: int +) -> Tuple[Set[int], Set[str], Set[str]]: + """ + Carga valid_client_ids (ClientProvider.id), valid_regimes (RegimenPedimento.code), + valid_pedimento_codes (PedimentoCode.code). + """ + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + + return valid_client_ids, valid_regimes, valid_pedimento_codes diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py new file mode 100644 index 00000000..6eed3b87 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/common/mappers.py @@ -0,0 +1,59 @@ +""" +Mapeo fila CSV → datos para PedimentosCreate. +""" +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional + + +def parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + return Decimal(str(val).strip().replace(",", ".")) + except (InvalidOperation, ValueError): + return None + + +def row_to_pedimento_data(row_norm: Dict[str, Any]) -> Dict[str, Any]: + """Build dict for PedimentosCreate from normalized CSV row.""" + year = (row_norm.get("AÑO") or "").strip()[:2] + customs_office = (row_norm.get("ADUANA") or "").strip()[:3] + license_val = (row_norm.get("PATENTE") or "").strip()[:4] + pedimento_number = (row_norm.get("NUMERO") or "").strip()[:7] + client_id_str = (row_norm.get("CLIENTE_ID") or "").strip() + client_id = int(client_id_str) if client_id_str else None + pedimento_code = (row_norm.get("CODIGO_PEDIMENTO") or "").strip()[:2] + regime = (row_norm.get("REGIMEN") or "").strip()[:3] + + data = { + "year": year, + "customs_office": customs_office, + "license": license_val, + "pedimento_number": pedimento_number, + "client_id": client_id, + "pedimento_code": pedimento_code, + "regime": regime, + } + + op = (row_norm.get("TIPO_OPERACION") or "").strip().lower() + if op in ("imp", "exp"): + data["operation_type"] = op + + ptype = (row_norm.get("TIPO_PEDIMENTO") or "").strip().lower() + if ptype in ("normal", "consolidated", "complementary", "automobile"): + data["pedimento_type"] = ptype + + status = (row_norm.get("ESTATUS") or "").strip() + if status: + data["status"] = status[:30] + + data["usd_value"] = parse_decimal(row_norm.get("VALOR_USD")) + data["paid_price"] = parse_decimal(row_norm.get("PRECIO_PAGADO")) + data["gross_weight"] = parse_decimal(row_norm.get("PESO_BRUTO")) + data["exchange_rate"] = parse_decimal(row_norm.get("TIPO_CAMBIO")) + + obs = (row_norm.get("OBSERVACIONES") or "").strip() + if obs: + data["observations"] = obs + + return data diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py index 8bfea495..96da8f38 100644 --- a/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py @@ -1,445 +1,142 @@ """ Tareas Celery para importación CSV de Pedimentos. -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader), fk_loader, validators, mappers. """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from decimal import Decimal, InvalidOperation +import os from datetime import datetime -from typing import Dict, Any, Optional, List, Set +from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader from .template_config import row_from_template +from .validators import validate_row_pedimento +from .common.fk_loader import load_pedimentos_fk_sets +from .common.mappers import row_to_pedimento_data logger = logging.getLogger(__name__) -# Redis keys (prefijo propio para no colisionar con otros imports) +JOB_TYPE = "ped" +TEMPLATE_ID = "pedimentos" + +# Para routes.py PED_IMPORT_FILE_PREFIX = "ped_import_file:" PED_IMPORT_META_PREFIX = "ped_import_meta:" PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" -PED_IMPORT_REDIS_TTL = 3600 # 1 hour +PED_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) +def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: + return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"ped_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{PED_IMPORT_FILE_PREFIX}{job_id}", - f"{PED_IMPORT_META_PREFIX}{job_id}", - f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _validate_row_pedimento( - row: Dict[str, Any], - line_num: int, - valid_client_ids: Set[int], - valid_regimes: Set[str], - valid_pedimento_codes: Set[str], -) -> Optional[Dict[str, Any]]: - """Valida una fila para Pedimento. Retorna error dict o None.""" - # Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN - year = (row.get("AÑO") or "").strip() - if not year: - return {"line": line_num, "col": "AÑO", "msg": "Requerido"} - if len(year) > 2: - return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"} - - customs_office = (row.get("ADUANA") or "").strip() - if not customs_office: - return {"line": line_num, "col": "ADUANA", "msg": "Requerido"} - if len(customs_office) > 3: - return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"} - - license_val = (row.get("PATENTE") or "").strip() - if not license_val: - return {"line": line_num, "col": "PATENTE", "msg": "Requerido"} - if len(license_val) > 4: - return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"} - - pedimento_number = (row.get("NUMERO") or "").strip() - if not pedimento_number: - return {"line": line_num, "col": "NUMERO", "msg": "Requerido"} - if len(pedimento_number) > 7: - return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"} - - client_id_str = (row.get("CLIENTE_ID") or "").strip() - if not client_id_str: - return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"} - try: - client_id = int(client_id_str) - except ValueError: - return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"} - if client_id not in valid_client_ids: - return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"} - - pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip() - if not pedimento_code: - return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"} - if len(pedimento_code) > 2: - return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"} - if pedimento_code not in valid_pedimento_codes: - return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"} - - regime = (row.get("REGIMEN") or "").strip() - if not regime: - return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"} - if len(regime) > 3: - return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"} - if regime not in valid_regimes: - return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"} - - # Optional numeric/string fields - validate format if present - status = (row.get("ESTATUS") or "").strip() - if status and len(status) > 30: - return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"} - - for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]: - val = (row.get(col) or "").strip() - if not val: - continue - try: - Decimal(val.replace(",", ".")) - except (InvalidOperation, ValueError): - return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} - - operation_type = (row.get("TIPO_OPERACION") or "").strip().lower() - if operation_type and operation_type not in ("imp", "exp", ""): - return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"} - - pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower() - if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""): - return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"} - - return None - - -def _parse_decimal(val: Any) -> Optional[Decimal]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - try: - return Decimal(str(val).strip().replace(",", ".")) - except (InvalidOperation, ValueError): - return None - - -def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]: - """Build dict for PedimentosCreate from normalized CSV row (canonical names).""" - year = (row.get("AÑO") or "").strip()[:2] - customs_office = (row.get("ADUANA") or "").strip()[:3] - license_val = (row.get("PATENTE") or "").strip()[:4] - pedimento_number = (row.get("NUMERO") or "").strip()[:7] - client_id_str = (row.get("CLIENTE_ID") or "").strip() - client_id = int(client_id_str) if client_id_str else None - pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2] - regime = (row.get("REGIMEN") or "").strip()[:3] - - data = { - "year": year, - "customs_office": customs_office, - "license": license_val, - "pedimento_number": pedimento_number, - "client_id": client_id, - "pedimento_code": pedimento_code, - "regime": regime, - } - - op = (row.get("TIPO_OPERACION") or "").strip().lower() - if op in ("imp", "exp"): - data["operation_type"] = op - - ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower() - if ptype in ("normal", "consolidated", "complementary", "automobile"): - data["pedimento_type"] = ptype - - status = (row.get("ESTATUS") or "").strip() - if status: - data["status"] = status[:30] - - data["usd_value"] = _parse_decimal(row.get("VALOR_USD")) - data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO")) - data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO")) - data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO")) - - obs = (row.get("OBSERVACIONES") or "").strip() - if obs: - data["observations"] = obs - - return data - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"Pedimentos import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Pedimentos import: failed to read meta: {e}") + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - # Load valid FK sets for validation - valid_client_ids: Set[int] = set() - valid_regimes: Set[str] = set() - valid_pedimento_codes: Set[str] = set() try: with CoreSessionLocal() as session: - from api.v1.modules.a76.clients_and_providers.models import ClientProvider - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode - - for cp in session.query(ClientProvider).filter( - ClientProvider.tenant_id == tenant_id, - ClientProvider.company_id == company_id, - ).all(): - valid_client_ids.add(cp.id) - for r in session.query(RegimenPedimento).all(): - valid_regimes.add(r.code) - for pc in session.query(PedimentoCode).all(): - valid_pedimento_codes.add(pc.code) + valid_client_ids, valid_regimes, valid_pedimento_codes = load_pedimentos_fk_sets( + session, tenant_id, company_id + ) except Exception as e: - logger.error(f"Pedimentos import: failed to load FK sets: {e}") - return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"} + logger.error("Pedimentos import: failed to load FK sets: %s", e) + return {"status": "failed", "error": "No se pudo cargar catálogos"} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header, "pedimentos") - err = _validate_row_pedimento( + row_norm = _norm_row(row) + err = validate_row_pedimento( row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes ) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"Pedimentos import scan failed: {e}") + logger.error("Pedimentos import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=PED_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create. - """ - logger.info(f"Pedimentos import: starting commit for job {job_id}") +def scan_file(self, job_id: str, config: str = None): + logger.info("Pedimentos import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} - - # Reload FK sets for commit-time validation - valid_client_ids: Set[int] = set() - valid_regimes: Set[str] = set() - valid_pedimento_codes: Set[str] = set() try: with CoreSessionLocal() as session: - from api.v1.modules.a76.clients_and_providers.models import ClientProvider - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode - - for cp in session.query(ClientProvider).filter( - ClientProvider.tenant_id == tenant_id, - ClientProvider.company_id == company_id, - ).all(): - valid_client_ids.add(cp.id) - for r in session.query(RegimenPedimento).all(): - valid_regimes.add(r.code) - for pc in session.query(PedimentoCode).all(): - valid_pedimento_codes.add(pc.code) + valid_client_ids, valid_regimes, valid_pedimento_codes = load_pedimentos_fk_sets( + session, tenant_id, company_id + ) except Exception as e: - logger.error(f"Pedimentos import: failed to load FK sets: {e}") - return {"status": "failed", "error": str(e)} + logger.error("Pedimentos import: failed to load FK sets: %s", e) + return {"status": "failed", "error": "No se pudo cargar catálogos"} from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate @@ -450,114 +147,94 @@ def insert_valid_rows(self, job_id: str): skipped_missing_fk = 0 skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] - response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = _norm_row(row) + err = validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header, "pedimentos") - err = _validate_row_pedimento( - row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes - ) - if err: - skipped_invalid += 1 - skipped_details.append( - {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + data = row_to_pedimento_data(row_norm) + if "pedimento_dates" not in data or data.get("pedimento_dates") is None: + data["pedimento_dates"] = PedimentoDatesCreate( + entry_date=datetime.now(), + end_date=datetime.now(), ) - continue - - try: - data = _row_to_pedimentos_create(row_norm) - # Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL - if "pedimento_dates" not in data or data.get("pedimento_dates") is None: - data["pedimento_dates"] = PedimentoDatesCreate( - entry_date=datetime.now(), - end_date=datetime.now(), - ) - create_data = PedimentosCreate(**data) - PedimentosService.create(session, create_data, tenant_id, company_id) - inserted_count += 1 - except ValueError as ve: - if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): - skipped_duplicate += 1 - skipped_details.append({"line": i, "reason": str(ve)}) - else: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": str(ve)}) - except Exception as e: - logger.warning(f"Pedimentos import line {i}: {e}") + create_data = PedimentosCreate(**data) + PedimentosService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except ValueError as ve: + if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): + skipped_duplicate += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + else: skipped_invalid += 1 - skipped_details.append({"line": i, "reason": str(e)}) - - total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": skipped_missing_fk, - "skipped_duplicate": skipped_duplicate, - "skipped_details": skipped_details, - } + skipped_details.append({"line": i, "reason": str(ve)}) + except Exception as e: + logger.warning("Pedimentos import line %s: %s", i, e) + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(e)}) except Exception as e: - logger.error(f"Pedimentos import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("Pedimentos import task failed") return {"status": "failed", "error": str(e)} - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + return { + "status": "warning", "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + if inserted_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, } - return response + return { + "status": "finished", + "inserted": inserted_count, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("Pedimentos import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py new file mode 100644 index 00000000..6ea80cef --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_pedimento + +__all__ = ["validate_row_pedimento"] diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py new file mode 100644 index 00000000..2c3e381a --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/common.py @@ -0,0 +1,78 @@ +""" +Validaciones comunes de fila para import CSV de pedimentos (por tipo). +""" +from typing import Dict, Any, Optional, Set + +from ..common.common_validators import ( + check_required_max, + check_int_in_set, + check_in_set, + check_optional_max_length, + check_optional_decimal, + check_optional_in_set, + PEDIMENTO_CODE_MAX, + REGIMEN_MAX, +) + + +def validate_row_pedimento_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + for col, max_len in [ + ("AÑO", 2), + ("ADUANA", 3), + ("PATENTE", 4), + ("NUMERO", 7), + ]: + err = check_required_max(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_pedimento_fk( + row: Dict[str, Any], + line_num: int, + valid_client_ids: Set[int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], +) -> Optional[Dict[str, Any]]: + err = check_int_in_set(row, "CLIENTE_ID", valid_client_ids, line_num) + if err: + return err + err = check_in_set( + row, "CODIGO_PEDIMENTO", valid_pedimento_codes, line_num, + max_len=PEDIMENTO_CODE_MAX, catalog_name="código pedimento", + ) + if err: + return err + err = check_in_set( + row, "REGIMEN", valid_regimes, line_num, + max_len=REGIMEN_MAX, catalog_name="régimen", + ) + if err: + return err + return None + + +def validate_row_pedimento_optionals(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + err = check_optional_max_length(row, "ESTATUS", 30, line_num) + if err: + return err + for col in ("VALOR_USD", "PRECIO_PAGADO", "PESO_BRUTO", "TIPO_CAMBIO"): + err = check_optional_decimal(row, col, line_num) + if err: + return err + err = check_optional_in_set( + row, "TIPO_OPERACION", {"imp", "exp"}, line_num, + "Debe ser imp o exp", + ) + if err: + return err + err = check_optional_in_set( + row, "TIPO_PEDIMENTO", + {"normal", "consolidated", "complementary", "automobile"}, + line_num, + "Tipo no válido (normal, consolidated, complementary, automobile)", + ) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py new file mode 100644 index 00000000..4d6149a0 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/validators/create.py @@ -0,0 +1,37 @@ +""" +Punto de entrada de validación para import de una fila pedimento. +Recibe conjuntos FK (valid_client_ids, valid_regimes, valid_pedimento_codes) para validar referencias. +""" +from typing import Dict, Any, Optional, Set + +from .common import ( + validate_row_pedimento_required, + validate_row_pedimento_fk, + validate_row_pedimento_optionals, +) + + +def validate_row_pedimento( + row: Dict[str, Any], + line_num: int, + valid_client_ids: Set[int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], +) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de pedimentos. + Requeridos: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN (y FKs en catálogos). + Opcionales: ESTATUS, decimales, TIPO_OPERACION, TIPO_PEDIMENTO. + """ + err = validate_row_pedimento_required(row, line_num) + if err: + return err + err = validate_row_pedimento_fk( + row, line_num, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + return err + err = validate_row_pedimento_optionals(row, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py new file mode 100644 index 00000000..55e6cb8b --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for trailers) diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py new file mode 100644 index 00000000..ca5c6231 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/common_validators.py @@ -0,0 +1,39 @@ +""" +Validadores reutilizables para import CSV de trailers (longitudes, requerido). +""" +from typing import Dict, Any, Optional + +# Max lengths from Trailer model (a76.trailer) +MAX_LEN = { + "trailer_number": 20, + "ace_trailer_number": 10, + "trailer_type_key": 2, + "seal": 15, + "entity_code": 1, + "plate_number": 17, + "state": 30, + "country": 3, + "container_key": 3, +} + + +def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length( + row: Dict[str, Any], col: str, max_len: int, line_num: int, required: bool = False +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py new file mode 100644 index 00000000..62d3ec97 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/mappers.py @@ -0,0 +1,35 @@ +""" +Mapeo fila CSV → datos para Trailer. +""" +from typing import Dict, Any, Optional + +from .common_validators import MAX_LEN + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_trailer_data(row_norm: Dict[str, Any]) -> Dict[str, Any]: + """Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row.""" + trailer_number = _str_or_none(row_norm.get("NUMERO TRAILER"), MAX_LEN["trailer_number"]) + if not trailer_number: + return {} + return { + "trailer_number": trailer_number, + "ace_trailer_number": _str_or_none(row_norm.get("CLAVE ACE"), MAX_LEN["ace_trailer_number"]), + "trailer_type_key": _str_or_none(row_norm.get("TIPO TRAILER"), MAX_LEN["trailer_type_key"]), + "seal": _str_or_none(row_norm.get("PRECINTO"), MAX_LEN["seal"]), + "entity_code": _str_or_none(row_norm.get("CODIGO ENTIDAD"), MAX_LEN["entity_code"]), + "plate_number": _str_or_none(row_norm.get("PLACAS"), MAX_LEN["plate_number"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "container_key": _str_or_none(row_norm.get("CLAVE CONTENEDOR"), MAX_LEN["container_key"]), + } diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py index 73c43aad..448788ee 100644 --- a/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py @@ -1,30 +1,35 @@ """ Tareas Celery para importación CSV de Trailers y Cajas. Flujo: scan_file (validación) → insert_valid_rows (commit). -Upsert por trailer_number usando TrailerService. +Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe). """ -import os -import base64 import csv import json import logging -import re -import unicodedata +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses from .template_config import row_from_template +from .validators import validate_row_trailer +from .common.mappers import row_to_trailer_data logger = logging.getLogger(__name__) +JOB_TYPE = "trl" + +# Para routes.py TRL_IMPORT_FILE_PREFIX = "trl_import_file:" TRL_IMPORT_META_PREFIX = "trl_import_meta:" TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:" TRL_IMPORT_STATUS_PREFIX = "trl_import_status:" -TRL_IMPORT_REDIS_TTL = 3600 +TRL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _get_redis(): @@ -33,80 +38,6 @@ def _get_redis(): return redis.Redis.from_url(url, decode_responses=False) -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Trailers import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"trl_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Trailers import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{TRL_IMPORT_FILE_PREFIX}{job_id}", - f"{TRL_IMPORT_META_PREFIX}{job_id}", - f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Trailers import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -# Max lengths from Trailer model (a76.trailer) -_MAX = { - "trailer_number": 20, - "ace_trailer_number": 10, - "trailer_type_key": 2, - "seal": 15, - "entity_code": 1, - "plate_number": 17, - "state": 30, - "country": 3, - "container_key": 3, -} - - def _dedupe_headers(headers: List[str]) -> List[str]: counts: Dict[str, int] = {} unique: List[str] = [] @@ -114,101 +45,33 @@ def _dedupe_headers(headers: List[str]) -> List[str]: name = str(header or "").strip() or "COL" count = counts.get(name, 0) + 1 counts[name] = count - if count == 1: - unique.append(name) - else: - unique.append(f"{name} {count}") + unique.append(name if count == 1 else f"{name} {count}") return unique -def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Trailer. Retorna error dict o None.""" - trailer_number = (row.get("NUMERO TRAILER") or "").strip() - if not trailer_number: - return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"} - if len(trailer_number) > _MAX["trailer_number"]: - return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"} - - for col, max_len in [ - ("CLAVE ACE", _MAX["ace_trailer_number"]), - ("TIPO TRAILER", _MAX["trailer_type_key"]), - ("PRECINTO", _MAX["seal"]), - ("CODIGO ENTIDAD", _MAX["entity_code"]), - ("PLACAS", _MAX["plate_number"]), - ("ESTADO", _MAX["state"]), - ("PAIS", _MAX["country"]), - ("CLAVE CONTENEDOR", _MAX["container_key"]), - ]: - val = (row.get(col) or "").strip() - if val and len(val) > max_len: - return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} - - return None - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: - """Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row.""" - trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"]) - if not trailer_number: - return {} - return { - "trailer_number": trailer_number, - "ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]), - "trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]), - "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), - "entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]), - "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), - "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), - "country": _str_or_none(row.get("PAIS"), _MAX["country"]), - "container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]), - } - - def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Trailers import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Trailers import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: with open(file_path, "r", encoding="utf-8-sig") as f: total_rows = sum(1 for _ in f) - 1 except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Trailers import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: with open(file_path, "r", encoding="utf-8-sig") as f_in, open( @@ -232,51 +95,27 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, if progress_callback and i % 500 == 0: progress_callback(i, total_rows, error_count) - row_norm = row_from_template(row, normalize_header) - err = _validate_row_trailer(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_trailer(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"Trailers import scan failed: {e}") + logger.error("Trailers import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Trailers import: failed to store error lines: {e}") - - result = { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - return result + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def run_scan_sync(job_id: str) -> Dict[str, Any]: @@ -289,19 +128,16 @@ def run_scan_sync(job_id: str) -> Dict[str, Any]: ex=TRL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") + logger.warning("Trailers import: failed to store scan status in Redis: %s", e) return result @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): - logger.info(f"Trailers import: starting scan for job {job_id}") + logger.info("Trailers import: starting scan for job %s", job_id) def on_progress(current: int, total: int, errors: int) -> None: - self.update_state( - state="PROGRESS", - meta={"current": current, "total": total, "errors": errors}, - ) + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) result = _do_scan(job_id, progress_callback=on_progress) try: @@ -312,58 +148,27 @@ def scan_file(self, job_id: str, config: str = None): ex=TRL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") + logger.warning("Trailers import: failed to store scan status in Redis: %s", e) return result def _do_commit(job_id: str) -> Dict[str, Any]: - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Trailers import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Trailers import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Trailers import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.transportation.trailers.services import TrailerService from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO @@ -374,6 +179,7 @@ def _do_commit(job_id: str) -> Dict[str, Any]: skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: @@ -396,21 +202,20 @@ def _do_commit(job_id: str) -> Dict[str, Any]: if i in error_lines: continue - row_norm = row_from_template(row, normalize_header) - err = _validate_row_trailer(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_trailer(row_norm, i) if err: skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", - "invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) + tn = (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-" + skipped_details.append({ + "line": i, + "trailer_number": tn, + "invoice": tn, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) continue - data = _row_to_trailer_dto(row_norm, tenant_id, company_id) + data = row_to_trailer_data(row_norm) if not data or not data.get("trailer_number"): skipped_invalid += 1 continue @@ -418,14 +223,12 @@ def _do_commit(job_id: str) -> Dict[str, Any]: tn = data["trailer_number"] if tn in seen_keys_in_file: skipped_duplicate += 1 - skipped_details.append( - { - "line": i, - "trailer_number": tn, - "invoice": tn, - "reason": "Clave duplicada en el archivo (se usa la primera)", - } - ) + skipped_details.append({ + "line": i, + "trailer_number": tn, + "invoice": tn, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) continue seen_keys_in_file[tn] = i @@ -442,34 +245,33 @@ def _do_commit(job_id: str) -> Dict[str, Any]: except Exception as db_err: session.rollback() skipped_invalid += 1 - skipped_details.append( - {"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)} - ) + skipped_details.append({ + "line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err), + }) continue try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"Trailers import DB error: {db_err}") + logger.error("Trailers import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} except Exception as e: - logger.error(f"Trailers import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("Trailers import task failed") return {"status": "failed", "error": str(e)} + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Trailers import cleanup failed: {cleanup_err}") + r = _get_redis() + r.delete(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}") + except Exception as e: + logger.warning("Trailers import: failed to delete status key: %s", e) total_ok = inserted_count + updated_count if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: @@ -515,13 +317,13 @@ def run_commit_sync(job_id: str) -> Dict[str, Any]: ex=TRL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") + logger.warning("Trailers import: failed to store commit status in Redis: %s", e) return result @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): - logger.info(f"Trailers import: starting commit for job {job_id}") + logger.info("Trailers import: starting commit for job %s", job_id) result = _do_commit(job_id) try: r = _get_redis() @@ -531,5 +333,5 @@ def insert_valid_rows(self, job_id: str): ex=TRL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") + logger.warning("Trailers import: failed to store commit status in Redis: %s", e) return result diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py new file mode 100644 index 00000000..406c2f15 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_trailer + +__all__ = ["validate_row_trailer"] diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py new file mode 100644 index 00000000..d10b3760 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/common.py @@ -0,0 +1,32 @@ +""" +Validaciones comunes de fila para import CSV de trailers. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + MAX_LEN, + check_required, + check_max_length, +) + + +def validate_row_trailer_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_required(row, "NUMERO TRAILER", MAX_LEN["trailer_number"], line_num) + + +def validate_row_trailer_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("CLAVE ACE", MAX_LEN["ace_trailer_number"]), + ("TIPO TRAILER", MAX_LEN["trailer_type_key"]), + ("PRECINTO", MAX_LEN["seal"]), + ("CODIGO ENTIDAD", MAX_LEN["entity_code"]), + ("PLACAS", MAX_LEN["plate_number"]), + ("ESTADO", MAX_LEN["state"]), + ("PAIS", MAX_LEN["country"]), + ("CLAVE CONTENEDOR", MAX_LEN["container_key"]), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py new file mode 100644 index 00000000..d01491b5 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/validators/create.py @@ -0,0 +1,22 @@ +""" +Punto de entrada de validación para import de una fila trailer. +""" +from typing import Dict, Any, Optional + +from .common import ( + validate_row_trailer_required, + validate_row_trailer_lengths, +) + + +def validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de trailers. + """ + err = validate_row_trailer_required(row, line_num) + if err: + return err + err = validate_row_trailer_lengths(row, line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py new file mode 100644 index 00000000..d7025515 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for us_tariff_fractions) diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py new file mode 100644 index 00000000..80af4417 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/common_validators.py @@ -0,0 +1,62 @@ +""" +Validadores reutilizables para import CSV de fracciones arancelarias americanas. +""" +from decimal import Decimal +from typing import Dict, Any, Optional + +CODE_MAX = 16 +PREFIX_MAX = 10 +UNIT_MAX = 10 +TYPE_MAX = 10 + + +def normalize_code(raw: Optional[str]) -> str: + """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" + if not raw: + return "" + s = str(raw).strip().replace(".", "").replace("-", "") + return s[:CODE_MAX] if len(s) > CODE_MAX else s + + +def check_required_code(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + code_raw = (row.get(col) or "").strip() + if not code_raw: + return {"line": line_num, "col": col, "msg": "Requerido"} + code_norm = normalize_code(code_raw) + if not code_norm: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(code_norm) > CODE_MAX: + return {"line": line_num, "col": col, "msg": f"Máximo {CODE_MAX} caracteres"} + return None + + +def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def parse_float_min_zero(val: Any) -> Optional[float]: + if val is None or str(val).strip() == "": + return None + try: + v = float(str(val).strip().replace(",", ".")) + return v if v >= 0 else None + except (ValueError, TypeError): + return None + + +def check_optional_decimal_min_zero(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or not str(val).strip(): + return None + try: + v = float(str(val).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": col, "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": col, "msg": "Debe ser un número"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py new file mode 100644 index 00000000..7c87b773 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/common/mappers.py @@ -0,0 +1,45 @@ +""" +Mapeo fila CSV → datos para USTariffFraction. +""" +from decimal import Decimal +from typing import Dict, Any, Optional + +from .common_validators import ( + normalize_code, + parse_float_min_zero, +) + +MAX_LEN = {"prefix": 10, "unit_of_measure": 10, "type_code": 10} + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_us_tariff_fraction_data( + row_norm: Dict[str, Any], tenant_id: int, company_id: int +) -> Dict[str, Any]: + """Build dict for USTariffFraction model.""" + code = normalize_code(row_norm.get("FRACCION_ARANCELARIA")) + if not code: + return {} + fixed_cost_raw = parse_float_min_zero(row_norm.get("ADVALOREM_DLLS")) + fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None + return { + "tenant_id": tenant_id, + "company_id": company_id, + "code": code, + "prefix": _str_or_none(row_norm.get("PREFIJO"), MAX_LEN["prefix"]), + "type_code": _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), MAX_LEN["type_code"]), + "ad_valorem": parse_float_min_zero(row_norm.get("ADVALOREM_PCT")), + "fixed_cost": fixed_cost, + "unit_of_measure": _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), MAX_LEN["unit_of_measure"]), + "description": _str_or_none(row_norm.get("DESCRIPCION")), + } diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py index 5d58a79c..339599f3 100644 --- a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py @@ -1,474 +1,224 @@ """ Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions). -Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from decimal import Decimal +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader from .template_config import row_from_template +from .validators import validate_row_us_tariff_fraction +from .common.mappers import row_to_us_tariff_fraction_data logger = logging.getLogger(__name__) +JOB_TYPE = "fa" +TEMPLATE_ID = "us_tariff_fractions" + +# Para routes.py FA_IMPORT_FILE_PREFIX = "fa_import_file:" FA_IMPORT_META_PREFIX = "fa_import_meta:" FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:" -FA_IMPORT_REDIS_TTL = 3600 # 1 hour - -TEMPLATE_ID = "us_tariff_fractions" +FA_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL -def _get_redis(): - import redis - url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) - return redis.Redis.from_url(url, decode_responses=False) +def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: + return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"FA import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"fa_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"FA import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{FA_IMPORT_FILE_PREFIX}{job_id}", - f"{FA_IMPORT_META_PREFIX}{job_id}", - f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"FA import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def _normalize_code(raw: Optional[str]) -> str: - """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" - if not raw: - return "" - s = str(raw).strip().replace(".", "").replace("-", "") - return s[:16] if len(s) > 16 else s - - -def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Fracción Americana. Retorna error dict o None.""" - code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip() - if not code_raw: - return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} - code_norm = _normalize_code(code_raw) - if not code_norm: - return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} - if len(code_norm) > 16: - return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"} - - prefix_raw = (row.get("PREFIJO") or "").strip() - if prefix_raw and len(prefix_raw) > 10: - return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"} - - um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip() - if um_raw and len(um_raw) > 10: - return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"} - - tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip() - if tipo_raw and len(tipo_raw) > 10: - return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"} - - adv_pct = row.get("ADVALOREM_PCT") - if adv_pct is not None and str(adv_pct).strip(): - try: - v = float(str(adv_pct).strip().replace(",", ".")) - if v < 0: - return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"} - except ValueError: - return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"} - - adv_dlls = row.get("ADVALOREM_DLLS") - if adv_dlls is not None and str(adv_dlls).strip(): - try: - v = float(str(adv_dlls).strip().replace(",", ".")) - if v < 0: - return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"} - except ValueError: - return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"} - - return None - - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"FA import: starting scan for job {job_id}") - - file_path = _ensure_worker_has_file_from_redis(job_id) +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "FA import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "FA import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"FA import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) - for i, row in enumerate(reader, start=1): - if i % 500 == 0: - self.update_state( - state="PROGRESS", - meta={"current": i, "total": total_rows, "errors": error_count}, - ) - - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_us_tariff_fraction(row_norm, i) + row_norm = _norm_row(row) + err = validate_row_us_tariff_fraction(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"FA import scan failed: {e}") + logger.error("FA import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=FA_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"FA import: failed to store error lines in Redis: {e}") - - return { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - - -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _parse_float(val: Any) -> Optional[float]: - if val is None or str(val).strip() == "": - return None - try: - return float(str(val).strip().replace(",", ".")) - except (ValueError, TypeError): - return None + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code). - """ - logger.info(f"FA import: starting commit for job {job_id}") +def scan_file(self, job_id: str, config: str = None): + logger.info("FA import: starting scan for job %s", job_id) - file_path = _ensure_worker_has_file_from_redis(job_id) + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) + + return _do_scan(job_id, progress_callback=on_progress) + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "FA import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "FA import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"FA import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction inserted_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] - response = None + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue + row_norm = _norm_row(row) + err = validate_row_us_tariff_fraction(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue - row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) - err = _validate_row_us_tariff_fraction(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue + data = row_to_us_tariff_fraction_data(row_norm, tenant_id, company_id) + if not data or not data.get("code"): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA vacío"}) + continue - code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA")) - if not code: - skipped_invalid += 1 - skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"}) - continue - - prefix = _str_or_none(row_norm.get("PREFIJO"), 10) - unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10) - description = _str_or_none(row_norm.get("DESCRIPCION")) - type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10) - ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT")) - fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS")) - fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None - - existing = ( - session.query(USTariffFraction) - .filter( - USTariffFraction.tenant_id == tenant_id, - USTariffFraction.company_id == company_id, - USTariffFraction.code == code, - ) - .first() + existing = ( + session.query(USTariffFraction) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + USTariffFraction.code == data["code"], ) - - if existing: - existing.prefix = prefix - existing.type_code = type_code - existing.ad_valorem = ad_valorem - existing.fixed_cost = fixed_cost - existing.unit_of_measure = unit_of_measure - existing.description = description - session.add(existing) - inserted_count += 1 - else: - new_row = USTariffFraction( - tenant_id=tenant_id, - company_id=company_id, - code=code, - prefix=prefix, - type_code=type_code, - ad_valorem=ad_valorem, - fixed_cost=fixed_cost, - unit_of_measure=unit_of_measure, - description=description, - ) - session.add(new_row) - inserted_count += 1 + .first() + ) + if existing: + existing.prefix = data.get("prefix") + existing.type_code = data.get("type_code") + existing.ad_valorem = data.get("ad_valorem") + existing.fixed_cost = data.get("fixed_cost") + existing.unit_of_measure = data.get("unit_of_measure") + existing.description = data.get("description") + session.add(existing) + else: + new_row = USTariffFraction(**data) + session.add(new_row) + inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"FA import DB error: {db_err}") + logger.error("FA import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} - total_skipped = skipped_invalid - if inserted_count == 0 and total_skipped > 0: - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - "message": f"No se insertaron registros. {total_skipped} rechazados.", - } - elif inserted_count == 0: - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - else: - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_details": skipped_details, - } - except Exception as e: - logger.error(f"FA import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("FA import task failed") return {"status": "failed", "error": str(e)} - try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - meta_path_clean = file_path.replace(".csv", ".meta.json") - if os.path.exists(meta_path_clean): - os.remove(meta_path_clean) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"FA import cleanup failed: {cleanup_err}") + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) - if response is None: - response = { - "status": "failed", - "error": "Error inesperado", + if inserted_count == 0 and skipped_invalid > 0: + return { + "status": "warning", "inserted": 0, + "updated": 0, "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid} rechazados.", + } + if inserted_count == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, "skipped_missing_fk": 0, "skipped_details": skipped_details, } - return response + return { + "status": "finished", + "inserted": inserted_count, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": 0, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info("FA import: starting commit for job %s", job_id) + return _do_commit(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py new file mode 100644 index 00000000..b6f23b72 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_us_tariff_fraction + +__all__ = ["validate_row_us_tariff_fraction"] diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py new file mode 100644 index 00000000..1f35b596 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/common.py @@ -0,0 +1,40 @@ +""" +Validaciones comunes de fila para import CSV de fracciones arancelarias americanas. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + check_required_code, + check_optional_max_length, + check_optional_decimal_min_zero, + PREFIX_MAX, + UNIT_MAX, + TYPE_MAX, +) + + +def validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de fracción arancelaria americana. + FRACCION_ARANCELARIA requerida (max 16); PREFIJO, UNIDAD_DE_MEDIDA, TIPO_DE_ADVALOREM opcionales (max 10); + ADVALOREM_PCT y ADVALOREM_DLLS opcionales numéricos >= 0. + """ + err = check_required_code(row, "FRACCION_ARANCELARIA", line_num) + if err: + return err + err = check_optional_max_length(row, "PREFIJO", PREFIX_MAX, line_num) + if err: + return err + err = check_optional_max_length(row, "UNIDAD_DE_MEDIDA", UNIT_MAX, line_num) + if err: + return err + err = check_optional_max_length(row, "TIPO_DE_ADVALOREM", TYPE_MAX, line_num) + if err: + return err + err = check_optional_decimal_min_zero(row, "ADVALOREM_PCT", line_num) + if err: + return err + err = check_optional_decimal_min_zero(row, "ADVALOREM_DLLS", line_num) + if err: + return err + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py new file mode 100644 index 00000000..c553cd21 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/validators/create.py @@ -0,0 +1,6 @@ +""" +Punto de entrada de validación para import de una fila fracción arancelaria americana. +""" +from .common import validate_row_us_tariff_fraction + +__all__ = ["validate_row_us_tariff_fraction"] diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py new file mode 100644 index 00000000..0dd687d4 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/__init__.py @@ -0,0 +1 @@ +# common_validators, mappers (no fk_loader for vehicles) diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py new file mode 100644 index 00000000..2f4fb5ae --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/common_validators.py @@ -0,0 +1,104 @@ +""" +Validadores reutilizables para import CSV de vehículos (longitudes, decimal, fecha aseguradora). +""" +import re +from decimal import Decimal +from typing import Dict, Any, Optional + +# Max lengths from Vehicle model (a76.vehicle) +MAX_LEN = { + "vehicle_key": 14, + "ace_vehicle_key": 10, + "transporter_key": 23, + "transport_identifier": 30, + "transport_type": 2, + "entity_code": 1, + "transponder_number": 16, + "dot_number": 8, + "plate_number": 17, + "city": 30, + "state": 30, + "country": 3, + "seal": 49, + "insurance_company_name": 30, + "insurance_number": 20, + "series": 30, +} + + +def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + return {"line": line_num, "col": col, "msg": "Requerido"} + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def check_max_length( + row: Dict[str, Any], col: str, max_len: int, line_num: int, required: bool = False +) -> Optional[Dict[str, Any]]: + val = (row.get(col) or "").strip() + if not val: + if required: + return {"line": line_num, "col": col, "msg": "Requerido"} + return None + if len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + return None + + +def parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + s = str(val).strip().replace(",", "") + return Decimal(s) + except Exception: + return None + + +def parse_insurance_date(val: Any) -> Optional[int]: + """Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD.""" + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if not s: + return None + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + pass + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + pass + break + return None + + +def check_optional_decimal(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or (isinstance(val, str) and not val.strip()): + return None + if parse_decimal(val) is None: + return {"line": line_num, "col": col, "msg": "Debe ser numérico"} + return None + + +def check_optional_insurance_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]: + val = row.get(col) + if val is None or (isinstance(val, str) and not val.strip()): + return None + if parse_insurance_date(val) is None: + return {"line": line_num, "col": col, "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py new file mode 100644 index 00000000..882f9099 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/mappers.py @@ -0,0 +1,47 @@ +""" +Mapeo fila CSV → datos para Vehicle. +""" +from typing import Dict, Any, Optional + +from .common_validators import ( + MAX_LEN, + parse_decimal, + parse_insurance_date, +) + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def row_to_vehicle_data(row_norm: Dict[str, Any]) -> Dict[str, Any]: + """Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row.""" + vehicle_key = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["vehicle_key"]) + if not vehicle_key: + return {} + return { + "vehicle_key": vehicle_key, + "ace_vehicle_key": _str_or_none(row_norm.get("CLAVE ACE"), MAX_LEN["ace_vehicle_key"]), + "transporter_key": _str_or_none(row_norm.get("CLAVE TRANSPORTE"), MAX_LEN["transporter_key"]), + "series": _str_or_none(row_norm.get("VIN"), MAX_LEN["series"]), + "transport_type": _str_or_none(row_norm.get("TIPO TRANSPORTE"), MAX_LEN["transport_type"]), + "entity_code": _str_or_none(row_norm.get("CODIGO DE ENTIDAD"), MAX_LEN["entity_code"]), + "transponder_number": _str_or_none(row_norm.get("TRANSPONDEDOR"), MAX_LEN["transponder_number"]), + "dot_number": _str_or_none(row_norm.get("NUMERO DOT"), MAX_LEN["dot_number"]), + "plate_number": _str_or_none(row_norm.get("PLACAS"), MAX_LEN["plate_number"]), + "city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]), + "state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]), + "country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]), + "seal": _str_or_none(row_norm.get("PRECINTO"), MAX_LEN["seal"]), + "insurance_company_name": _str_or_none(row_norm.get("EMPRESA ASEGURADORA"), MAX_LEN["insurance_company_name"]), + "insurance_number": _str_or_none(row_norm.get("NUM. ASEGURADORA"), MAX_LEN["insurance_number"]), + "insurance_amount": parse_decimal(row_norm.get("MONTO ASEGURADO") or row_norm.get("MONTO")), + "insurance_date": parse_insurance_date(row_norm.get("FECHA DE ASEGURADORA") or row_norm.get("FECHA ASEGURADORA")), + } diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py index 533193e5..1bb4ac1e 100644 --- a/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py @@ -1,31 +1,35 @@ """ Tareas Celery para importación CSV de Vehículos (Transportes). Flujo: scan_file (validación) → insert_valid_rows (commit). -Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key. +Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). """ -import os -import base64 -import csv import json import logging -import re -import unicodedata -from decimal import Decimal +import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal -from core.paths import layout_path +from ..common import storage as common_storage +from ..common import normalize as common_normalize +from ..common import meta as common_meta +from ..common import responses as common_responses +from ..common import csv_reader as common_csv_reader from .template_config import row_from_template +from .validators import validate_row_vehicle +from .common.mappers import row_to_vehicle_data logger = logging.getLogger(__name__) +JOB_TYPE = "veh" + +# Para routes.py VEHL_IMPORT_FILE_PREFIX = "veh_import_file:" VEHL_IMPORT_META_PREFIX = "veh_import_meta:" VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:" VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:" -VEHL_IMPORT_REDIS_TTL = 3600 +VEHL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _get_redis(): @@ -34,283 +38,59 @@ def _get_redis(): return redis.Redis.from_url(url, decode_responses=False) -def _worker_upload_dir() -> str: - return layout_path("imports", "temp") - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - r = _get_redis() - data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}") - if not data: - return None - try: - raw = base64.b64decode(data) - except Exception as e: - logger.warning(f"Vehicles import: failed to decode file from Redis: {e}") - return None - upload_dir = _worker_upload_dir() - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"veh_{job_id}.csv") - with open(file_path, "wb") as f: - f.write(raw) - return file_path - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - r = _get_redis() - data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}") - if not data: - return False - try: - meta = json.loads(data.decode("utf-8")) - except Exception as e: - logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}") - return False - meta_path = file_path.replace(".csv", ".meta.json") - with open(meta_path, "w", encoding="utf-8") as f: - json.dump(meta, f) - return True - - -def _delete_import_from_redis(job_id: str) -> None: - try: - r = _get_redis() - r.delete( - f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", - f"{VEHL_IMPORT_META_PREFIX}{job_id}", - f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to delete Redis keys: {e}") - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -# Max lengths from Vehicle model (a76.vehicle) -_MAX = { - "vehicle_key": 14, - "ace_vehicle_key": 10, - "transporter_key": 23, - "transport_identifier": 30, - "transport_type": 2, - "entity_code": 1, - "transponder_number": 16, - "dot_number": 8, - "plate_number": 17, - "city": 30, - "state": 30, - "country": 3, - "seal": 49, - "insurance_company_name": 30, - "insurance_number": 20, - "series": 30, -} - - -def _parse_decimal(val: Any) -> Optional[Decimal]: - if val is None or (isinstance(val, str) and not val.strip()): - return None - try: - s = str(val).strip().replace(",", "") - return Decimal(s) - except Exception: - return None - - -def _parse_insurance_date(val: Any) -> Optional[int]: - """Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD.""" - if val is None or (isinstance(val, str) and not val.strip()): - return None - s = str(val).strip() - if not s: - return None - # Already integer-like - if re.match(r"^\d{8}$", s): - try: - return int(s) - except ValueError: - pass - # Try DD/MM/YYYY or similar - for sep in ["/", "-", "."]: - if sep in s: - parts = s.split(sep) - if len(parts) == 3: - try: - a, b, c = [p.strip() for p in parts] - if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year - return int(c) * 10000 + int(b) * 100 + int(a) - if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year - return int(a) * 10000 + int(b) * 100 + int(c) - except (ValueError, TypeError): - pass - break - return None - - -def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: - """Valida una fila para Vehículo. Retorna error dict o None.""" - clave = (row.get("CLAVE") or "").strip() - if not clave: - return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} - if len(clave) > _MAX["vehicle_key"]: - return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"} - - # Optional fields: max lengths only - for col, max_len in [ - ("CLAVE ACE", _MAX["ace_vehicle_key"]), - ("CLAVE TRANSPORTE", _MAX["transporter_key"]), - ("VIN", _MAX["series"]), - ("TIPO TRANSPORTE", _MAX["transport_type"]), - ("CODIGO DE ENTIDAD", _MAX["entity_code"]), - ("TRANSPONDEDOR", _MAX["transponder_number"]), - ("NUMERO DOT", _MAX["dot_number"]), - ("PLACAS", _MAX["plate_number"]), - ("CIUDAD", _MAX["city"]), - ("ESTADO", _MAX["state"]), - ("PAIS", _MAX["country"]), - ("PRECINTO", _MAX["seal"]), - ("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]), - ("NUM. ASEGURADORA", _MAX["insurance_number"]), - ]: - val = (row.get(col) or "").strip() - if val and len(val) > max_len: - return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} - - # MONTO ASEGURADO: must be numeric if present - monto = row.get("MONTO ASEGURADO") or row.get("MONTO") - if monto is not None and str(monto).strip(): - if _parse_decimal(monto) is None: - return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"} - - # FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None) - # Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either - # reject or set None. We reject invalid date to keep data quality. - fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA") - if fecha is not None and str(fecha).strip(): - if _parse_insurance_date(fecha) is None: - return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} - - return None - - -def _do_scan( - job_id: str, - progress_callback: Optional[Any] = None, -) -> Dict[str, Any]: - """ - Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync. - progress_callback(current, total, error_count) opcional. - """ - file_path = _ensure_worker_has_file_from_redis(job_id) +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import") - error_dir = layout_path("imports", "errors") - os.makedirs(error_dir, exist_ok=True) - error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - total_rows = 0 try: - with open(file_path, "r", encoding="utf-8-sig") as f: - total_rows = sum(1 for _ in f) - 1 + total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} - meta_path = file_path.replace(".csv", ".meta.json") - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - except Exception as e: - logger.warning(f"Vehicles import: failed to read meta: {e}") - - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] try: - with open(file_path, "r", encoding="utf-8-sig") as f_in, open( - error_path, "w", encoding="utf-8" - ) as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in common_csv_reader.iter_csv_rows(file_path): if progress_callback and i % 500 == 0: progress_callback(i, total_rows, error_count) - row_norm = row_from_template(row, normalize_header) - err = _validate_row_vehicle(row_norm, i) + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_vehicle(row_norm, i) if err: error_count += 1 + error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: - errors_detail.append( - {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} - ) + errors_detail.append({ + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + }) processed_rows += 1 + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: - logger.error(f"Vehicles import scan failed: {e}") + logger.error("Vehicles import scan failed: %s", e) return {"status": "failed", "error": str(e)} - error_lines_list = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - except Exception: - pass - if error_lines_list: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", - json.dumps(error_lines_list).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, - ) - except Exception as e: - logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}") - - result = { - "status": "waiting_confirmation", - "job_id": job_id, - "total_rows": processed_rows, - "error_count": error_count, - "valid_rows": processed_rows - error_count, - "errors": errors_detail, - } - return result + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def run_scan_sync(job_id: str) -> Dict[str, Any]: - """ - Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis. - Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery. - """ result = _do_scan(job_id, progress_callback=None) try: r = _get_redis() @@ -320,23 +100,16 @@ def run_scan_sync(job_id: str) -> Dict[str, Any]: ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") + logger.warning("Vehicles import: failed to store scan status in Redis: %s", e) return result @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): - """ - Fase 1: Leer CSV, validar filas, escribir errores en JSONL. - Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. - """ - logger.info(f"Vehicles import: starting scan for job {job_id}") + logger.info("Vehicles import: starting scan for job %s", job_id) def on_progress(current: int, total: int, errors: int) -> None: - self.update_state( - state="PROGRESS", - meta={"current": current, "total": total, "errors": errors}, - ) + self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) result = _do_scan(job_id, progress_callback=on_progress) try: @@ -347,99 +120,27 @@ def scan_file(self, job_id: str, config: str = None): ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") + logger.warning("Vehicles import: failed to store scan status in Redis: %s", e) return result -def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: - if val is None: - return None - s = str(val).strip() - if not s: - return None - if max_len and len(s) > max_len: - return s[:max_len] - return s - - -def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]: - """Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row.""" - vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"]) - if not vehicle_key: - return {} - data = { - "vehicle_key": vehicle_key, - "ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]), - "transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]), - "series": _str_or_none(row.get("VIN"), _MAX["series"]), - "transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]), - "entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]), - "transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]), - "dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]), - "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), - "city": _str_or_none(row.get("CIUDAD"), _MAX["city"]), - "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), - "country": _str_or_none(row.get("PAIS"), _MAX["country"]), - "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), - "insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]), - "insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]), - "insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")), - "insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")), - } - return data - - def _do_commit(job_id: str) -> Dict[str, Any]: - """ - Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync. - """ - file_path = _ensure_worker_has_file_from_redis(job_id) + file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import") if not file_path: - alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv") + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): - return { - "status": "failed", - "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", - } + return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: - _ensure_worker_has_meta_from_redis(job_id, file_path) + common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import") - error_dir = layout_path("imports", "errors") - error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - error_lines = set() try: - r = _get_redis() - raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}") - if raw: - error_lines = set(json.loads(raw.decode("utf-8"))) - except Exception as e: - logger.debug(f"Vehicles import: could not load error lines from Redis: {e}") - if not error_lines and os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - error_lines.add(err["line"]) - except Exception: - pass - - meta_path = file_path.replace(".csv", ".meta.json") - tenant_id = None - company_id = None - meta = {} - if os.path.exists(meta_path): - try: - with open(meta_path, "r", encoding="utf-8") as f: - meta = json.load(f) or {} - tenant_id = meta.get("tenant_id") - company_id = meta.get("company_id") - except Exception: - pass - - if not tenant_id or not company_id: - return {"status": "failed", "error": "Falta contexto (tenant/company)"} + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} from api.v1.modules.a76.transportation.vehicles.services import VehicleService from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO @@ -450,91 +151,84 @@ def _do_commit(job_id: str) -> Dict[str, Any]: skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] seen_keys_in_file: Dict[str, int] = {} + meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) + for i, row in common_csv_reader.iter_csv_rows(file_path): + if i in error_lines: + continue + + row_norm = row_from_template(row, common_normalize.normalize_header) + err = validate_row_vehicle(row_norm, i) + if err: + skipped_invalid += 1 + vk = (row_norm.get("CLAVE") or "").strip()[:14] or "-" + skipped_details.append({ + "line": i, + "vehicle_key": vk, + "invoice": vk, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + }) + continue + + data = row_to_vehicle_data(row_norm) + if not data or not data.get("vehicle_key"): + skipped_invalid += 1 + continue + + vk = data["vehicle_key"] + if vk in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append({ + "line": i, + "vehicle_key": vk, + "invoice": vk, + "reason": "Clave duplicada en el archivo (se usa la primera)", + }) + continue + seen_keys_in_file[vk] = i + + existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, normalize_header) - err = _validate_row_vehicle(row_norm, i) - if err: - skipped_invalid += 1 - skipped_details.append( - { - "line": i, - "vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-", - "invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-", - "reason": f"{err.get('col', '')}: {err.get('msg', '')}", - } - ) - continue - - data = _row_to_vehicle_dto(row_norm) - if not data or not data.get("vehicle_key"): - skipped_invalid += 1 - continue - - vk = data["vehicle_key"] - if vk in seen_keys_in_file: - skipped_duplicate += 1 - skipped_details.append( - {"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"} - ) - continue - seen_keys_in_file[vk] = i - - existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) - try: - if existing: - update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) - VehicleService.update(session, vk, tenant_id, update_data, company_id) - updated_count += 1 - else: - create_data = VehicleCreateDTO(**data) - VehicleService.create(session, create_data, tenant_id, company_id) - inserted_count += 1 - except Exception as db_err: - session.rollback() - skipped_invalid += 1 - skipped_details.append( - {"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)} - ) - continue + if existing: + update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) + VehicleService.update(session, vk, tenant_id, update_data, company_id) + updated_count += 1 + else: + create_data = VehicleCreateDTO(**data) + VehicleService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append({ + "line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err), + }) + continue try: session.commit() except Exception as db_err: session.rollback() - logger.error(f"Vehicles import DB error: {db_err}") + logger.error("Vehicles import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} except Exception as e: - logger.error(f"Vehicles import task failed: {e}") - import traceback - logger.error(traceback.format_exc()) + logger.exception("Vehicles import task failed") return {"status": "failed", "error": str(e)} + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) try: - if file_path and os.path.exists(file_path): - os.remove(file_path) - if os.path.exists(error_path): - os.remove(error_path) - if os.path.exists(meta_path): - os.remove(meta_path) - _delete_import_from_redis(job_id) - except Exception as cleanup_err: - logger.warning(f"Vehicles import cleanup failed: {cleanup_err}") + r = _get_redis() + r.delete(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}") + except Exception as e: + logger.warning("Vehicles import: failed to delete status key: %s", e) total_ok = inserted_count + updated_count if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: @@ -571,10 +265,6 @@ def _do_commit(job_id: str) -> Dict[str, Any]: def run_commit_sync(job_id: str) -> Dict[str, Any]: - """ - Ejecuta el commit en el proceso actual y guarda el resultado en Redis. - Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery. - """ result = _do_commit(job_id) try: r = _get_redis() @@ -584,16 +274,13 @@ def run_commit_sync(job_id: str) -> Dict[str, Any]: ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") + logger.warning("Vehicles import: failed to store commit status in Redis: %s", e) return result @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): - """ - Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService. - """ - logger.info(f"Vehicles import: starting commit for job {job_id}") + logger.info("Vehicles import: starting commit for job %s", job_id) result = _do_commit(job_id) try: r = _get_redis() @@ -603,5 +290,5 @@ def insert_valid_rows(self, job_id: str): ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: - logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") + logger.warning("Vehicles import: failed to store commit status in Redis: %s", e) return result diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py new file mode 100644 index 00000000..a6c52f52 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/__init__.py @@ -0,0 +1,3 @@ +from .create import validate_row_vehicle + +__all__ = ["validate_row_vehicle"] diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py new file mode 100644 index 00000000..8cb5533f --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/common.py @@ -0,0 +1,58 @@ +""" +Validaciones comunes de fila para import CSV de vehículos. +""" +from typing import Dict, Any, Optional + +from ..common.common_validators import ( + MAX_LEN, + check_required, + check_max_length, +) + + +def validate_row_vehicle_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + return check_required(row, "CLAVE", MAX_LEN["vehicle_key"], line_num) + + +def validate_row_vehicle_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + checks = [ + ("CLAVE ACE", MAX_LEN["ace_vehicle_key"]), + ("CLAVE TRANSPORTE", MAX_LEN["transporter_key"]), + ("VIN", MAX_LEN["series"]), + ("TIPO TRANSPORTE", MAX_LEN["transport_type"]), + ("CODIGO DE ENTIDAD", MAX_LEN["entity_code"]), + ("TRANSPONDEDOR", MAX_LEN["transponder_number"]), + ("NUMERO DOT", MAX_LEN["dot_number"]), + ("PLACAS", MAX_LEN["plate_number"]), + ("CIUDAD", MAX_LEN["city"]), + ("ESTADO", MAX_LEN["state"]), + ("PAIS", MAX_LEN["country"]), + ("PRECINTO", MAX_LEN["seal"]), + ("EMPRESA ASEGURADORA", MAX_LEN["insurance_company_name"]), + ("NUM. ASEGURADORA", MAX_LEN["insurance_number"]), + ] + for col, max_len in checks: + err = check_max_length(row, col, max_len, line_num) + if err: + return err + return None + + +def validate_row_vehicle_amount(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + from ..common.common_validators import parse_decimal + monto = row.get("MONTO ASEGURADO") or row.get("MONTO") + if monto is None or not str(monto).strip(): + return None + if parse_decimal(monto) is None: + return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"} + return None + + +def validate_row_vehicle_insurance_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + from ..common.common_validators import parse_insurance_date + fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA") + if fecha is None or not str(fecha).strip(): + return None + if parse_insurance_date(fecha) is None: + return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} + return None diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py new file mode 100644 index 00000000..1375a80b --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/validators/create.py @@ -0,0 +1,31 @@ +""" +Punto de entrada de validación para import de una fila vehículo. +""" +from typing import Dict, Any, Optional + +from .common import ( + validate_row_vehicle_required, + validate_row_vehicle_lengths, + validate_row_vehicle_amount, + validate_row_vehicle_insurance_date, +) + + +def validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """ + Valida una fila de CSV de vehículos. + Encadena: requerido (CLAVE) → longitudes → monto opcional → fecha aseguradora opcional. + """ + err = validate_row_vehicle_required(row, line_num) + if err: + return err + err = validate_row_vehicle_lengths(row, line_num) + if err: + return err + err = validate_row_vehicle_amount(row, line_num) + if err: + return err + err = validate_row_vehicle_insurance_date(row, line_num) + if err: + return err + return None diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ca30b42e..f4a4b01d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -370,6 +370,8 @@ export const api = { validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`) }, + // CSV import for Operaciones de Importación/Exportación (facturas: encabezados y partidas). + // Backend: api/v1/modules/a76/layouts_csv/facturas (rutas bajo /v1/a76/imports/). imports: { upload: ( file: File, diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index c51ceaae..8c7420ec 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -31,6 +31,8 @@ export interface CsvUploadItem { /** Backend template id for CSV download (e.g. customs_brokers, part_numbers). No physical file. */ templateId?: string; disabled?: boolean; // New property to mark items as "Coming Soon" + /** Backend module path (layouts_csv) for traceability, e.g. "layouts_csv/facturas", "layouts_csv/parts". */ + layoutModule?: string; } export interface CsvUploadField { @@ -125,42 +127,47 @@ export const tabSettings: Record = { }; // --- DATA DEFINITIONS (Items only, no config) --- - +// Catálogos: la mayoría usa backend en layouts_csv (customs_brokers, clients_and_providers, etc.). export const catalogosConfig: CsvUploadItem[] = [ { id: 'customs_brokers', title: 'Agentes Aduanales', icon: User, modelTarget: 'CustomsBroker', - templateId: 'customs_brokers' + templateId: 'customs_brokers', + layoutModule: 'layouts_csv/customs_brokers' }, { id: 'clients_providers', title: 'Clientes y Proveedores', icon: Users, modelTarget: 'ClientProvider', - templateId: 'clients_providers' + templateId: 'clients_providers', + layoutModule: 'layouts_csv/clients_and_providers' }, { id: 'exchange_rates', title: 'Tipo de Cambios', icon: DollarSign, modelTarget: 'ExchangeRate', - templateId: 'exchange_rates' + templateId: 'exchange_rates', + layoutModule: 'layouts_csv/exchange_rate' }, { id: 'american_fractions', title: 'Fracc. Ame.', icon: Globe, modelTarget: 'AmericanFraction', - templateId: 'american_fractions' + templateId: 'american_fractions', + layoutModule: 'layouts_csv/us_tariff_fractions' }, { id: 'material_classes', title: 'Clases de Materiales', icon: Package, modelTarget: 'MaterialClass', - templateId: 'material_classes' + templateId: 'material_classes', + layoutModule: 'layouts_csv/classes' }, { id: 'part_numbers', @@ -168,6 +175,7 @@ export const catalogosConfig: CsvUploadItem[] = [ icon: Hash, modelTarget: 'Part', templateId: 'part_numbers', + layoutModule: 'layouts_csv/parts' }, { id: 'boms', @@ -175,6 +183,7 @@ export const catalogosConfig: CsvUploadItem[] = [ icon: Briefcase, modelTarget: 'Bom', templateId: 'boms', + layoutModule: 'layouts_csv/boms' }, { id: 'items', @@ -204,10 +213,12 @@ export const catalogosConfig: CsvUploadItem[] = [ title: 'Pedimentos', icon: FileDigit, modelTarget: 'Pedimento', - templateId: 'pedimentos' + templateId: 'pedimentos', + layoutModule: 'layouts_csv/pedmientos' }, ]; +// Transportes: conductores, trailers y “Transportes” (vehículos) usan layouts_csv; transportistas sin layouts_csv aún. export const transportesConfig: CsvUploadItem[] = [ { id: 'transporters', @@ -221,24 +232,29 @@ export const transportesConfig: CsvUploadItem[] = [ title: 'Transportes', icon: Truck, modelTarget: 'Transport', - templateId: 'transports' + templateId: 'transports', + layoutModule: 'layouts_csv/vehicles' }, { id: 'drivers', title: 'Conductores', icon: User, modelTarget: 'Driver', - templateId: 'drivers' + templateId: 'drivers', + layoutModule: 'layouts_csv/drivers' }, { id: 'trailers', title: 'Trailers y Cajas', icon: Container, modelTarget: 'Trailer', - templateId: 'trailers' + templateId: 'trailers', + layoutModule: 'layouts_csv/trailers' }, ]; +// --- Operaciones de Importación (facturas: encabezados y partidas) +// Backend: layouts_csv/facturas — rutas /v1/a76/imports/ (upload, status, commit). export const importacionConfig: CsvUploadItem[] = [ // Impo Temp { @@ -247,7 +263,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: FileText, group: 'Impo. Temp.', modelTarget: 'invoice_header', - templateId: 'imp_temp_header' + templateId: 'imp_temp_header', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_temp_details', @@ -255,7 +272,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: Package, group: 'Impo. Temp.', modelTarget: 'invoice_details', - templateId: 'imp_temp_details' + templateId: 'imp_temp_details', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_temp_series', @@ -272,7 +290,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: FileText, group: 'Impo. Def.', modelTarget: 'invoice_header', - templateId: 'imp_def_header' + templateId: 'imp_def_header', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_def_details', @@ -280,7 +299,8 @@ export const importacionConfig: CsvUploadItem[] = [ icon: Package, group: 'Impo. Def.', modelTarget: 'invoice_details', - templateId: 'imp_def_details' + templateId: 'imp_def_details', + layoutModule: 'layouts_csv/facturas' }, { id: 'imp_def_series', @@ -317,6 +337,8 @@ export const importacionConfig: CsvUploadItem[] = [ }, ]; +// --- Operaciones de Exportación (facturas: encabezados y partidas) +// Backend: layouts_csv/facturas — mismas rutas /v1/a76/imports/ con operation_type=exp. export const exportacionConfig: CsvUploadItem[] = [ // Expo Def / Cam. Reg. { @@ -325,7 +347,8 @@ export const exportacionConfig: CsvUploadItem[] = [ icon: FileText, group: 'Expo. Def./Cam. Reg.', modelTarget: 'invoice_header', - templateId: 'exp_def_header' + templateId: 'exp_def_header', + layoutModule: 'layouts_csv/facturas' }, { id: 'exp_def_details', @@ -333,7 +356,8 @@ export const exportacionConfig: CsvUploadItem[] = [ icon: Package, group: 'Expo. Def./Cam. Reg.', modelTarget: 'invoice_details', - templateId: 'exp_def_details' + templateId: 'exp_def_details', + layoutModule: 'layouts_csv/facturas' }, { id: 'exp_def_series', diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 5a1901f1..e2c43003 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -449,6 +449,7 @@
+

Catálogos Generales

@@ -456,6 +457,7 @@ +

Logística y Transporte

@@ -463,6 +465,7 @@ +

Operaciones de Importación