diff --git a/.gitignore b/.gitignore index 2db7f63a..b8b65467 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,11 @@ wheels/ *.egg .pnpm-store/ -# Environment +# Environment (no subir: cada quien puede usar puertos distintos vía .env) .env .env.local +backend/.env +frontend/.env backend/SCRIPTS/ # IDEs .vscode/ @@ -67,3 +69,5 @@ postgres-data/ backend/uploads/ docker-compose.yml .mypy_cache/ + +backend/celerybeat-schedule \ No newline at end of file diff --git a/backend/api/__init__.py b/backend/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/__init__.py b/backend/api/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/__init__.py b/backend/api/v1/modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/a76/__init__.py b/backend/api/v1/modules/a76/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/a76/classes/__init__.py b/backend/api/v1/modules/a76/classes/__init__.py index 1a3d8bed..fa36b353 100644 --- a/backend/api/v1/modules/a76/classes/__init__.py +++ b/backend/api/v1/modules/a76/classes/__init__.py @@ -2,6 +2,20 @@ Módulo de Class """ -from .routes import router +from typing import Any __all__ = ["router"] + + +def __getattr__(name: str) -> Any: + """ + Lazy export to avoid circular imports during Celery init. + + This package is imported when models are loaded (e.g. a76.items.models), + so importing FastAPI routes at import-time can break Celery startup. + """ + if name == "router": + from .routes import router + + return router + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/backend/api/v1/modules/a76/classes/imports/__init__.py b/backend/api/v1/modules/a76/classes/imports/__init__.py new file mode 100644 index 00000000..e9eb448f --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Clases de Materiales (upload → scan → status → commit). diff --git a/backend/api/v1/modules/a76/classes/imports/routes.py b/backend/api/v1/modules/a76/classes/imports/routes.py new file mode 100644 index 00000000..5c9c869f --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para Clases de Materiales. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CLS_IMPORT_FILE_PREFIX, + CLS_IMPORT_META_PREFIX, + CLS_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Classes import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "material_classes", + } + + try: + r = _get_redis() + r.set( + f"{CLS_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CLS_IMPORT_REDIS_TTL, + ) + r.set( + f"{CLS_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CLS_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Classes import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Classes import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/classes/imports/schemas.py b/backend/api/v1/modules/a76/classes/imports/schemas.py new file mode 100644 index 00000000..2d362f01 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/schemas.py @@ -0,0 +1,21 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/classes/imports/tasks.py b/backend/api/v1/modules/a76/classes/imports/tasks.py new file mode 100644 index 00000000..ca754d06 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/tasks.py @@ -0,0 +1,528 @@ +""" +Tareas Celery para importación CSV de Clases de Materiales. +Flujo: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +CLS_IMPORT_FILE_PREFIX = "cls_import_file:" +CLS_IMPORT_META_PREFIX = "cls_import_meta:" +CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" +CLS_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Classes import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cls_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Classes import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CLS_IMPORT_FILE_PREFIX}{job_id}", + f"{CLS_IMPORT_META_PREFIX}{job_id}", + f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Classes import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_class( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + class_code = (row.get("CLASE") or "").strip() + if not class_code: + return {"line": line_num, "col": "CLASE", "msg": "Requerido"} + if len(class_code) > 8: + return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} + + desc_es = (row.get("DESCRIPCIONE") or "").strip() + if desc_es and len(desc_es) > 500: + return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} + desc_en = (row.get("DESCRIPCIONI") or "").strip() + if desc_en and len(desc_en) > 500: + return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} + + material_key = (row.get("CLAVEMAT") or "").strip() + if material_key: + if len(material_key) > 10: + return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"} + if valid_material_keys is not None and material_key not in valid_material_keys: + return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"} + + uom = (row.get("UNIMED") or "").strip() + if uom: + if len(uom) > 5: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} + if valid_uom_codes is not None and uom not in valid_uom_codes: + return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"} + + fraction = (row.get("FRACCION") or "").strip() + if fraction and len(fraction) > 20: + return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"} + us_fraction = (row.get("FRACCIONAME") or "").strip() + if us_fraction and len(us_fraction) > 16: + return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} + sub_key = (row.get("CLAVESUB") or "").strip() + if sub_key and len(sub_key) > 5: + return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"} + iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip() + if iva_exempt and len(iva_exempt) > 4: + return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"} + + rev_fisica = row.get("REVFISICA") + if rev_fisica is not None and rev_fisica != "": + try: + v = int(rev_fisica) + if v < -32768 or v > 32767: + return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"} + except (ValueError, TypeError): + return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Classes import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Classes import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + logger.warning(f"Classes import: could not load FK sets: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_class( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Classes import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CLS_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Classes import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Classes import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Classes import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.classes.models import Class + + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + logger.warning(f"Classes import: could not load FK sets: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_code: Dict[str, Class] = {} + for c in ( + session.query(Class) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + existing_by_code[c.class_code] = c + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_class( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + class_code = _str_or_none(row_norm.get("CLASE"), 8) + if not class_code: + skipped_invalid += 1 + continue + + existing = existing_by_code.get(class_code) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) + if material_key and material_key not in valid_material_keys: + material_key = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + fraction = _str_or_none(row_norm.get("FRACCION"), 20) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) + physical_review = _int_or_none(row_norm.get("REVFISICA")) + iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) + + if existing: + existing.description_es = desc_es + existing.description_en = desc_en + existing.material_key = material_key + existing.unit_of_measure = unit_of_measure + existing.fraction = fraction + existing.us_fraction = us_fraction + existing.sub_key = sub_key + existing.physical_review = physical_review + existing.iva_exempt_fraction = iva_exempt_fraction + session.add(existing) + inserted_count += 1 + else: + new_class = Class( + tenant_id=tenant_id, + company_id=company_id, + class_code=class_code, + description_es=desc_es, + description_en=desc_en, + material_key=material_key, + unit_of_measure=unit_of_measure, + fraction=fraction, + us_fraction=us_fraction, + sub_key=sub_key, + physical_review=physical_review, + iva_exempt_fraction=iva_exempt_fraction, + ) + session.add(new_class) + existing_by_code[class_code] = new_class + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Classes import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Classes import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Classes import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/classes/imports/template_config.py b/backend/api/v1/modules/a76/classes/imports/template_config.py new file mode 100644 index 00000000..4a389bd8 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/template_config.py @@ -0,0 +1,45 @@ +""" +Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "material_classes": [ + {"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]}, + {"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, + {"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]}, + {"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]}, + {"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("material_classes") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/classes/routes.py b/backend/api/v1/modules/a76/classes/routes.py index 329294b0..2166610f 100644 --- a/backend/api/v1/modules/a76/classes/routes.py +++ b/backend/api/v1/modules/a76/classes/routes.py @@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse from .service import ClassService +from .imports.routes import router as imports_router # Create a new router for custom endpoints router = APIRouter() +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"]) + # Add consolidated catalog endpoints FIRST (before generic CRUD routes) # This ensures they have priority over the generic /{id} route @router.get( diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py b/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py new file mode 100644 index 00000000..94b7cb55 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py @@ -0,0 +1,2 @@ +# CSV import flow for Clientes y Proveedores (Client Providers). +# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py b/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py new file mode 100644 index 00000000..68029417 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py @@ -0,0 +1,161 @@ +""" +Rutas de importación CSV para Clientes y Proveedores. +Mismo flujo que a76.imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CP_IMPORT_FILE_PREFIX, + CP_IMPORT_META_PREFIX, + CP_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"CP import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "client_providers", + } + + try: + r = _get_redis() + r.set( + f"{CP_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CP_IMPORT_REDIS_TTL, + ) + r.set( + f"{CP_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"CP import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"CP import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("CP import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py b/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py new file mode 100644 index 00000000..34fea87c --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py @@ -0,0 +1,500 @@ +""" +Tareas Celery para importación CSV de Clientes y Proveedores. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template +from api.v1.modules.a76.clients_and_providers.models import ( + ClientProvider, + ClientProviderAddress, + ClientOrProviderEnum, +) + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports) +CP_IMPORT_FILE_PREFIX = "cp_import_file:" +CP_IMPORT_META_PREFIX = "cp_import_meta:" +CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:" +CP_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"CP import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cp_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"CP import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CP_IMPORT_FILE_PREFIX}{job_id}", + f"{CP_IMPORT_META_PREFIX}{job_id}", + f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"CP import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: + """Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido.""" + if not val or not str(val).strip(): + return None + v = str(val).strip().lower() + if v in ("client", "cliente", "c"): + return ClientOrProviderEnum.CLIENT + if v in ("provider", "proveedor", "p"): + return ClientOrProviderEnum.PROVIDER + if v in ("both", "ambos", "b", "cliente y proveedor"): + return ClientOrProviderEnum.BOTH + return None + + +def _parse_active(val: Optional[str]) -> bool: + """Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True.""" + if not val or not str(val).strip(): + return True + v = str(val).strip().lower() + if v in ("1", "true", "si", "sí", "yes", "s", "x"): + return True + if v in ("0", "false", "no", "n"): + return False + return True + + +def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Cliente/Proveedor. Retorna error dict o None.""" + rfc = (row.get("RFC") or "").strip() + if not rfc: + return {"line": line_num, "col": "RFC", "msg": "Requerido"} + if len(rfc) > 30: + return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"} + + tipo_raw = (row.get("TIPO") or "").strip() + if tipo_raw and _parse_client_or_provider(tipo_raw) is None: + return { + "line": line_num, + "col": "TIPO", + "msg": "Valor no válido. Use Cliente, Proveedor o Ambos.", + } + + name = (row.get("NOMBRE") or "").strip() + if len(name) > 256: + return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"} + + short_name = (row.get("SHORT_NAME") or "").strip() + if short_name and len(short_name) > 10: + return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"} + + curp = (row.get("CURP") or "").strip() + if curp and len(curp) > 19: + return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"CP import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"CP import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_client_provider(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"CP import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"CP import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider. + Upsert por (tenant_id, company_id, rfc). + """ + logger.info(f"CP import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"CP import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + # Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key + existing_by_rfc: Dict[str, ClientProvider] = {} + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + key = (cp.rfc or "").strip() + existing_by_rfc[key] = cp + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_client_provider(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + rfc = _str_or_none(row_norm.get("RFC"), 30) + if not rfc: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "RFC requerido"}) + continue + + client_or_provider = _parse_client_or_provider(row_norm.get("TIPO")) + if client_or_provider is None: + client_or_provider = ClientOrProviderEnum.BOTH + + is_active = _parse_active(row_norm.get("ACTIVO")) + + existing = existing_by_rfc.get(rfc) + if existing: + existing.name = _str_or_none(row_norm.get("NOMBRE"), 256) + existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10) + existing.curp = _str_or_none(row_norm.get("CURP"), 19) + existing.client_or_provider = client_or_provider + existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80) + existing.position = _str_or_none(row_norm.get("POSICION"), 30) + existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19) + existing.is_active = is_active + session.add(existing) + inserted_count += 1 + else: + new_cp = ClientProvider( + tenant_id=tenant_id, + company_id=company_id, + rfc=rfc, + name=_str_or_none(row_norm.get("NOMBRE"), 256), + short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10), + curp=_str_or_none(row_norm.get("CURP"), 19), + client_or_provider=client_or_provider, + responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80), + position=_str_or_none(row_norm.get("POSICION"), 30), + incoterm=_str_or_none(row_norm.get("INCOTERM"), 19), + is_active=is_active, + ) + session.add(new_cp) + session.flush() + existing_by_rfc[rfc] = new_cp + inserted_count += 1 + + # Opcional: crear dirección si hay email/teléfono/dirección + email = _str_or_none(row_norm.get("EMAIL"), 100) + phone = _str_or_none(row_norm.get("TELEFONO"), 30) + address_str = _str_or_none(row_norm.get("DIRECCION"), 100) + if email or phone or address_str: + addr = ClientProviderAddress( + client_id=new_cp.id, + tenant_id=tenant_id, + company_id=company_id, + streets=address_str, + postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), + city=_str_or_none(row_norm.get("CIUDAD"), 30), + state=_str_or_none(row_norm.get("ESTADO"), 30), + country=_str_or_none(row_norm.get("PAIS"), 3), + phone=phone, + email=email, + contact=_str_or_none(row_norm.get("CONTACTO"), 50), + ) + session.add(addr) + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"CP import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"CP import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"CP import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py new file mode 100644 index 00000000..f0968d9d --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py @@ -0,0 +1,56 @@ +""" +Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "client_providers": [ + {"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]}, + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]}, + {"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]}, + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]}, + {"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]}, + {"canonical": "CURP", "aliases": []}, + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]}, + {"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + {"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]}, + {"canonical": "ESTADO", "aliases": []}, + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + {"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]}, + {"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "INCOTERM", "aliases": []}, + {"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla client_providers.""" + cols = TEMPLATE_COLUMNS.get("client_providers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/clients_and_providers/routes.py b/backend/api/v1/modules/a76/clients_and_providers/routes.py index 6b31d570..c9c67efe 100644 --- a/backend/api/v1/modules/a76/clients_and_providers/routes.py +++ b/backend/api/v1/modules/a76/clients_and_providers/routes.py @@ -20,10 +20,14 @@ from .dto import ( ) from .service import ClientProviderService from .models import ClientProvider +from .imports.routes import router as imports_router # Create main router to add custom endpoints router = APIRouter(prefix="/clients-providers") +# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit) +router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"]) + @router.get("/", response_model=ClientProviderPaginatedResponseDTO) async def get_clients_and_providers( diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py new file mode 100644 index 00000000..c1c90d12 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py @@ -0,0 +1,2 @@ +# CSV import flow for Agentes Aduanales (Customs Brokers). +# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py new file mode 100644 index 00000000..99f7c609 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py @@ -0,0 +1,161 @@ +""" +Rutas de importación CSV para Agentes Aduanales. +Mismo flujo que a76.imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CB_IMPORT_FILE_PREFIX, + CB_IMPORT_META_PREFIX, + CB_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"CB import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "customs_brokers", + } + + try: + r = _get_redis() + r.set( + f"{CB_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CB_IMPORT_REDIS_TTL, + ) + r.set( + f"{CB_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CB_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"CB import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"CB import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("CB import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py b/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py new file mode 100644 index 00000000..7d9b548c --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + +class CommitRequest(BaseModel): + pass # no body needed for single model + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py new file mode 100644 index 00000000..ecaed98f --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py @@ -0,0 +1,447 @@ +""" +Tareas Celery para importación CSV de Agentes Aduanales. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con a76.imports) +CB_IMPORT_FILE_PREFIX = "cb_import_file:" +CB_IMPORT_META_PREFIX = "cb_import_meta:" +CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:" +CB_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"CB import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cb_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"CB import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CB_IMPORT_FILE_PREFIX}{job_id}", + f"{CB_IMPORT_META_PREFIX}{job_id}", + f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"CB import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Agente Aduanal. Retorna error dict o None.""" + clave = (row.get("CLAVE") or "").strip() + if not clave: + return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} + if len(clave) > 5: + return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"} + if not re.match(r"^[a-zA-Z0-9]+$", clave): + return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"} + + licencia = (row.get("LICENCIA") or "").strip() + if licencia and (len(licencia) > 4 or not licencia.isdigit()): + return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"CB import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"CB import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_customs_broker(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"CB import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + # Guardar números de línea con error en Redis para insert_valid_rows + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CB_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"CB import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker. + """ + logger.info(f"CB import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"CB import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_key: Dict[str, CustomsBroker] = {} + for b in ( + session.query(CustomsBroker) + .filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ) + .all() + ): + existing_by_key[b.broker_key] = b + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_customs_broker(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + clave = (row_norm.get("CLAVE") or "").strip()[:5] + if not clave: + skipped_invalid += 1 + continue + + existing = existing_by_key.get(clave) + if existing: + existing.type = _str_or_none(row_norm.get("TIPO"), 9) + existing.name = _str_or_none(row_norm.get("NOMBRE"), 80) + existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500) + existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15) + existing.city = _str_or_none(row_norm.get("CIUDAD"), 30) + existing.state = _str_or_none(row_norm.get("ESTADO"), 30) + existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30) + existing.fax = _str_or_none(row_norm.get("FAX"), 30) + existing.email = _str_or_none(row_norm.get("EMAIL"), 100) + existing.country = _str_or_none(row_norm.get("PAIS"), 3) + existing.tax_id = _str_or_none(row_norm.get("RFC"), 30) + existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20) + existing.position = _str_or_none(row_norm.get("POSICION"), 30) + lic = (row_norm.get("LICENCIA") or "").strip() + existing.license = lic[:4] if lic and lic.isdigit() else None + existing.company = _str_or_none(row_norm.get("EMPRESA"), 200) + existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80) + session.add(existing) + inserted_count += 1 + else: + lic = (row_norm.get("LICENCIA") or "").strip() + license_val = lic[:4] if lic and lic.isdigit() else None + new_broker = CustomsBroker( + tenant_id=tenant_id, + company_id=company_id, + broker_key=clave, + type=_str_or_none(row_norm.get("TIPO"), 9), + name=_str_or_none(row_norm.get("NOMBRE"), 80), + address=_str_or_none(row_norm.get("DIRECCION"), 1500), + postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), + city=_str_or_none(row_norm.get("CIUDAD"), 30), + state=_str_or_none(row_norm.get("ESTADO"), 30), + phone=_str_or_none(row_norm.get("TELEFONO"), 30), + fax=_str_or_none(row_norm.get("FAX"), 30), + email=_str_or_none(row_norm.get("EMAIL"), 100), + country=_str_or_none(row_norm.get("PAIS"), 3), + tax_id=_str_or_none(row_norm.get("RFC"), 30), + personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20), + position=_str_or_none(row_norm.get("POSICION"), 30), + license=license_val, + company=_str_or_none(row_norm.get("EMPRESA"), 200), + contact=_str_or_none(row_norm.get("CONTACTO"), 80), + ) + session.add(new_broker) + existing_by_key[clave] = new_broker + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"CB import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"CB import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + # Limpieza + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"CB import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py b/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py new file mode 100644 index 00000000..f4801729 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py @@ -0,0 +1,55 @@ +""" +Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "customs_brokers": [ + {"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]}, + {"canonical": "TIPO"}, + {"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]}, + {"canonical": "DIRECCION", "aliases": ["DOMICILIO"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + {"canonical": "CIUDAD"}, + {"canonical": "ESTADO"}, + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]}, + {"canonical": "FAX"}, + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]}, + {"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]}, + {"canonical": "EMPRESA", "aliases": ["COMPANY"]}, + {"canonical": "CONTACTO", "aliases": ["CONTACT"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla customs_brokers.""" + cols = TEMPLATE_COLUMNS.get("customs_brokers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/customs_brokers/routes.py b/backend/api/v1/modules/a76/customs_brokers/routes.py index 541f41d9..a94eccd1 100644 --- a/backend/api/v1/modules/a76/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/customs_brokers/routes.py @@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from . import dto, services +from .imports.routes import router as imports_router router = APIRouter() +# CSV import (mismo flujo que a76.imports: upload → scan → commit) +router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"]) + customs_broker_crud = TenantCRUDRoutes( service=services.CustomsBrokerService, create_schema=dto.CustomsBrokerCreateDTO, diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py new file mode 100644 index 00000000..4ac8a216 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py @@ -0,0 +1 @@ +# Exchange rate CSV import module diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py new file mode 100644 index 00000000..75a9aef8 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Tipos de Cambio. +Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + ER_IMPORT_FILE_PREFIX, + ER_IMPORT_META_PREFIX, + ER_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"ER import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "exchange_rates", + } + + try: + r = _get_redis() + r.set( + f"{ER_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=ER_IMPORT_REDIS_TTL, + ) + r.set( + f"{ER_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=ER_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"ER import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"ER import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("ER import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py new file mode 100644 index 00000000..c214b80b --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py @@ -0,0 +1,465 @@ +""" +Tareas Celery para importación CSV de Tipos de Cambio. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +ER_IMPORT_FILE_PREFIX = "er_import_file:" +ER_IMPORT_META_PREFIX = "er_import_meta:" +ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:" +ER_IMPORT_REDIS_TTL = 3600 # 1 hour + +DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] +TEMPLATE_ID = "exchange_rates" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"ER import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"er_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"ER import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{ER_IMPORT_FILE_PREFIX}{job_id}", + f"{ER_IMPORT_META_PREFIX}{job_id}", + f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"ER import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _parse_date(val: Optional[str]) -> Optional[datetime]: + """Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc.""" + if not val or not str(val).strip(): + return None + raw = str(val).strip() + for fmt in DATE_FORMATS: + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + continue + return None + + +def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Tipo de Cambio. Retorna error dict o None.""" + fecha_raw = (row.get("FECHA") or "").strip() + if not fecha_raw: + return {"line": line_num, "col": "FECHA", "msg": "Requerido"} + if _parse_date(fecha_raw) is None: + return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} + + valor_raw = (row.get("VALOR") or "").strip() + if not valor_raw: + return {"line": line_num, "col": "VALOR", "msg": "Requerido"} + try: + v = float(valor_raw.replace(",", ".")) + if v <= 0: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} + except ValueError: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} + + local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper() + if local_raw and len(local_raw) > 7: + return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"} + + foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper() + if foreign_raw and len(foreign_raw) > 7: + return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"ER import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"ER import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_exchange_rate(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"ER import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=ER_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"ER import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha). + """ + logger.info(f"ER import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"ER import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_date: Dict[tuple, ExchangeRate] = {} + for er in ( + session.query(ExchangeRate) + .filter( + ExchangeRate.tenant_id == tenant_id, + ExchangeRate.company_id == company_id, + ) + .all() + ): + d = er.date.date() if hasattr(er.date, "date") else er.date + existing_by_date[(tenant_id, company_id, d)] = er + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_exchange_rate(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + parsed_date = _parse_date(row_norm.get("FECHA")) + if not parsed_date: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FECHA: no parseable"}) + continue + + try: + v = float((row_norm.get("VALOR") or "").strip().replace(",", ".")) + value_decimal = Decimal(str(round(v, 6))) + except (ValueError, TypeError): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "VALOR: no numérico"}) + continue + + local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7) + if local_currency: + local_currency = local_currency.upper() + foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7) + if foreign_currency: + foreign_currency = foreign_currency.upper() + + key_date = parsed_date.date() + existing = existing_by_date.get((tenant_id, company_id, key_date)) + + if existing: + existing.value = value_decimal + existing.local_currency = local_currency or None + existing.foreign_currency = foreign_currency or None + session.add(existing) + inserted_count += 1 + else: + new_er = ExchangeRate( + tenant_id=tenant_id, + company_id=company_id, + date=parsed_date, + value=value_decimal, + local_currency=local_currency, + foreign_currency=foreign_currency, + ) + session.add(new_er) + existing_by_date[(tenant_id, company_id, key_date)] = new_er + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"ER import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"ER import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"ER import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py new file mode 100644 index 00000000..c907f9a0 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py @@ -0,0 +1,42 @@ +""" +Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "exchange_rates": [ + {"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]}, + {"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]}, + {"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]}, + {"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla exchange_rates.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py index 22ab8ec9..09086071 100644 --- a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py @@ -34,6 +34,10 @@ crud_router = route_handler.router from fastapi import APIRouter custom_router = APIRouter(prefix="/exchange-rate", tags=[]) +# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered +from .imports.routes import router as imports_router +custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"]) + @custom_router.get("/test-ping") async def test_ping(): return {"message": "pong"} diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py new file mode 100644 index 00000000..cff9f961 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py new file mode 100644 index 00000000..bf5083a3 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Fracción Americana (US Tariff Fractions). +Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + FA_IMPORT_FILE_PREFIX, + FA_IMPORT_META_PREFIX, + FA_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"FA import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "us_tariff_fractions", + } + + try: + r = _get_redis() + r.set( + f"{FA_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=FA_IMPORT_REDIS_TTL, + ) + r.set( + f"{FA_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=FA_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"FA import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"FA import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("FA import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py new file mode 100644 index 00000000..edac5df2 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py @@ -0,0 +1,474 @@ +""" +Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions). +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +FA_IMPORT_FILE_PREFIX = "fa_import_file:" +FA_IMPORT_META_PREFIX = "fa_import_meta:" +FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:" +FA_IMPORT_REDIS_TTL = 3600 # 1 hour + +TEMPLATE_ID = "us_tariff_fractions" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"FA import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"fa_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"FA import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{FA_IMPORT_FILE_PREFIX}{job_id}", + f"{FA_IMPORT_META_PREFIX}{job_id}", + f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"FA import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _normalize_code(raw: Optional[str]) -> str: + """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" + if not raw: + return "" + s = str(raw).strip().replace(".", "").replace("-", "") + return s[:16] if len(s) > 16 else s + + +def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Fracción Americana. Retorna error dict o None.""" + code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip() + if not code_raw: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} + code_norm = _normalize_code(code_raw) + if not code_norm: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} + if len(code_norm) > 16: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"} + + prefix_raw = (row.get("PREFIJO") or "").strip() + if prefix_raw and len(prefix_raw) > 10: + return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"} + + um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip() + if um_raw and len(um_raw) > 10: + return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"} + + tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip() + if tipo_raw and len(tipo_raw) > 10: + return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"} + + adv_pct = row.get("ADVALOREM_PCT") + if adv_pct is not None and str(adv_pct).strip(): + try: + v = float(str(adv_pct).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"} + + adv_dlls = row.get("ADVALOREM_DLLS") + if adv_dlls is not None and str(adv_dlls).strip(): + try: + v = float(str(adv_dlls).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"FA import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"FA import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_us_tariff_fraction(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"FA import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=FA_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"FA import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _parse_float(val: Any) -> Optional[float]: + if val is None or str(val).strip() == "": + return None + try: + return float(str(val).strip().replace(",", ".")) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code). + """ + logger.info(f"FA import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"FA import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_us_tariff_fraction(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA")) + if not code: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"}) + continue + + prefix = _str_or_none(row_norm.get("PREFIJO"), 10) + unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10) + description = _str_or_none(row_norm.get("DESCRIPCION")) + type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10) + ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT")) + fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS")) + fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None + + existing = ( + session.query(USTariffFraction) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + USTariffFraction.code == code, + ) + .first() + ) + + if existing: + existing.prefix = prefix + existing.type_code = type_code + existing.ad_valorem = ad_valorem + existing.fixed_cost = fixed_cost + existing.unit_of_measure = unit_of_measure + existing.description = description + session.add(existing) + inserted_count += 1 + else: + new_row = USTariffFraction( + tenant_id=tenant_id, + company_id=company_id, + code=code, + prefix=prefix, + type_code=type_code, + ad_valorem=ad_valorem, + fixed_cost=fixed_cost, + unit_of_measure=unit_of_measure, + description=description, + ) + session.add(new_row) + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"FA import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"FA import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path_clean = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path_clean): + os.remove(meta_path_clean) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"FA import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py new file mode 100644 index 00000000..aa5a5e7f --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py @@ -0,0 +1,47 @@ +""" +Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "us_tariff_fractions": [ + {"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]}, + {"canonical": "PREFIJO"}, + {"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]}, + {"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]}, + {"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla us_tariff_fractions.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template( + row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions" +) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py index 158bf765..61907ffa 100644 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py @@ -17,23 +17,28 @@ from .dto import ( ) from .service import USTariffFractionService -# Create router using TenantCRUDRoutes factory for basic CRUD operations +# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router) crud_router = TenantCRUDRoutes( service=USTariffFractionService, create_schema=USTariffFractionCreateDTO, update_schema=USTariffFractionUpdateDTO, response_schema=USTariffFractionResponseDTO, - prefix="/us-tariff-fractions", + prefix="", tags=["a76 / general catalogs / us tariff fractions"], resource_name="US Tariff Fraction", id_name="id", enable_list=False, # We implement our custom list endpoint ) -router = crud_router.router +# Master router with prefix so all routes live under /us-tariff-fractions +from .imports.routes import router as imports_router +main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"]) +main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"]) +main_router.include_router(crud_router.router) -# Custom list endpoint with search filter -@router.get( + +# Custom list endpoint with search filter (under /us-tariff-fractions/) +@main_router.get( "/", response_model=Dict[str, Any], summary="List US Tariff Fractions", @@ -65,3 +70,6 @@ async def list_us_tariff_fractions( "page_size": page_size, "pages": (total + page_size - 1) // page_size, } + + +router = main_router diff --git a/backend/api/v1/modules/a76/imports/routes.py b/backend/api/v1/modules/a76/imports/routes.py index 1ea76a74..f3c3d9ee 100644 --- a/backend/api/v1/modules/a76/imports/routes.py +++ b/backend/api/v1/modules/a76/imports/routes.py @@ -1,5 +1,6 @@ from datetime import datetime from uuid import uuid4 +import base64 import os import json import logging @@ -12,24 +13,39 @@ from core.config import settings from core.database import get_core_db from core.security import get_current_user, validate_access_to_resource -from .tasks import scan_file, insert_valid_rows +from .tasks import ( + scan_file, + insert_valid_rows, + IMPORT_FILE_KEY_PREFIX, + IMPORT_META_KEY_PREFIX, + IMPORT_REDIS_TTL, +) from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest router = APIRouter() logger = logging.getLogger(__name__) + +def _get_redis(): + """Redis client (same broker as Celery so worker can read).""" + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + @router.post("/upload/{model_target}", response_model=ImportJobResponse) async def upload_import_file( model_target: Literal["invoice_header", "invoice_details"], file: UploadFile = File(...), footer_config: Optional[str] = Form(None), # JSON string with settings - company_id: int = Query(..., description="Company ID"), # Required for context + template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas + company_id: int = Query(..., description="Company ID"), # Required for context operation_type: Optional[str] = Query("imp"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): """ Step 1: Upload CSV, save to temp, trigger scan task. + Si se envía template_id, solo se leen las columnas de esa plantilla. """ # 1. Validate Access & Get Tenant try: @@ -40,40 +56,51 @@ async def upload_import_file( if not file.filename.endswith(".csv"): raise HTTPException(status_code=400, detail="Only .csv files allowed") - + job_id = str(uuid4()) - - # Ensure directory exists (Safety check) - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") - os.makedirs(upload_dir, exist_ok=True) - - file_path = os.path.join(upload_dir, f"{job_id}.csv") - meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") - + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "footer_config": footer_config, + "operation_type": operation_type, + "template_id": template_id, + } + + # Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed) try: - # Save CSV - contents = await file.read() + redis_client = _get_redis() + redis_client.set( + f"{IMPORT_FILE_KEY_PREFIX}{job_id}", + base64.b64encode(contents), + ex=IMPORT_REDIS_TTL, + ) + redis_client.set( + f"{IMPORT_META_KEY_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Redis store error: {e}") + raise HTTPException(status_code=500, detail="Failed to queue file for processing.") + + # Optional: also write to local disk (e.g. for same-machine worker or debugging) + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"{job_id}.csv") + meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") with open(file_path, "wb") as f: f.write(contents) - - # Save Metadata (Context) - meta_data = { - "tenant_id": tenant_id, - "company_id": company_id, - "user_id": current_user.get("id"), - "footer_config": footer_config, - "operation_type": operation_type, - } with open(meta_path, "w") as f: json.dump(meta_data, f) - except Exception as e: - logger.error(f"File save error: {e}") - raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}") + logger.warning(f"Local file save failed (worker will use Redis): {e}") - # Trigger Celery Task (Async) - # Use our job_id as the Celery task_id for easier tracking - scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id) + # Trigger Celery Task (Async). Worker loads file from Redis. + scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id) return ImportJobResponse( job_id=job_id, @@ -84,24 +111,56 @@ async def upload_import_file( @router.get("/{job_id}/status") async def get_import_status(job_id: str): """ - Poll this endpoint to get % progress or final report. + Poll to get progress or final report. Always returns an object with "status". """ - # In a real app, query Redis or DB. - # For MVP, we might mock or use Celery AsyncResult if backend shares Redis. task_result = celery_app.AsyncResult(job_id) - - if task_result.state == 'PENDING': + + if task_result.state == "PENDING": return {"status": "processing", "progress": 0} - elif task_result.state == 'PROGRESS': + if task_result.state == "PROGRESS": return { - "status": "processing", - "progress": task_result.info.get('current', 0), - "total": task_result.info.get('total', 0) + "status": "processing", + "progress": (task_result.info or {}).get("current", 0), + "total": (task_result.info or {}).get("total", 0), } - elif task_result.state == 'SUCCESS': - return task_result.result # Should return the report - else: - return {"status": task_result.state, "error": str(task_result.info)} + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + # FAILURE: obtener mensaje real (traceback, result o get(propagate=False)) + logger.warning("Import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb: + logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg and len(lines) > 1: + err_msg = lines[-2] + " " + (lines[-1] or "") + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg: + result = getattr(task_result, "result", None) + info = getattr(task_result, "info", None) + if result is not None and not isinstance(result, dict): + err_msg = str(result) + elif isinstance(result, dict) and (result.get("error") or result.get("message")): + err_msg = result.get("error") or result.get("message") + if not err_msg and isinstance(info, str): + err_msg = info + elif not err_msg and isinstance(info, dict) and "error" in info: + err_msg = str(info["error"]) + if not err_msg: + err_msg = "Task failed" + return {"status": "failed", "error": err_msg} @router.post("/{job_id}/commit") diff --git a/backend/api/v1/modules/a76/imports/tasks.py b/backend/api/v1/modules/a76/imports/tasks.py index 58ef97e3..b6e8e8d9 100644 --- a/backend/api/v1/modules/a76/imports/tasks.py +++ b/backend/api/v1/modules/a76/imports/tasks.py @@ -1,4 +1,5 @@ import os +import base64 from datetime import datetime from decimal import Decimal import csv @@ -6,9 +7,12 @@ import json import logging import re import unicodedata -from celery import shared_task -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app from core.database import CoreSessionLocal + +from .template_config import row_from_template # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process # We'll need schemas for validation @@ -17,6 +21,78 @@ from core.database import CoreSessionLocal logger = logging.getLogger(__name__) +# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem) +IMPORT_FILE_KEY_PREFIX = "import_file:" +IMPORT_META_KEY_PREFIX = "import_meta:" +IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" +IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + """Redis client using same URL as Celery broker (worker and API can share data).""" + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + """Directory on the worker for temp CSV and meta (same structure as API, but local to worker).""" + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + """ + Load file content from Redis and write to worker's upload dir. + Returns local file_path if successful, None otherwise. + """ + redis_client = _get_redis() + key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}" + data = redis_client.get(key) + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Failed to decode import file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + """Load meta from Redis and write to worker's meta file. Returns True if meta was found and written.""" + redis_client = _get_redis() + key = f"{IMPORT_META_KEY_PREFIX}{job_id}" + data = redis_client.get(key) + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Failed to decode import meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + """Remove file, meta and error lines from Redis after commit (cleanup).""" + try: + r = _get_redis() + r.delete( + f"{IMPORT_FILE_KEY_PREFIX}{job_id}", + f"{IMPORT_META_KEY_PREFIX}{job_id}", + f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Failed to delete import keys from Redis: {e}") + class ForeignKeyValidator: def __init__(self, session, tenant_id, company_id): self.session = session @@ -40,26 +116,90 @@ class ForeignKeyValidator: self.cache[key] = exists return exists -@shared_task(bind=True) -def scan_file(self, job_id: str, file_path: str, model_target: str, config: str = None): + +TRANSPORT_TYPE_VALUES = { + "none", + "transport", + "box", + "licence plates", + "truck", + "vessel", + "rail_barge", + "container", + "airplane", + "gondola", + "flatbed", +} + + +def normalize_public_code(value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = str(value).strip().upper() + return text or None + + +def validate_public_code( + validator: ForeignKeyValidator, + model, + value: Optional[str], + line_num: int, + col_name: str, + field_name: str = "code", + required: bool = False, +) -> Optional[Dict[str, Any]]: + code = normalize_public_code(value) + if not code: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, code, field_name=field_name, is_public=True): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + + +def validate_tenant_fk_id( + validator: ForeignKeyValidator, + model, + value: Optional[int], + line_num: int, + col_name: str, + required: bool = False, +) -> Optional[Dict[str, Any]]: + if value is None: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, value): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, model_target: str, config: str = None): """ Pass 1: Read CSV, Validate types, Write Errors to JSONL. + File content is loaded from Redis (written by API on upload) so worker does not need shared filesystem. """ logger.info(f"Starting scan for job {job_id} target {model_target}") - - # 1. Setup Error Log + + # 1. Get file from Redis and write to worker local disk + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + # 2. Setup Error Log error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") os.makedirs(os.path.dirname(error_path), exist_ok=True) - + total_rows = 0 error_count = 0 processed_rows = 0 - - # 2. Count Total (Quick Pass) or just estimate - # For better progress, we can get file line count first + + # 3. Count Total (Quick Pass) or just estimate try: with open(file_path, 'r', encoding='utf-8-sig') as f: - total_rows = sum(1 for _ in f) - 1 # Minus header + total_rows = sum(1 for _ in f) - 1 # Minus header except Exception as e: return {"status": "failed", "error": f"Cannot read file: {e}"} @@ -70,10 +210,65 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + tenant_id = None + company_id = None + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f_meta: + meta = json.load(f_meta) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception as e: + logger.warning(f"Failed to read meta for job {job_id}: {e}") + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Missing context (tenant/company)"} + + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + + inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM") + if not inv_type_value: + inv_type_value = "TEM" try: - with open(file_path, 'r', encoding='utf-8-sig') as f_in, \ + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, + ) + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm + from api.v1.modules.a76.parts.models import Part + + models = { + "InvoiceHeader": InvoiceHeader, + "InvoiceType": InvoiceType, + "ClientProvider": ClientProvider, + "CustomsBroker": CustomsBroker, + "RegimenPedimento": RegimenPedimento, + "CodePedimentoRegimen": CodePedimentoRegimen, + "PedimentoCode": PedimentoCode, + "CurrencyType": CurrencyType, + "CustomsSection": CustomsSection, + "Incoterm": Incoterm, + "Part": Part, + } + + with CoreSessionLocal() as session, \ + open(file_path, 'r', encoding='utf-8-sig') as f_in, \ open(error_path, 'w', encoding='utf-8') as f_err: + validator = ForeignKeyValidator(session, tenant_id, company_id) + invoice_id_cache: Dict[str, Optional[int]] = {} # Detect Delimiter sample = f_in.read(2048) @@ -94,9 +289,18 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str 'errors': error_count }) - # Validation (Phase 1: Minimal) - row_norm = normalize_row(row) - errors = validate_row_phase_1(row_norm, model_target, i, date_format) + # Solo columnas de la plantilla (respetar plantilla tal cual) + row_norm = row_from_template(row, template_id, normalize_header) + errors = validate_row_strict( + row_norm, + model_target, + i, + date_format, + validator, + inv_type_value, + invoice_id_cache, + models, + ) if errors: error_count += 1 @@ -109,14 +313,45 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str logger.error(f"Scan failed: {e}") return {"status": "failed", "error": str(e)} - # 4. Result + # 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker) + error_lines_list = [] + errors_detail: List[Dict[str, Any]] = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + if len(errors_detail) < 500: + errors_detail.append( + { + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + } + ) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Failed to store error lines in Redis: {e}") + + # 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV) return { "status": "waiting_confirmation", "job_id": job_id, "total_rows": processed_rows, "error_count": error_count, "valid_rows": processed_rows - error_count, - "error_file": error_path + "errors": errors_detail, } def validate_row_phase_1( @@ -124,13 +359,61 @@ def validate_row_phase_1( target: str, line_num: int, date_format: Optional[str], -) -> Dict[str, Any]: +) -> Optional[Dict[str, Any]]: """ - Minimal validation: Unique IDs and Dates. + Validation: Unique IDs, Dates, and Numeric constraint checks. Target: 'invoice_header' or 'invoice_details' """ - errors = {} - + def check_decimal(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_decimal(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"} + return None + + def check_int(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_int(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"} + return None + + def check_date(col_name): + date_str = row.get(col_name) + if date_str and str(date_str).strip(): + if not is_valid_date(date_str, date_format): + expected = display_date_format(date_format) + return { + "line": line_num, + "col": col_name, + "msg": f"Formato de fecha inválido ({expected})", + } + return None + + def check_weight(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_weight_unit(val) is None: + return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"} + return None + + def check_currency(col_name): + val = row.get(col_name) + if val and str(val).strip(): + parsed_currency = parse_currency(val, None) + val_norm = normalize_header(val) + # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"} + return None + + def check_transport_type(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES: + return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"} + return None + # A. Invoice Header if target == 'invoice_header': # 1. Unique ID @@ -138,33 +421,180 @@ def validate_row_phase_1( return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} # 2. Date Format - date_str = row.get('FECHA FACTURA') - if date_str: - if not is_valid_date(date_str, date_format): - expected = display_date_format(date_format) - return { - "line": line_num, - "col": "FECHA FACTURA", - "msg": f"Formato inválido ({expected})", - } - else: + date_str = row.get('FECHA FACTURA') or row.get('FECHA') + if not date_str or not str(date_str).strip(): return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"} + + err = check_date('FECHA FACTURA') or check_date('FECHA') + if err: return err + + err = check_date('FECHA EMISION') + if err: return err + + # 3. Numeric Fields + for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']: + err = check_decimal(col) + if err: return err + + # 4. Integer FKs + for col in ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA']: + err = check_int(col) + if err: return err + + # 5. Enums + for col in ['TIPO PESO']: + err = check_weight(col) + if err: return err + + for col in ['TIPO MONEDA']: + err = check_currency(col) + if err: return err + + for col in ['TIPO TRANSPORTE']: + err = check_transport_type(col) + if err: return err # B. Invoice Details (Parts) elif target == 'invoice_details': # 1. Line Number - if not row.get('LINEA'): + if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'): return {"line": line_num, "col": "LINEA", "msg": "Requerido"} # 2. Parent Link (Invoice Number) if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')): return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} - - # 2. Parent Link (Simplified for now, we assume parent exists or is in same batch) - # In a real scenario, we'd check if the invoice exists. - pass + + # 3. Numeric Fields + for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']: + err = check_decimal(col) + if err: return err + + for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']: + err = check_int(col) + if err: return err - return errors if errors else None + return None + + +def validate_row_strict( + row: Dict[str, Any], + target: str, + line_num: int, + date_format: Optional[str], + validator: ForeignKeyValidator, + inv_type_value: str, + invoice_id_cache: Dict[str, Optional[int]], + models: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + err = validate_row_phase_1(row, target, line_num, date_format) + if err: + return err + + InvoiceHeader = models["InvoiceHeader"] + InvoiceType = models["InvoiceType"] + ClientProvider = models["ClientProvider"] + CustomsBroker = models["CustomsBroker"] + RegimenPedimento = models["RegimenPedimento"] + CurrencyType = models["CurrencyType"] + CustomsSection = models["CustomsSection"] + Incoterm = models["Incoterm"] + Part = models["Part"] + + if target == "invoice_header": + if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): + return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"} + + provider_id = parse_int(row.get("CLAVE PROVEEDOR")) + err = validate_tenant_fk_id(validator, ClientProvider, provider_id, line_num, "CLAVE PROVEEDOR", required=True) + if err: + return err + + sold_to_id = parse_int(row.get("CLAVE VENDIDO A")) + err = validate_tenant_fk_id(validator, ClientProvider, sold_to_id, line_num, "CLAVE VENDIDO A", required=True) + if err: + return err + + shipped_to_id = parse_int(row.get("CLAVE ENVIADO A")) + err = validate_tenant_fk_id(validator, ClientProvider, shipped_to_id, line_num, "CLAVE ENVIADO A", required=True) + if err: + return err + + broker_id = parse_int(row.get("AGENTE ADUANAL")) + err = validate_tenant_fk_id(validator, CustomsBroker, broker_id, line_num, "AGENTE ADUANAL") + if err: + return err + + err = validate_public_code( + validator, + RegimenPedimento, + row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"), + line_num, + "CLAVEDOCUMENTO", + ) + if err: + return err + + err = validate_public_code( + validator, + CustomsSection, + row.get("ADUANA DE CRUCE"), + line_num, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + return err + + err = validate_public_code( + validator, + CurrencyType, + row.get("CLAVE MONEDA"), + line_num, + "CLAVE MONEDA", + ) + if err: + return err + + err = validate_public_code( + validator, + Incoterm, + row.get("CLAVE INCOTERM"), + line_num, + "CLAVE INCOTERM", + ) + if err: + return err + + elif target == "invoice_details": + invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() + if not invoice_number: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} + + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] + else: + invoice_id = ( + validator.session.query(InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == validator.tenant_id, + InvoiceHeader.company_id == validator.company_id, + InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, + ) + .scalar() + ) + invoice_id_cache[cache_key] = invoice_id + if not invoice_id: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"} + + part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip() + if not part_num: + return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} + if not validator.check_exists(Part, part_num, field_name="part_number"): + return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"} + + return None def parse_footer_config(config: Optional[str]) -> Dict[str, Any]: if not config: @@ -247,6 +677,16 @@ def parse_decimal(value: Any) -> Optional[Decimal]: return None +def decimal_or_zero(value: Any) -> Decimal: + """Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default).""" + return parse_decimal(value) or Decimal("0") + + +def int_or_zero(value: Any) -> int: + """Return parsed int or 0 for CSV nulls/empty (vanilla default).""" + return parse_int(value) if parse_int(value) is not None else 0 + + def parse_currency(value: Optional[str], currency_type: Optional[str]): from api.v1.modules.a76.invoices.models import Currency if value: @@ -319,13 +759,24 @@ def resolve_public_code( cache[normalized] = normalized if exists is not None else None return cache[normalized] -@shared_task(bind=True) +@celery_app.task(bind=True) def insert_valid_rows(self, job_id: str, model_target: str): """ Pass 2: Re-read CSV, Skip Errors, Bulk Insert. + File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem. """ logger.info(f"Starting Commit for {job_id} target {model_target}") - + + # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + upload_dir = _worker_upload_dir() + file_path = os.path.join(upload_dir, f"{job_id}.csv") + if not os.path.exists(file_path): + return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + try: from api.v1.modules.a76.invoices.models import ( InvoiceHeader, @@ -334,16 +785,18 @@ def insert_valid_rows(self, job_id: str, model_target: str): InvoiceLogistics, InvoiceSalesDetails, OperationType, + TransportType, WeightUnit, ) from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.line_financials.models import LineFinancial @@ -352,30 +805,37 @@ def insert_valid_rows(self, job_id: str, model_target: str): from api.v1.modules.a76.items.line_descriptions.models import LineDescription from api.v1.modules.a76.parts.models import Part - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") - file_path = os.path.join(upload_dir, f"{job_id}.csv") error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") - - # 1. Load Error Line Numbers + + # 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file) error_lines = set() - if os.path.exists(error_path): + try: + r = _get_redis() + raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): with open(error_path, 'r', encoding='utf-8') as f: for line in f: try: err = json.loads(line) error_lines.add(err['line']) - except: pass + except Exception: + pass # Load Metadata (Context) - meta_path = file_path.replace("temp", "temp").replace(".csv", ".meta.json") + meta_path = file_path.replace(".csv", ".meta.json") tenant_id = None company_id = None footer_config = {} + meta = {} if os.path.exists(meta_path): try: with open(meta_path, 'r') as f: - meta = json.load(f) + meta = json.load(f) or {} tenant_id = meta.get('tenant_id') company_id = meta.get('company_id') operation_type_raw = meta.get('operation_type', 'imp') @@ -407,7 +867,7 @@ def insert_valid_rows(self, job_id: str, model_target: str): # Default types from config or fallback op_type_value = OperationType(meta.get('operation_type', 'imp').lower()) - inv_type_value = footer_config.get('invoice_type', 'TEM') + inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM' logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") @@ -436,13 +896,17 @@ def insert_valid_rows(self, job_id: str, model_target: str): reader = csv.DictReader(f, dialect=dialect) + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + for i, row in enumerate(reader, start=1): if i in error_lines: continue - row_norm = normalize_row(row) + row_norm = row_from_template(row, template_id, normalize_header) - # Mapping Logic + # Mapping Logic (solo campos que acepta el modelo de facturas) if model_target == 'invoice_header': invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format) @@ -461,25 +925,148 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue - - # 2. Client/Provider (Tenant) + provider_id = parse_int(row_norm.get('CLAVE PROVEEDOR')) - if provider_id and not validator.check_exists(ClientProvider, provider_id): - skipped_missing_fk += 1 - reason = f"Proveedor ID '{provider_id}' no existe" + err = validate_tenant_fk_id( + validator, + ClientProvider, + provider_id, + i, + "CLAVE PROVEEDOR", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue - # 3. Customs Broker (Tenant) - broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) - if broker_id and not validator.check_exists(CustomsBroker, broker_id): - skipped_missing_fk += 1 - reason = f"Agente Aduanal ID '{broker_id}' no existe" + sold_to_id = parse_int(row_norm.get('CLAVE VENDIDO A')) + err = validate_tenant_fk_id( + validator, + ClientProvider, + sold_to_id, + i, + "CLAVE VENDIDO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue + shipped_to_id = parse_int(row_norm.get('CLAVE ENVIADO A')) + err = validate_tenant_fk_id( + validator, + ClientProvider, + shipped_to_id, + i, + "CLAVE ENVIADO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) + err = validate_tenant_fk_id( + validator, + CustomsBroker, + broker_id, + i, + "AGENTE ADUANAL", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + RegimenPedimento, + row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'), + i, + "CLAVEDOCUMENTO", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CustomsSection, + row_norm.get('ADUANA DE CRUCE'), + i, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CurrencyType, + row_norm.get('CLAVE MONEDA'), + i, + "CLAVE MONEDA", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + Incoterm, + row_norm.get('CLAVE INCOTERM'), + i, + "CLAVE INCOTERM", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + transport_type_val = row_norm.get('TIPO TRANSPORTE') + if transport_type_val and str(transport_type_val).strip().lower() not in TRANSPORT_TYPE_VALUES: + skipped_invalid += 1 + reason = "TIPO TRANSPORTE: Tipo de transporte invalido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + currency_val = row_norm.get('TIPO MONEDA') + if currency_val and str(currency_val).strip(): + parsed_currency = parse_currency(currency_val, None) + val_norm = normalize_header(currency_val) + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + skipped_invalid += 1 + reason = "TIPO MONEDA: Moneda invalida" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + # 2. Client/Provider and broker checks are handled above + # --- 4. Check for Existing Invoice (Upsert Logic) --- existing_header = None if invoice_number: @@ -608,12 +1195,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): financials = InvoiceFinancials( currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type), currency_type=financials_currency_type, - exchange_rate=parse_decimal(row_norm.get('TIPO DE CAMBIO')), - freight=parse_decimal(row_norm.get('FLETES')), - insurance_value=parse_decimal(row_norm.get('VALOR SEGUROS')), - insurance=parse_decimal(row_norm.get('SEGUROS')), - packaging=parse_decimal(row_norm.get('EMBALAJES')), - other_increments=parse_decimal(row_norm.get('OTROS INCREMENTABLES')), + exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')), + freight=decimal_or_zero(row_norm.get('FLETES')), + insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')), + insurance=decimal_or_zero(row_norm.get('SEGUROS')), + packaging=decimal_or_zero(row_norm.get('EMBALAJES')), + other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')), tenant_id=tenant_id, company_id=company_id, ) @@ -621,10 +1208,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) logistics = None if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): + raw_transport = (row_norm.get('TIPO TRANSPORTE') or "none") + transport_str = str(raw_transport).strip().lower() or "none" + try: + transport_type = TransportType(transport_str) + except ValueError: + transport_type = TransportType.NONE logistics = InvoiceLogistics( carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), - transport_type=str(row_norm.get('TIPO TRANSPORTE') or "none").lower(), + transport_type=transport_type, transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), weight_type=weight_type or WeightUnit.KGS, seal_number=(row_norm.get('PRECINTO') or None), @@ -647,8 +1240,9 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_invalid += 1 continue - if invoice_number in invoice_id_cache: - invoice_id = invoice_id_cache[invoice_number] + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] else: invoice_id = ( session.query(InvoiceHeader.id) @@ -656,10 +1250,11 @@ def insert_valid_rows(self, job_id: str, model_target: str): InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, ) .scalar() ) - invoice_id_cache[invoice_number] = invoice_id + invoice_id_cache[cache_key] = invoice_id if not invoice_id: logger.warning( @@ -670,6 +1265,20 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_missing_invoice += 1 continue + part_num = (row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or '').strip() + if not part_num: + skipped_invalid += 1 + reason = "NUMPARTE: Requerido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + if not validator.check_exists(Part, part_num, field_name="part_number"): + skipped_missing_fk += 1 + reason = f"NUMPARTE '{part_num}' no existe" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + # --- Prevent Duplicates: Clear existing items for this invoice (Once per job) --- if invoice_id not in cleared_invoices: logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates") @@ -688,19 +1297,17 @@ def insert_valid_rows(self, job_id: str, model_target: str): # --- NEW LOGIC: Expanded Anexo 76 Structure --- # A. Find/Cache Part - part_num = (row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or '').strip() part_id = None - if part_num: - part_id = part_cache.get(part_num) - if part_id is None: - p = session.query(Part.id).filter( - Part.part_number == part_num, - Part.tenant_id == tenant_id, - Part.company_id == company_id - ).first() - if p: - part_id = p.id - part_cache[part_num] = part_id + part_id = part_cache.get(part_num) + if part_id is None: + p = session.query(Part.id).filter( + Part.part_number == part_num, + Part.tenant_id == tenant_id, + Part.company_id == company_id + ).first() + if p: + part_id = p.id + part_cache[part_num] = part_id line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA')) line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) @@ -727,23 +1334,23 @@ def insert_valid_rows(self, job_id: str, model_target: str): session.add(line) session.flush() # Need line.id - # 3. Financial Data + # 3. Financial Data (vanilla: nulls from CSV -> 0) price = parse_decimal(row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO')) val_com = parse_decimal(row_norm.get('VALOR COMERCIAL') or row_norm.get('VALORCOMERCIAL')) qty = parse_decimal(row_norm.get('CANTIDAD')) - + commercial_total = val_com or (price * qty if price and qty else None) + session.add(LineFinancial( item_line_id=line.id, - unit_price=price, - commercial_value=val_com or (price * qty if price and qty else None), + unit_cost_capture=decimal_or_zero(price), + total_commercial_value=decimal_or_zero(commercial_total), )) - # 4. Quantities - if qty: - session.add(LineQuantity( - item_line_id=line.id, - quantity=qty, - )) + # 4. Quantities (vanilla: nulls -> 0 so we always have a quantity row) + session.add(LineQuantity( + item_line_id=line.id, + quantity=decimal_or_zero(qty), + )) # 5. Customs/Fraction origin = row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') @@ -763,12 +1370,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): description_spanish=desc, )) - # 7. Legacy Sales Details (For specific audit/UI fields) + # 7. Legacy Sales Details (For specific audit/UI fields; vanilla: nulls -> 0) detail = InvoiceSalesDetails( invoice_id=invoice_id, line_number=line_num, sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), - line_bundles=parse_int(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), + line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), tenant_id=tenant_id, company_id=company_id, ) @@ -850,15 +1457,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): logger.error(traceback.format_exc()) return {"status": "failed", "error": str(e)} - # 5. Cleanup + # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely try: - if os.path.exists(file_path): + if file_path and os.path.exists(file_path): os.remove(file_path) if os.path.exists(error_path): os.remove(error_path) - except: - logger.warning("Failed to cleanup temp files") - + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) + # Ensure response is defined (fallback in case of unexpected errors) if response is None: logger.error(f"Unexpected error: response not set for job {job_id}") diff --git a/backend/api/v1/modules/a76/imports/template_config.py b/backend/api/v1/modules/a76/imports/template_config.py new file mode 100644 index 00000000..1a3fc274 --- /dev/null +++ b/backend/api/v1/modules/a76/imports/template_config.py @@ -0,0 +1,118 @@ +""" +Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean. +La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora. +Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models). +""" + +from typing import Dict, List, Any, Optional + +# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV). +# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que +# espera la lógica de validación e insert (tasks.py). +# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA). + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + # --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) --- + "imp_temp_header": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, + {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, + {"canonical": "FECHA EMISION"}, + {"canonical": "CLAVE PROVEEDOR"}, + {"canonical": "CLAVE VENDIDO A"}, + {"canonical": "CLAVE ENVIADO A"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, + {"canonical": "ADUANA DE CRUCE"}, + {"canonical": "CLAVE MONEDA"}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "TIPO PESO"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "REMESA"}, + {"canonical": "AGENTE ADUANAL"}, + {"canonical": "FLETES"}, + {"canonical": "VALOR SEGUROS"}, + {"canonical": "SEGUROS"}, + {"canonical": "EMBALAJES"}, + {"canonical": "OTROS INCREMENTABLES"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "OBSERVACIONES E"}, + {"canonical": "OBSERVACIONES I"}, + {"canonical": "E DOCUMENT"}, + {"canonical": "NUM OPERACION"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "PRECINTO"}, + ], + # --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura --- + "imp_def_header": None, # se resuelve igual que imp_temp_header + # --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura --- + "exp_def_header": None, + # --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) --- + "imp_temp_details": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]}, + {"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]}, + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]}, + {"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]}, + {"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]}, + {"canonical": "CANTIDAD"}, + {"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]}, + {"canonical": "FRACCION"}, + {"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]}, + ], + # --- Partidas: Impo Def y Expo - misma estructura --- + "imp_def_details": None, + "exp_def_details": None, +} + + +def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]: + cols = TEMPLATE_COLUMNS.get(template_id) + if cols is not None: + return cols + if template_id in ("imp_def_header", "exp_def_header"): + return TEMPLATE_COLUMNS.get("imp_temp_header") + if template_id in ("imp_def_details", "exp_def_details"): + return TEMPLATE_COLUMNS.get("imp_temp_details") + return None + + +def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]: + """ + Construye un diccionario: normalized_header -> canonical_name. + normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos). + """ + cols = _resolve_template_columns(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]: + """ + A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict + solo con las columnas de la plantilla, usando nombres canónicos. + Así la plantilla se respeta: solo entran columnas definidas en la plantilla. + """ + lookup = build_normalized_lookup(template_id, normalize_header_fn) + if not lookup: + # Sin template definido: comportamiento legacy (normalizar todo) + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/invoices/catalog_service.py b/backend/api/v1/modules/a76/invoices/catalog_service.py index 2dee90a7..2c7feb47 100644 --- a/backend/api/v1/modules/a76/invoices/catalog_service.py +++ b/backend/api/v1/modules/a76/invoices/catalog_service.py @@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.public.reference_data.transport_types.models import TransportType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection +from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.transport_modes.models import TransportMode diff --git a/backend/api/v1/modules/a76/invoices/schemas.py b/backend/api/v1/modules/a76/invoices/schemas.py index f117377e..55124a29 100644 --- a/backend/api/v1/modules/a76/invoices/schemas.py +++ b/backend/api/v1/modules/a76/invoices/schemas.py @@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel): operation_type: Optional[OperationType] = Field( ..., description="Operation type: imp/exp/sm/ctm" ) + + @field_validator("operation_type", mode="before") + @classmethod + def normalize_operation_type(cls, v): + """Accept DB string (e.g. 'IMP') and coerce to enum value ('imp').""" + if isinstance(v, str): + return v.lower() if v else v + return v invoice_type: Optional[str] = Field( None, max_length=5, description="Invoice type key" ) @@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel): class InvoiceFinancialsBase(BaseModel): """Base fields for Financials""" - currency: Currency = Field(None, max_length=7, description="Currency code") + currency: Optional[Currency] = Field(None, max_length=7, description="Currency code") currency_type: Optional[str] = Field("USD", description="Currency type") - exchange_rate: Decimal = Field(0.00, description="Exchange rate") + exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate") exchange_rate_mm: Optional[Decimal] = Field( None, description="Exchange rate currency to currency" ) diff --git a/backend/api/v1/modules/a76/invoices/services.py b/backend/api/v1/modules/a76/invoices/services.py index f232366d..90e6a84a 100644 --- a/backend/api/v1/modules/a76/invoices/services.py +++ b/backend/api/v1/modules/a76/invoices/services.py @@ -67,11 +67,13 @@ class InvoiceService: # Apply filters if provided if filters: - if filters.get("status"): + if filters.get("status") is not None: query = query.filter(models.InvoiceHeader.is_updated == filters["status"]) if filters.get("operation_type"): + ot = filters["operation_type"] + ot_val = ot.value if hasattr(ot, "value") else ot query = query.filter( - models.InvoiceHeader.operation_type == filters["operation_type"] + models.InvoiceHeader.operation_type == ot_val ) if filters.get("invoice_type"): query = query.filter( @@ -89,10 +91,9 @@ class InvoiceService: f"%{filters['pedimento']}%" ) ) - if ( - not filters.get("invoice_type") - and filters.get("operation_type") == "exp" - ): + ot_exp = filters.get("operation_type") + ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp + if not filters.get("invoice_type") and ot_exp_val == "exp": query = query.filter(models.InvoiceHeader.operation_type != "REPAR") if filters.get("manifest_number"): diff --git a/backend/api/v1/modules/a76/pedmientos/imports/__init__.py b/backend/api/v1/modules/a76/pedmientos/imports/__init__.py new file mode 100644 index 00000000..8378bf88 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Pedimentos (upload → scan → commit) diff --git a/backend/api/v1/modules/a76/pedmientos/imports/routes.py b/backend/api/v1/modules/a76/pedmientos/imports/routes.py new file mode 100644 index 00000000..f624e5dc --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Pedimentos. +Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + PED_IMPORT_FILE_PREFIX, + PED_IMPORT_META_PREFIX, + PED_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Pedimentos import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "pedimentos", + } + + try: + r = _get_redis() + r.set( + f"{PED_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=PED_IMPORT_REDIS_TTL, + ) + r.set( + f"{PED_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=PED_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Pedimentos import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Pedimentos import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/pedmientos/imports/schemas.py b/backend/api/v1/modules/a76/pedmientos/imports/schemas.py new file mode 100644 index 00000000..6fe69385 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: Optional[str] = None + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/pedmientos/imports/tasks.py b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py new file mode 100644 index 00000000..82b630d2 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py @@ -0,0 +1,563 @@ +""" +Tareas Celery para importación CSV de Pedimentos. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from datetime import datetime +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con otros imports) +PED_IMPORT_FILE_PREFIX = "ped_import_file:" +PED_IMPORT_META_PREFIX = "ped_import_meta:" +PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" +PED_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"ped_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{PED_IMPORT_FILE_PREFIX}{job_id}", + f"{PED_IMPORT_META_PREFIX}{job_id}", + f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_pedimento( + row: Dict[str, Any], + line_num: int, + valid_client_ids: Set[int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], +) -> Optional[Dict[str, Any]]: + """Valida una fila para Pedimento. Retorna error dict o None.""" + # Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN + year = (row.get("AÑO") or "").strip() + if not year: + return {"line": line_num, "col": "AÑO", "msg": "Requerido"} + if len(year) > 2: + return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"} + + customs_office = (row.get("ADUANA") or "").strip() + if not customs_office: + return {"line": line_num, "col": "ADUANA", "msg": "Requerido"} + if len(customs_office) > 3: + return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"} + + license_val = (row.get("PATENTE") or "").strip() + if not license_val: + return {"line": line_num, "col": "PATENTE", "msg": "Requerido"} + if len(license_val) > 4: + return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"} + + pedimento_number = (row.get("NUMERO") or "").strip() + if not pedimento_number: + return {"line": line_num, "col": "NUMERO", "msg": "Requerido"} + if len(pedimento_number) > 7: + return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"} + + client_id_str = (row.get("CLIENTE_ID") or "").strip() + if not client_id_str: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"} + try: + client_id = int(client_id_str) + except ValueError: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"} + if client_id not in valid_client_ids: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"} + + pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip() + if not pedimento_code: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"} + if len(pedimento_code) > 2: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"} + if pedimento_code not in valid_pedimento_codes: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"} + + regime = (row.get("REGIMEN") or "").strip() + if not regime: + return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"} + if len(regime) > 3: + return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"} + if regime not in valid_regimes: + return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"} + + # Optional numeric/string fields - validate format if present + status = (row.get("ESTATUS") or "").strip() + if status and len(status) > 30: + return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"} + + for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]: + val = (row.get(col) or "").strip() + if not val: + continue + try: + Decimal(val.replace(",", ".")) + except (InvalidOperation, ValueError): + return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} + + operation_type = (row.get("TIPO_OPERACION") or "").strip().lower() + if operation_type and operation_type not in ("imp", "exp", ""): + return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"} + + pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower() + if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""): + return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"} + + return None + + +def _parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + return Decimal(str(val).strip().replace(",", ".")) + except (InvalidOperation, ValueError): + return None + + +def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]: + """Build dict for PedimentosCreate from normalized CSV row (canonical names).""" + year = (row.get("AÑO") or "").strip()[:2] + customs_office = (row.get("ADUANA") or "").strip()[:3] + license_val = (row.get("PATENTE") or "").strip()[:4] + pedimento_number = (row.get("NUMERO") or "").strip()[:7] + client_id_str = (row.get("CLIENTE_ID") or "").strip() + client_id = int(client_id_str) if client_id_str else None + pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2] + regime = (row.get("REGIMEN") or "").strip()[:3] + + data = { + "year": year, + "customs_office": customs_office, + "license": license_val, + "pedimento_number": pedimento_number, + "client_id": client_id, + "pedimento_code": pedimento_code, + "regime": regime, + } + + op = (row.get("TIPO_OPERACION") or "").strip().lower() + if op in ("imp", "exp"): + data["operation_type"] = op + + ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower() + if ptype in ("normal", "consolidated", "complementary", "automobile"): + data["pedimento_type"] = ptype + + status = (row.get("ESTATUS") or "").strip() + if status: + data["status"] = status[:30] + + data["usd_value"] = _parse_decimal(row.get("VALOR_USD")) + data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO")) + data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO")) + data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO")) + + obs = (row.get("OBSERVACIONES") or "").strip() + if obs: + data["observations"] = obs + + return data + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"Pedimentos import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Pedimentos import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + # Load valid FK sets for validation + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + for cp in session.query(ClientProvider).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + except Exception as e: + logger.error(f"Pedimentos import: failed to load FK sets: {e}") + return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, "pedimentos") + err = _validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Pedimentos import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=PED_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create. + """ + logger.info(f"Pedimentos import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + # Reload FK sets for commit-time validation + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + for cp in session.query(ClientProvider).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + except Exception as e: + logger.error(f"Pedimentos import: failed to load FK sets: {e}") + return {"status": "failed", "error": str(e)} + + from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate + from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate + from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService + + inserted_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, "pedimentos") + err = _validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + try: + data = _row_to_pedimentos_create(row_norm) + # Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL + if "pedimento_dates" not in data or data.get("pedimento_dates") is None: + data["pedimento_dates"] = PedimentoDatesCreate( + entry_date=datetime.now(), + end_date=datetime.now(), + ) + create_data = PedimentosCreate(**data) + PedimentosService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except ValueError as ve: + if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): + skipped_duplicate += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + else: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + except Exception as e: + logger.warning(f"Pedimentos import line {i}: {e}") + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(e)}) + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Pedimentos import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/pedmientos/imports/template_config.py b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py new file mode 100644 index 00000000..06fea167 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py @@ -0,0 +1,53 @@ +""" +Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "pedimentos": [ + {"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]}, + {"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]}, + {"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]}, + {"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]}, + {"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]}, + {"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]}, + {"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]}, + {"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]}, + {"canonical": "REGIMEN", "aliases": ["REGIME"]}, + {"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]}, + {"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]}, + {"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]}, + {"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]}, + {"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]}, + {"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla pedimentos.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/pedmientos/router.py b/backend/api/v1/modules/a76/pedmientos/router.py index f294cba4..2b987b46 100644 --- a/backend/api/v1/modules/a76/pedmientos/router.py +++ b/backend/api/v1/modules/a76/pedmientos/router.py @@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import ( from .routes.pedimento_transport_means import router as pedimento_transport_means_router from .routes.pedimento_validation import router as pedimento_validation_router from .routes.pedimentos import router as pedimentos_router +from .imports.routes import router as pedimentos_imports_router router = APIRouter() @@ -117,3 +118,8 @@ router.include_router( router.include_router( pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"] ) +router.include_router( + pedimentos_imports_router, + prefix="/pedimentos/imports", + tags=["a76 / pedimentos / csv_import"], +) diff --git a/backend/api/v1/modules/a76/router.py b/backend/api/v1/modules/a76/router.py index 570b247b..5f5d23bd 100644 --- a/backend/api/v1/modules/a76/router.py +++ b/backend/api/v1/modules/a76/router.py @@ -12,7 +12,7 @@ from .general_catalogs.router import router as general_catalogs_router from .invoices.routes import router as invoices_router from .items.routes import router as items_router from .classes import router as classes_router -from .classes import router as classes_router + from .clients_and_providers import router as client_and_provider_router from .imports.routes import router as imports_router from .invoice_settings.routes import router as invoice_settings_router diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index e5648d42..9fb07c5c 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -1,6 +1,13 @@ import os from celery import Celery +# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen (mapper) +from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode +from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento +from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, +) + # Import models in correct order for SQLAlchemy relationship resolution # CRITICAL: FaLineItem must be imported BEFORE LineItem from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem # noqa: F401 @@ -26,6 +33,12 @@ celery_app.conf.update( "api.v1.modules.a76.reports.movements.invoices.tasks", "api.v1.modules.a76.reports.exportacion.descargo.task", "api.v1.modules.a76.imports.tasks", + "api.v1.modules.a76.customs_brokers.imports.tasks", + "api.v1.modules.a76.clients_and_providers.imports.tasks", + "api.v1.modules.a76.pedmientos.imports.tasks", + "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", + "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", + "api.v1.modules.a76.classes.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/backend/core/error_handlers.py b/backend/core/error_handlers.py index d038f4ab..36705ca5 100644 --- a/backend/core/error_handlers.py +++ b/backend/core/error_handlers.py @@ -11,11 +11,23 @@ from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from .config import settings from .exceptions import BaseAPIException logger = logging.getLogger(__name__) +def _cors_headers(request: Request) -> Dict[str, str]: + """CORS headers for error responses so browser does not block on 4xx/5xx.""" + origin = request.headers.get("origin") + if not origin or origin not in settings.cors_origins_list: + return {} + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + } + + async def base_exception_handler( request: Request, exc: BaseAPIException, @@ -36,10 +48,13 @@ async def base_exception_handler( if hasattr(exc, "errors") and exc.errors: logger.warning(f"Validation errors details: {exc.errors}") - return JSONResponse( + response = JSONResponse( status_code=exc.status_code, content=jsonable_encoder(exc.to_dict()), ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def validation_exception_handler( @@ -65,7 +80,7 @@ async def validation_exception_handler( extra={"errors": errors}, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, content={ "error": "VALIDATION_ERROR", @@ -74,6 +89,9 @@ async def validation_exception_handler( "errors": errors, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def integrity_error_handler( @@ -105,7 +123,7 @@ async def integrity_error_handler( else: error_message = "Error de integridad en la base de datos" - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_409_CONFLICT, content={ "error": "DATABASE_INTEGRITY_ERROR", @@ -113,6 +131,9 @@ async def integrity_error_handler( "status_code": status.HTTP_409_CONFLICT, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def sqlalchemy_error_handler( @@ -131,7 +152,7 @@ async def sqlalchemy_error_handler( exc_info=True, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "error": "DATABASE_ERROR", @@ -139,6 +160,9 @@ async def sqlalchemy_error_handler( "status_code": status.HTTP_500_INTERNAL_SERVER_ERROR, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def http_exception_handler( @@ -174,7 +198,7 @@ async def general_exception_handler( exc_info=True, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "error": "INTERNAL_SERVER_ERROR", @@ -182,6 +206,9 @@ async def general_exception_handler( "status_code": status.HTTP_500_INTERNAL_SERVER_ERROR, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response def register_exception_handlers(app) -> None: diff --git a/backend/main.py b/backend/main.py index 8b468c74..db63a12d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -17,8 +17,13 @@ from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.material_types.models import MaterialType from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod +# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen para que +# SQLAlchemy resuelva los nombres en relationship() al configurar el mapper from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento +from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, +) from api.v1.modules.public.reference_data.sectors.models import Sector from api.v1.modules.public.reference_data.states.models import State from api.v1.modules.public.reference_data.transport_modes.models import TransportMode @@ -134,6 +139,20 @@ logger = logging.getLogger(__name__) register_exception_handlers(app) +def _cors_headers_for_request(request: Request): + """Return CORS headers if request Origin is allowed (so error responses don't get blocked by browser).""" + origin = request.headers.get("origin") + if not origin: + return {} + allowed = settings.cors_origins_list + if origin in allowed: + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + } + return {} + + # Add validation error handler @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): @@ -141,12 +160,29 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE f"Validation error for {request.method} {request.url.path}: {exc.errors()}" ) logger.error(f"Request body: {await request.body()}") - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_400_BAD_REQUEST, content={"detail": exc.errors(), "body": exc.body}, ) + for k, v in _cors_headers_for_request(request).items(): + response.headers[k] = v + return response +# Add HTTP exception handler +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + logger.error( + f"HTTP {exc.status_code} for {request.method} {request.url.path}: {exc.detail}" + ) + response = JSONResponse( + status_code=exc.status_code, + content={"detail": exc.detail}, + ) + for k, v in _cors_headers_for_request(request).items(): + response.headers[k] = v + return response + def run_migrations(): subprocess.run(["alembic", "upgrade", "head"], check=True) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index baa8f75c..bb5f6a9a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -341,6 +341,129 @@ export const api = { validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`) }, + imports: { + upload: ( + file: File, + modelTarget: string, + footerConfig: any, + companyId: number, + operationType: string, + templateId?: string + ) => { + const formData = new FormData(); + formData.append('file', file); + if (footerConfig) { + formData.append('footer_config', JSON.stringify(footerConfig)); + } + if (templateId) { + formData.append('template_id', templateId); + } + + const queryParams = new URLSearchParams({ + company_id: String(companyId), + operation_type: operationType || 'imp' + }).toString(); + + return fetchApi(`/v1/a76/imports/upload/${modelTarget}?${queryParams}`, { + method: 'POST', + body: formData + }); + }, + status: (jobId: string) => api.get(`/v1/a76/imports/${jobId}/status`), + commit: (jobId: string, modelTarget: string) => + api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget }) + }, + + // CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports) + customsBrokerImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/customs-brokers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/customs-brokers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {}) + }, + + // CSV import for Clientes y Proveedores (flujo propio en clients_and_providers/imports) + clientProviderImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/clients-providers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/clients-providers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {}) + }, + + // CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports) + exchangeRateImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/exchange-rate/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {}) + }, + + // CSV import for Fracción Americana (us_tariff_fractions/imports) + americanFractionImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/us-tariff-fractions/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/us-tariff-fractions/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {}) + }, + + // CSV import for Pedimentos (pedimentos/imports) + pedimentosImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/pedimentos/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/pedimentos/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {}) + }, + + // CSV import for Clases de Materiales (classes/imports) + materialClassImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/classes/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/classes/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/classes/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte index 7375b3ab..bd7bac46 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte @@ -143,17 +143,53 @@ {#if scanResults.error_count > 0}

Se detectaron problemas en el archivo

- Las filas con errores serán omitidas automáticamente. Solo se importarán los - registros válidos. + Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para + importar solo las filas válidas (las erróneas se omitirán).

+ {#if scanResults.errors && scanResults.errors.length > 0} +
+
+
+ Detalle de errores (para corregir en el CSV) +
+ + {scanResults.errors.length} error(es) + +
+
+ + + + + + + + + + {#each scanResults.errors as err} + + + + + + {/each} + +
LíneaColumnaMensaje
{err.line}{err.col || '-'}{err.msg || '-'}
+
+
+ {/if} {:else}
diff --git a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte index 8084f135..62830c66 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte @@ -123,7 +123,7 @@ ondragover={(e) => handleDragOver(e, item.disabled)} ondrop={(e) => handleDrop(e, item)} oncontextmenu={(e) => handleContextMenu(e, item)} - roles="button" + role="button" tabindex={item.disabled ? -1 : 0} onclick={() => handleClick(item.id, item.disabled)} onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)} diff --git a/frontend/src/lib/components/dashboard/customs_brokers/columns.ts b/frontend/src/lib/components/dashboard/customs_brokers/columns.ts index 3ea92e34..add7af3f 100644 --- a/frontend/src/lib/components/dashboard/customs_brokers/columns.ts +++ b/frontend/src/lib/components/dashboard/customs_brokers/columns.ts @@ -8,6 +8,20 @@ export type { CustomsBroker }; export function createColumns(onSuccess?: () => void): ColumnDef[] { return [ + { + accessorKey: "id", + header: "ID", + cell: ({ row }) => { + const idSnippet = createRawSnippet<[{ id: number }]>((getId) => { + const { id } = getId(); + return { + render: () => + `${id}` + }; + }); + return renderSnippet(idSnippet, { id: row.original.id }); + } + }, { accessorKey: "broker_key", header: "Clave", diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index 656b0459..2deb6775 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -249,7 +249,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Encabezado', icon: FileText, group: 'Impo. Def.', - modelTarget: 'InvoiceHeader', + modelTarget: 'invoice_header', templateUrl: '/csv/EstructuraEncFacImpoDef.xls' }, { @@ -257,7 +257,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Partidas', icon: Package, group: 'Impo. Def.', - modelTarget: 'InvoiceSalesDetails', + modelTarget: 'invoice_details', templateUrl: '/csv/EstructuraParFacImpoDefAF.xls' }, { @@ -274,7 +274,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Encabezado', icon: FileText, group: 'Compras Mex.', - modelTarget: 'InvoiceHeader', + modelTarget: 'invoice_header', disabled: true, }, { @@ -282,7 +282,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Partidas', icon: Package, group: 'Compras Mex.', - modelTarget: 'InvoiceSalesDetails', + modelTarget: 'invoice_details', disabled: true, }, { @@ -302,7 +302,7 @@ export const exportacionConfig: CsvUploadItem[] = [ title: 'Encabezado', icon: FileText, group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceHeader', + modelTarget: 'invoice_header', templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls' }, { @@ -310,7 +310,7 @@ export const exportacionConfig: CsvUploadItem[] = [ title: 'Partidas', icon: Package, group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceSalesDetails', + modelTarget: 'invoice_details', templateUrl: '/csv/EstructuraParExpoCamReg.xls' }, { diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 6c55bc56..eb8598a9 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -24,6 +24,18 @@ let scanResults = $state(null); let commitResults = $state(null); let showResultModal = $state(false); + // Cuando es true, usamos API de importación de Agentes Aduanales (customs_brokers/imports) + let useCustomsBrokerImport = $state(false); + // Cuando es true, usamos API de importación de Clientes y Proveedores (clients_and_providers/imports) + let useClientProviderImport = $state(false); + // Cuando es true, usamos API de importación de Tipos de Cambio (exchange_rate/imports) + let useExchangeRateImport = $state(false); + // Cuando es true, usamos API de importación de Fracción Americana (us_tariff_fractions/imports) + let useAmericanFractionImport = $state(false); + // Cuando es true, usamos API de importación de Pedimentos (pedimentos/imports) + let usePedimentosImport = $state(false); + // Cuando es true, usamos API de importación de Clases de Materiales (classes/imports) + let useMaterialClassesImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -38,25 +50,153 @@ }); async function handleUpload(file: File, config: CsvUploadItem) { + console.log('handleUpload started', { file, config }); isUploading = true; activeModelTarget = config.modelTarget || null; scanResults = null; - const currentSettings = allSettings[activeTab] || {}; + useCustomsBrokerImport = config.id === 'customs_brokers'; + useClientProviderImport = config.id === 'clients_providers'; + useExchangeRateImport = config.id === 'exchange_rates'; + useAmericanFractionImport = config.id === 'american_fractions'; + usePedimentosImport = config.id === 'pedimentos'; + useMaterialClassesImport = config.id === 'material_classes'; + const companyId = companyStore.activeCompany?.id || 1; + + if (useCustomsBrokerImport) { + try { + const res = await api.customsBrokerImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (useClientProviderImport) { + try { + const res = await api.clientProviderImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (useExchangeRateImport) { + try { + const res = await api.exchangeRateImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (useAmericanFractionImport) { + try { + const res = await api.americanFractionImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (usePedimentosImport) { + try { + const res = await api.pedimentosImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (useMaterialClassesImport) { + try { + const res = await api.materialClassImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + const currentSettings = allSettings[activeTab] || {}; + const footerConfig = { ...currentSettings }; + if (activeTab === 'importacion') { + footerConfig.invoice_type = config.id?.startsWith('imp_def_') ? 'DEF' : 'TEM'; + } const opType = activeTab === 'exportacion' ? 'exp' : 'imp'; - const res = await api.imports.upload( - file, - config.modelTarget || '', - currentSettings, - companyId, - opType - ); - if (res.data?.job_id) { - currentJobId = res.data.job_id; - pollStatus(); - } else { - toast.error('Error al subir el archivo'); + try { + const res = await api.imports.upload( + file, + config.modelTarget || '', + footerConfig, + companyId, + opType, + config.id + ); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); isUploading = false; } } @@ -64,54 +204,91 @@ async function pollStatus() { if (!currentJobId) return; - const res = await api.imports.status(currentJobId); - if (res.data?.status === 'waiting_confirmation') { - scanResults = res.data; - showResultModal = true; - toast.success('Escaneo completado. Revisa los resultados.'); - isUploading = false; - } else if (res.data?.status === 'failed') { - toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido')); - isUploading = false; - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - } else if (res.data?.status === 'warning') { - // Caso cuando no se insertaron registros pero hay información de rechazo - commitResults = res.data; - showResultModal = true; - const inserted = res.data?.inserted || 0; - const skippedInvalid = res.data?.skipped_invalid || 0; - const skippedFk = res.data?.skipped_missing_fk || 0; - const totalSkipped = skippedInvalid + skippedFk; - - if (inserted === 0) { - toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`); - } else { - toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`); + try { + const res = useCustomsBrokerImport + ? await api.customsBrokerImports.status(currentJobId) + : useClientProviderImport + ? await api.clientProviderImports.status(currentJobId) + : useExchangeRateImport + ? await api.exchangeRateImports.status(currentJobId) + : useAmericanFractionImport + ? await api.americanFractionImports.status(currentJobId) + : usePedimentosImport + ? await api.pedimentosImports.status(currentJobId) + : useMaterialClassesImport + ? await api.materialClassImports.status(currentJobId) + : await api.imports.status(currentJobId); + console.log('Poll response', res); + if (res.error && !res.data) { + toast.error(res.error || 'Error al consultar el estado'); + isUploading = false; + currentJobId = null; + return; } - isUploading = false; - } else if (res.data?.status === 'finished') { - commitResults = res.data; - showResultModal = true; - const inserted = res.data?.inserted || 0; - const skippedInvalid = res.data?.skipped_invalid || 0; - const skippedFk = res.data?.skipped_missing_fk || 0; - const skippedDetails = res.data?.skipped_details || []; + if (res.data?.status === 'waiting_confirmation') { + scanResults = res.data; + showResultModal = true; + toast.success('Escaneo completado. Revisa los resultados.'); + isUploading = false; + } else if (res.data?.status === 'failed' || res.data?.status === 'FAILURE') { + const errRaw = res.data.error; + const errText = + typeof errRaw === 'string' + ? errRaw.includes('finished') && errRaw.includes('inserted') + ? 'La importación pudo completarse. Revisa el listado de registros.' + : errRaw + : (errRaw?.message ?? 'Error desconocido'); + toast.error('Error en el procesamiento: ' + errText); + isUploading = false; + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + } else if (res.data?.status === 'warning') { + // Caso cuando no se insertaron registros pero hay información de rechazo + commitResults = res.data; + showResultModal = true; + const inserted = res.data?.inserted || 0; + const skippedInvalid = res.data?.skipped_invalid || 0; + const skippedFk = res.data?.skipped_missing_fk || 0; + const skippedDup = res.data?.skipped_duplicate || 0; + const totalSkipped = skippedInvalid + skippedFk + skippedDup; - if (inserted > 0) { - toast.success(`Importación completada: ${inserted} registros insertados`); - if (skippedInvalid > 0 || skippedFk > 0) { - const totalSkipped = skippedInvalid + skippedFk; - toast.warning(`${totalSkipped} registros fueron rechazados`); + if (inserted === 0) { + toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`); + } else { + toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`); } + isUploading = false; + } else if (res.data?.status === 'finished') { + commitResults = res.data; + showResultModal = true; + const inserted = res.data?.inserted || 0; + const skippedInvalid = res.data?.skipped_invalid || 0; + const skippedFk = res.data?.skipped_missing_fk || 0; + const skippedDup = res.data?.skipped_duplicate || 0; + const skippedDetails = res.data?.skipped_details || []; + + if (inserted > 0) { + toast.success(`Importación completada: ${inserted} registros insertados`); + if (skippedInvalid > 0 || skippedFk > 0 || skippedDup > 0) { + const totalSkipped = skippedInvalid + skippedFk + skippedDup; + toast.warning(`${totalSkipped} registros fueron rechazados`); + } + } else { + toast.error('No se insertaron registros. Revisa los errores a continuación.'); + } + isUploading = false; } else { - toast.error('No se insertaron registros. Revisa los errores a continuación.'); + // Continue polling + console.log('Status not final, polling again in 2s...', res.data?.status); + setTimeout(pollStatus, 2000); } - isUploading = false; - } else { - // Continue polling + } catch (e) { + console.error('Poll exception', e); + // Retry on network error? Or fail? + // For now, let's keep retrying a few times or hard fail. + // Let's just log and retry. setTimeout(pollStatus, 2000); } } @@ -174,16 +351,29 @@ {/if}
- { - if (currentJobId && activeModelTarget) { +{#if scanResults || commitResults} + { + if (!currentJobId) return; try { isUploading = true; - const res = await api.imports.commit(currentJobId, activeModelTarget); + const res = useCustomsBrokerImport + ? await api.customsBrokerImports.commit(currentJobId) + : useClientProviderImport + ? await api.clientProviderImports.commit(currentJobId) + : useExchangeRateImport + ? await api.exchangeRateImports.commit(currentJobId) + : useAmericanFractionImport + ? await api.americanFractionImports.commit(currentJobId) + : usePedimentosImport + ? await api.pedimentosImports.commit(currentJobId) + : useMaterialClassesImport + ? await api.materialClassImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); @@ -192,18 +382,18 @@ toast.error('Error al iniciar la importación'); isUploading = false; } - } - }} - onCancel={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} - onClose={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} -/> + }} + onCancel={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + onClose={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + /> +{/if} diff --git a/scripts/backend-entrypoint.sh b/scripts/backend-entrypoint.sh index 20c5f89f..85cb5432 100755 --- a/scripts/backend-entrypoint.sh +++ b/scripts/backend-entrypoint.sh @@ -19,7 +19,7 @@ wait_for_tcp() { echo "Esperando a que $service esté disponible en ${host}:${port}..." while [ $attempt -le $max_attempts ]; do - if python -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + if python3 -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then echo "✓ $service está listo y accesible" return 0 fi diff --git a/scripts/init_first_time.sh b/scripts/init_first_time.sh index 601d72d4..de1c32ae 100755 --- a/scripts/init_first_time.sh +++ b/scripts/init_first_time.sh @@ -12,11 +12,6 @@ # 6. Relación usuario-tenant en tabla user_tenants # 7. Actualización del tenant_id del usuario con el valor real # 8. Licencia Enterprise para el tenant (ilimitada, 1 año de vigencia) -# 9. [OPCIONAL] Datos iniciales de ejemplo si se pasa --seed-data -# -# Uso: -# ./init_first_time.sh # Solo configuración básica -# ./init_first_time.sh --seed-data # Configuración + datos de ejemplo # # Requisitos: # - Keycloak corriendo en http://localhost:8080 @@ -24,6 +19,8 @@ # - Base de datos anexo76_core creada # - jq instalado (para procesamiento JSON) # +# Puertos: Keycloak en 18080/19000, frontend en 15173, API en 18000. +# # Nota: El atributo tenant_id se crea con valor inicial "1" y luego # se actualiza con el ID real del tenant creado en PostgreSQL. ############################################################################### @@ -33,22 +30,6 @@ set -euo pipefail # Modo strict: exit on error, undefined vars, pipe failures # Trap para cleanup en caso de error trap 'echo -e "\n${RED}✗ Error en línea $LINENO. Script abortado.${NC}" >&2' ERR -# Parsear argumentos -SEED_DATA=false -while [[ $# -gt 0 ]]; do - case $1 in - --seed-data) - SEED_DATA=true - shift - ;; - *) - echo "Uso: $0 [--seed-data]" - echo " --seed-data: Carga datos de ejemplo en las tablas" - exit 1 - ;; - esac -done - # Colores para output RED='\033[0;31m' GREEN='\033[0;32m' @@ -144,172 +125,7 @@ create_tenant_mapper() { fi } -############################################################################### -# Funciones de seed data -############################################################################### - -# Insertar datos de ejemplo para customs_brokers -seed_customs_brokers() { - echo " → Insertando customs brokers..." - exec_pg_sql " - INSERT INTO a76.customs_brokers (tenant_id, company_id, type, broker_key, name, address, postal_code, city, state, phone, email, country, tax_id, license, company, contact, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB001', 'Agente Aduanal García', 'Av. Reforma 123', '01000', 'Ciudad de México', 'CDMX', '5555555555', 'garcia@aduanas.com', 'MEX', 'GAAR800101ABC', '1234', 'García y Asociados', 'Juan García', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB002', 'Agente Aduanal López', 'Blvd. Díaz Ordaz 456', '22000', 'Tijuana', 'BC', '6641234567', 'lopez@customs.com', 'MEX', 'LOPL750505XYZ', '2345', 'López Customs', 'María López', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB003', 'Agente Aduanal Martínez', 'Calle Industria 789', '45000', 'Guadalajara', 'JAL', '3339876543', 'martinez@broker.com', 'MEX', 'MARM850315DEF', '3456', 'Martínez Brokerage', 'Pedro Martínez', now(), now()) - ON CONFLICT (broker_key, tenant_id, company_id) DO NOTHING; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para clients_and_providers -seed_clients_and_providers() { - echo " → Insertando clientes y proveedores..." - exec_pg_sql " - INSERT INTO a76.clients_and_providers (tenant_id, company_id, type_nat_foreign, name, short_name, rfc, client_or_provider, web_key, is_active, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Proveedor Tecnológico SA de CV', 'PROVTECH', 'PTE901201ABC', 'BOTH', 'PROV001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Cliente Industrial del Norte SA', 'CINORTE', 'CIN850615XYZ', 'BOTH', 'CLI001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'E', 'Global Supplies Inc', 'GLOBSUP', 'GSI123456789', 'BOTH', 'BOTH001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Manufacturas del Bajío SA', 'MANBAJIO', 'MDB920310DEF', 'BOTH', 'CLI002', true, now(), now()) - RETURNING id; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para packages -seed_packages() { - echo " → Insertando tipos de paquete..." - exec_pg_sql " - INSERT INTO a76.packages (tenant_id, company_id, key, description_es, description_en, weight_unit, plurals, plural_in, code_ace, code_aamex, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'PK01', 'Caja de Cartón', 'Cardboard Box', 0.5, 'CAJS', 'BOXS', 'CB01', 'CAJA001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK02', 'Pallet de Madera', 'Wooden Pallet', 15.0, 'PLTS', 'PLTS', 'WP01', 'PALL001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK03', 'Tambor Metálico', 'Metal Drum', 10.0, 'TMBS', 'DRMS', 'MD01', 'TAMB001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK04', 'Contenedor', 'Container', 2000.0, 'CONT', 'CONT', 'CT01', 'CONT001', now(), now()) - ON CONFLICT (tenant_id, company_id, key) DO NOTHING; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para classes -seed_classes() { - echo " → Insertando clases..." - exec_pg_sql "INSERT INTO a76.classes (tenant_id, company_id, class_code, description_es, description_en, material_key, unit_of_measure, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'CLS001', 'Componentes Electrónicos', 'Electronic Components', 'MP', 'PZA', '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS002', 'Partes Automotrices', 'Automotive Parts', 'MP', 'KGS', '8708.29.99', '8708.29.9900', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS003', 'Textiles y Telas', 'Textiles and Fabrics', 'MP', 'MT', '5407.20.01', '5407.20.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS004', 'Equipo de Computación', 'Computer Equipment', 'MP', 'PZA', '8471.30.01', '8471.30.0100', now(), now()) ON CONFLICT (tenant_id, company_id, class_code) DO NOTHING;" >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para parts -seed_parts() { - echo " → Insertando partes/componentes..." - - # Obtener un client_id para asociar las partes - local client_id - client_id=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} LIMIT 1;" | xargs) - - if [ -n "$client_id" ]; then - exec_pg_sql "INSERT INTO a76.parts (tenant_id, company_id, client_id, part_number, description_spanish, description_english, part_class, currency_key, unit_of_measure, unit_cost, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-001', 'Microcontrolador ARM Cortex-M4', 'ARM Cortex-M4 Microcontroller', 'CLS001', 'USD', 'PZA', 15.50, '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-002', 'Filtro de Aceite Automotriz', 'Automotive Oil Filter', 'CLS002', 'USD', 'PZA', 8.75, '8421.23.01', '8421.23.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-003', 'Tela de Algodón para Tapicería', 'Cotton Upholstery Fabric', 'CLS003', 'USD', 'MT', 12.00, '5208.31.01', '5208.31.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-004', 'Disco Duro SSD 500GB', '500GB SSD Hard Drive', 'CLS004', 'USD', 'PZA', 65.00, '8471.70.01', '8471.70.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-005', 'Sensor de Temperatura Digital', 'Digital Temperature Sensor', 'CLS001', 'USD', 'PZA', 5.25, '9025.19.01', '9025.19.0100', now(), now()) ON CONFLICT (tenant_id, company_id, part_number) DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Insertar datos de ejemplo para pedimentos -seed_pedimentos() { - echo " → Insertando pedimentos..." - - # Primero obtener IDs de clientes - local client_ids - client_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('CLIENT', 'BOTH') LIMIT 2;") - local client_id_1=$(echo "$client_ids" | sed -n '1p' | xargs) - local client_id_2=$(echo "$client_ids" | sed -n '2p' | xargs) - - if [ -n "$client_id_1" ]; then - exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001234', ${client_id_1}, 'imp', 'normal', 'V1', 'ITE', 'draft', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001235', ${client_id_1}, 'exp', 'normal', 'V1', 'ETE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$client_id_2" ]; then - exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001236', ${client_id_2}, 'imp', 'consolidated', 'V1', 'ITE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Insertar datos de ejemplo para invoices -seed_invoices() { - echo " → Insertando facturas..." - - # Obtener IDs de clientes/proveedores - local provider_ids - provider_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('PROVIDER', 'BOTH') LIMIT 2;") - local provider_id_1=$(echo "$provider_ids" | sed -n '1p' | xargs) - local provider_id_2=$(echo "$provider_ids" | sed -n '2p' | xargs) - - # Insertar facturas en invoice_header (mínimo requerido) - exec_pg_sql "INSERT INTO a76.invoice_header (tenant_id, company_id, system, operation_type, invoice_type, invoice_number, invoice_date, is_updated, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'DEF', 'INV-2024-001', '2024-01-10', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'exp', 'EXDEF', 'INV-2024-002', '2024-02-15', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'TEM', 'INV-2024-003', '2024-03-05', false, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - - # Obtener IDs de las facturas recién creadas - local invoice_ids - invoice_ids=$(exec_pg_sql "SELECT id FROM a76.invoice_header WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND invoice_number IN ('INV-2024-001', 'INV-2024-002', 'INV-2024-003') ORDER BY invoice_number;") - local invoice_id_1=$(echo "$invoice_ids" | sed -n '1p' | xargs) - local invoice_id_2=$(echo "$invoice_ids" | sed -n '2p' | xargs) - local invoice_id_3=$(echo "$invoice_ids" | sed -n '3p' | xargs) - - # Insertar datos financieros para las facturas - if [ -n "$invoice_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'foreign', 17.50, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'foreign', 17.45, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'local', 1.00, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - # Insertar datos de compliance mexicano con proveedores - if [ -n "$invoice_id_1" ] && [ -n "$provider_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ] && [ -n "$provider_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ] && [ -n "$provider_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, ${provider_id_2}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - # Insertar datos de logística con incoterm - if [ -n "$invoice_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'none', 'kgs', 'FOB', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'none', 'kgs', 'CIF', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'none', 'kgs', 'EXW', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Ejecutar todas las funciones de seed -execute_seed_data() { - echo -e "\n${YELLOW}[SEED] Cargando datos de ejemplo...${NC}" - - seed_customs_brokers - seed_clients_and_providers - seed_packages - seed_classes - seed_parts - seed_pedimentos - seed_invoices - - echo -e "${GREEN}✓ Datos de ejemplo cargados exitosamente${NC}" - echo -e "${YELLOW} • 3 Agentes aduanales${NC}" - echo -e "${YELLOW} • 4 Clientes/Proveedores${NC}" - echo -e "${YELLOW} • 4 Tipos de paquete${NC}" - echo -e "${YELLOW} • 4 Clases${NC}" - echo -e "${YELLOW} • 5 Parts/Componentes${NC}" - echo -e "${YELLOW} • 3 Pedimentos${NC}" - echo -e "${YELLOW} • 3 Facturas${NC}" -} - -# Variables de configuración +# Variables de configuración (puertos con prefijo 1 hardcodeados) KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080/kcauth}" KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}" KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}" @@ -328,7 +144,7 @@ DEMO_EMAIL="demo@aduanasoft.com" DEMO_FIRSTNAME="Demo" DEMO_LASTNAME="User" -TENANT_NAME="Aduanasoft" +TENANT_NAME="Aduanasoft"A TENANT_SLUG="aduanasoft" COMPANY_NAME="Aduanasoft S.A. de C.V." COMPANY_RFC="ADS010101AAA" @@ -797,12 +613,14 @@ else echo -e "${YELLOW}⚠ Company ya existe para este tenant${NC}" fi -# Obtener información de la company -COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;") +# Obtener ID de la company (necesario para user_tenants) COMPANY_ID=$(exec_pg_sql "SELECT id FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;" | xargs) - -echo -e "${GREEN}✓ Company: ${COMPANY_INFO}${NC}" -echo -e "${GREEN}✓ Company ID: ${COMPANY_ID}${NC}" +if [ -z "$COMPANY_ID" ]; then + echo -e "${RED}✗ Error: No se pudo obtener el ID de la company${NC}" + exit 1 +fi +COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;") +echo -e "${GREEN}✓ Company ID: ${COMPANY_ID} | ${COMPANY_INFO}${NC}" # Agregar tenant_id al usuario demo en Keycloak echo -e "\n${YELLOW}Asignando tenant_id al usuario demo...${NC}" @@ -818,7 +636,7 @@ curl -s -X PUT "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/users/${USER_ID}" echo -e "${GREEN}✓ Atributo tenant_id asignado al usuario${NC}" -# Agregar relación usuario-tenant en la base de datos +# Agregar relación usuario-tenant en la base de datos (usar company_id real) echo -e "\n${YELLOW}Creando relación usuario-tenant en la base de datos...${NC}" exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null @@ -880,13 +698,6 @@ else fi fi -############################################################################### -# 9. Cargar datos de ejemplo (opcional) -############################################################################### -if [ "$SEED_DATA" = true ]; then - execute_seed_data -fi - ############################################################################### # Resumen final ############################################################################### @@ -925,18 +736,6 @@ echo -e " ${GREEN}✓${NC} Plan: Enterprise (ilimitado)" echo -e " ${GREEN}✓${NC} Status: Activa" echo -e " ${GREEN}✓${NC} Features: API, Reportes Avanzados, Integraciones, Soporte Dedicado" echo -e " ${GREEN}✓${NC} Vigencia: 1 año" - -if [ "$SEED_DATA" = true ]; then -echo -e "\n${YELLOW}Datos de ejemplo:${NC}" -echo -e " ${GREEN}✓${NC} Agentes aduanales: 3" -echo -e " ${GREEN}✓${NC} Clientes/Proveedores: 4" -echo -e " ${GREEN}✓${NC} Tipos de paquete: 4" -echo -e " ${GREEN}✓${NC} Clases: 4" -echo -e " ${GREEN}✓${NC} Parts/Componentes: 5" -echo -e " ${GREEN}✓${NC} Pedimentos: 3" -echo -e " ${GREEN}✓${NC} Facturas: 3" -fi - echo -e "\n${YELLOW}Puedes acceder al sistema en:${NC}" echo -e " ${GREEN}http://localhost:5173${NC}" echo -e "\n${GREEN}════════════════════════════════════════════════════════${NC}\n"