diff --git a/.env.example b/.env.example index 87dcbf5f..46bb47bc 100644 --- a/.env.example +++ b/.env.example @@ -31,16 +31,6 @@ CORE_DB_PASSWORD=postgres # Ejemplo para Hub: http://localhost:5173,http://100.78.6.108:5174,http://100.78.6.108:8001 CORS_ORIGINS=http://localhost:5173,http://localhost:3000 -# ---- API Tipo cambio (External) --- -EXTERNAL_API_URL=http://74.208.80.245:3000 -EXTERNAL_API_USER=devAS -EXTERNAL_API_PASSWORD=T6Y4mqP09Y[-2-3H - -# ----- API SITAR (Warnings) --- -SITAR_API_URL="" -SITAR_API_USER="" -SITAR_API_PASSWORD="" - # ----- Frontend ----- NODE_ENV=development VITE_API_URL=http://localhost:8000/api @@ -52,6 +42,11 @@ VITE_KEYCLOAK_CLIENT_ID=anexo76-frontend #------ Celery / Valkey ---------- VALKEY_URL=redis://valkey:6379/0 +# ----- Sitar API ----- +SITAR_API_URL=http://api.sitar.aduanasoft.com +SITAR_API_USER=user_sitar_api +SITAR_API_PASSWORD=password123 + # ================================== # CONFIGURACIÓN DE SINCRONIZACIÓN # ================================== diff --git a/.gitignore b/.gitignore index cd5adff8..df6980ae 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,11 @@ wheels/ *.egg .pnpm-store/ -# Environment +# Environment (no subir: cada quien puede usar puertos distintos vía .env) .env .env.local +backend/.env +frontend/.env backend/SCRIPTS/ # IDEs .vscode/ @@ -54,7 +56,6 @@ backend/app_data/ .pytest_cache/ .coverage htmlcov/ -backend/celerybeat-schedule # Node (para frontend) **/node_modules/ @@ -69,3 +70,8 @@ backend/uploads/ docker-compose.yml .mypy_cache/ +# Celery +backend/celerybeat-schedule +celerybeat-schedule +celerybeat-schedule.* +celerybeat.pid diff --git a/backend/api/__init__.py b/backend/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/__init__.py b/backend/api/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/__init__.py b/backend/api/v1/modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/a76/__init__.py b/backend/api/v1/modules/a76/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/a76/boms/__init__.py b/backend/api/v1/modules/a76/boms/__init__.py new file mode 100644 index 00000000..ea1f7540 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/__init__.py @@ -0,0 +1,7 @@ +""" +Módulo de importación CSV para BOMs (Bills of Materials). +""" + +from .routes import router + +__all__ = ["router"] diff --git a/backend/api/v1/modules/a76/boms/imports/__init__.py b/backend/api/v1/modules/a76/boms/imports/__init__.py new file mode 100644 index 00000000..7585c5fe --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/__init__.py @@ -0,0 +1 @@ +# boms.imports diff --git a/backend/api/v1/modules/a76/boms/imports/routes.py b/backend/api/v1/modules/a76/boms/imports/routes.py new file mode 100644 index 00000000..4a87ed0e --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para BOMs. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + BOM_IMPORT_FILE_PREFIX, + BOM_IMPORT_META_PREFIX, + BOM_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"BOMs import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "boms", + } + + try: + r = _get_redis() + r.set( + f"{BOM_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=BOM_IMPORT_REDIS_TTL, + ) + r.set( + f"{BOM_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=BOM_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"BOMs import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"BOMs import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/boms/imports/schemas.py b/backend/api/v1/modules/a76/boms/imports/schemas.py new file mode 100644 index 00000000..7827dae6 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/boms/imports/tasks.py b/backend/api/v1/modules/a76/boms/imports/tasks.py new file mode 100644 index 00000000..ac72ad0d --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/tasks.py @@ -0,0 +1,430 @@ +""" +Tareas Celery para importación CSV de BOMs. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista. +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +BOM_IMPORT_FILE_PREFIX = "bom_import_file:" +BOM_IMPORT_META_PREFIX = "bom_import_meta:" +BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:" +BOM_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"BOMs import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"bom_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"BOMs import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{BOM_IMPORT_FILE_PREFIX}{job_id}", + f"{BOM_IMPORT_META_PREFIX}{job_id}", + f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"BOMs import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_bom( + row: Dict[str, Any], + line_num: int, + valid_part_numbers: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Valida una fila BOM según columnas del template. FK opcional a a76.parts.""" + parent = (row.get("NUMPARTE_PADRE") or "").strip() + if not parent: + return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"} + if len(parent) > 70: + return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"} + + component = (row.get("NUMPARTE_COMPONENTE") or "").strip() + if not component: + return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"} + if len(component) > 70: + return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"} + + qty = row.get("CANTIDAD") + if qty is None or qty == "": + return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"} + try: + val = Decimal(str(qty)) + if val < 0: + return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"} + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"} + + uom = (row.get("UNIMED") or "").strip() + if uom and len(uom) > 10: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"} + + def _optional_number(val: Any) -> bool: + if val is None: + return True + s = re.sub(r"\s+", "", str(val).strip()) + if not s: + return True + try: + float(s.replace(",", ".")) + return True + except (ValueError, TypeError): + return False + + version_bom = row.get("VERSION_BOM") + if not _optional_number(version_bom): + return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"} + + version_bill = row.get("VERSION_BILL") + if not _optional_number(version_bill): + return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"} + + # Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas) + if valid_part_numbers is not None and len(valid_part_numbers) > 0: + if parent not in valid_part_numbers: + return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"} + if component not in valid_part_numbers: + return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"BOMs import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"BOMs import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_part_numbers: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.parts.models import Part + for p in ( + session.query(Part.part_number) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + valid_part_numbers.add(p[0]) + except Exception as e: + logger.warning(f"BOMs import: could not load parts for FK validation: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"BOMs import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=BOM_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"BOMs import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"BOMs import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"BOMs import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_part_numbers: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.parts.models import Part + for p in ( + session.query(Part.part_number) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + valid_part_numbers.add(p[0]) + except Exception as e: + logger.warning(f"BOMs import: could not load parts: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + valid_count = 0 + + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + valid_count += 1 + # Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas. + # Cuando exista la tabla de destino, aquí se hará insert/update. + + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": skipped_details, + } + if valid_count > 0 and inserted_count == 0: + response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada." + + except Exception as e: + logger.error(f"BOMs import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + response = { + "status": "failed", + "error": str(e), + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": skipped_details, + } + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"BOMs import cleanup failed: {cleanup_err}") + + return response diff --git a/backend/api/v1/modules/a76/boms/imports/template_config.py b/backend/api/v1/modules/a76/boms/imports/template_config.py new file mode 100644 index 00000000..1d4efa48 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/template_config.py @@ -0,0 +1,42 @@ +""" +Configuración de plantilla CSV para BOMs (EstructuraBOMS.xlsx). +Placeholders hasta tener el XLS definitivo; ajustar canónicos y aliases según el archivo. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "boms": [ + {"canonical": "NUMPARTE_PADRE", "aliases": ["PARTE PADRE", "PART NUMBER", "PARENT PART", "NUM PARTE PADRE"]}, + {"canonical": "NUMPARTE_COMPONENTE", "aliases": ["PARTE COMPONENTE", "COMPONENT PART", "NUM PARTE COMPONENTE"]}, + {"canonical": "CANTIDAD", "aliases": ["QTY", "QUANTITY", "CANT"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]}, + {"canonical": "VERSION_BOM", "aliases": ["VERSION BOM", "BOM VERSION", "VERSIONBOM"]}, + {"canonical": "VERSION_BILL", "aliases": ["VERSION BILL", "BILL VERSION", "VERSIONBILL"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("boms") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/boms/routes.py b/backend/api/v1/modules/a76/boms/routes.py new file mode 100644 index 00000000..ddd64d15 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/routes.py @@ -0,0 +1,13 @@ +""" +Endpoints para importación CSV de BOMs. +Mismo patrón que parts y classes: upload → scan → status → commit. +""" + +from fastapi import APIRouter + +from .imports.routes import router as imports_router + +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / boms / csv_import"]) diff --git a/backend/api/v1/modules/a76/classes/__init__.py b/backend/api/v1/modules/a76/classes/__init__.py index 1a3d8bed..fa36b353 100644 --- a/backend/api/v1/modules/a76/classes/__init__.py +++ b/backend/api/v1/modules/a76/classes/__init__.py @@ -2,6 +2,20 @@ Módulo de Class """ -from .routes import router +from typing import Any __all__ = ["router"] + + +def __getattr__(name: str) -> Any: + """ + Lazy export to avoid circular imports during Celery init. + + This package is imported when models are loaded (e.g. a76.items.models), + so importing FastAPI routes at import-time can break Celery startup. + """ + if name == "router": + from .routes import router + + return router + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/backend/api/v1/modules/a76/classes/imports/__init__.py b/backend/api/v1/modules/a76/classes/imports/__init__.py new file mode 100644 index 00000000..e9eb448f --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Clases de Materiales (upload → scan → status → commit). diff --git a/backend/api/v1/modules/a76/classes/imports/routes.py b/backend/api/v1/modules/a76/classes/imports/routes.py new file mode 100644 index 00000000..5c9c869f --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para Clases de Materiales. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CLS_IMPORT_FILE_PREFIX, + CLS_IMPORT_META_PREFIX, + CLS_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Classes import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "material_classes", + } + + try: + r = _get_redis() + r.set( + f"{CLS_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CLS_IMPORT_REDIS_TTL, + ) + r.set( + f"{CLS_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CLS_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Classes import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Classes import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/classes/imports/schemas.py b/backend/api/v1/modules/a76/classes/imports/schemas.py new file mode 100644 index 00000000..2d362f01 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/schemas.py @@ -0,0 +1,21 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/classes/imports/tasks.py b/backend/api/v1/modules/a76/classes/imports/tasks.py new file mode 100644 index 00000000..ca754d06 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/tasks.py @@ -0,0 +1,528 @@ +""" +Tareas Celery para importación CSV de Clases de Materiales. +Flujo: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +CLS_IMPORT_FILE_PREFIX = "cls_import_file:" +CLS_IMPORT_META_PREFIX = "cls_import_meta:" +CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" +CLS_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Classes import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cls_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Classes import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CLS_IMPORT_FILE_PREFIX}{job_id}", + f"{CLS_IMPORT_META_PREFIX}{job_id}", + f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Classes import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_class( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + class_code = (row.get("CLASE") or "").strip() + if not class_code: + return {"line": line_num, "col": "CLASE", "msg": "Requerido"} + if len(class_code) > 8: + return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} + + desc_es = (row.get("DESCRIPCIONE") or "").strip() + if desc_es and len(desc_es) > 500: + return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} + desc_en = (row.get("DESCRIPCIONI") or "").strip() + if desc_en and len(desc_en) > 500: + return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} + + material_key = (row.get("CLAVEMAT") or "").strip() + if material_key: + if len(material_key) > 10: + return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"} + if valid_material_keys is not None and material_key not in valid_material_keys: + return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"} + + uom = (row.get("UNIMED") or "").strip() + if uom: + if len(uom) > 5: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} + if valid_uom_codes is not None and uom not in valid_uom_codes: + return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"} + + fraction = (row.get("FRACCION") or "").strip() + if fraction and len(fraction) > 20: + return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"} + us_fraction = (row.get("FRACCIONAME") or "").strip() + if us_fraction and len(us_fraction) > 16: + return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} + sub_key = (row.get("CLAVESUB") or "").strip() + if sub_key and len(sub_key) > 5: + return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"} + iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip() + if iva_exempt and len(iva_exempt) > 4: + return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"} + + rev_fisica = row.get("REVFISICA") + if rev_fisica is not None and rev_fisica != "": + try: + v = int(rev_fisica) + if v < -32768 or v > 32767: + return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"} + except (ValueError, TypeError): + return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Classes import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Classes import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + logger.warning(f"Classes import: could not load FK sets: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_class( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Classes import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CLS_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Classes import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Classes import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Classes import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.classes.models import Class + + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + logger.warning(f"Classes import: could not load FK sets: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_code: Dict[str, Class] = {} + for c in ( + session.query(Class) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + existing_by_code[c.class_code] = c + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_class( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + class_code = _str_or_none(row_norm.get("CLASE"), 8) + if not class_code: + skipped_invalid += 1 + continue + + existing = existing_by_code.get(class_code) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) + if material_key and material_key not in valid_material_keys: + material_key = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + fraction = _str_or_none(row_norm.get("FRACCION"), 20) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) + physical_review = _int_or_none(row_norm.get("REVFISICA")) + iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) + + if existing: + existing.description_es = desc_es + existing.description_en = desc_en + existing.material_key = material_key + existing.unit_of_measure = unit_of_measure + existing.fraction = fraction + existing.us_fraction = us_fraction + existing.sub_key = sub_key + existing.physical_review = physical_review + existing.iva_exempt_fraction = iva_exempt_fraction + session.add(existing) + inserted_count += 1 + else: + new_class = Class( + tenant_id=tenant_id, + company_id=company_id, + class_code=class_code, + description_es=desc_es, + description_en=desc_en, + material_key=material_key, + unit_of_measure=unit_of_measure, + fraction=fraction, + us_fraction=us_fraction, + sub_key=sub_key, + physical_review=physical_review, + iva_exempt_fraction=iva_exempt_fraction, + ) + session.add(new_class) + existing_by_code[class_code] = new_class + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Classes import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Classes import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Classes import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/classes/imports/template_config.py b/backend/api/v1/modules/a76/classes/imports/template_config.py new file mode 100644 index 00000000..4a389bd8 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/template_config.py @@ -0,0 +1,45 @@ +""" +Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "material_classes": [ + {"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]}, + {"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, + {"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]}, + {"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]}, + {"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("material_classes") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/classes/routes.py b/backend/api/v1/modules/a76/classes/routes.py index 329294b0..2166610f 100644 --- a/backend/api/v1/modules/a76/classes/routes.py +++ b/backend/api/v1/modules/a76/classes/routes.py @@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse from .service import ClassService +from .imports.routes import router as imports_router # Create a new router for custom endpoints router = APIRouter() +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"]) + # Add consolidated catalog endpoints FIRST (before generic CRUD routes) # This ensures they have priority over the generic /{id} route @router.get( diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py b/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py new file mode 100644 index 00000000..94b7cb55 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py @@ -0,0 +1,2 @@ +# CSV import flow for Clientes y Proveedores (Client Providers). +# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py b/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py new file mode 100644 index 00000000..68029417 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py @@ -0,0 +1,161 @@ +""" +Rutas de importación CSV para Clientes y Proveedores. +Mismo flujo que a76.imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CP_IMPORT_FILE_PREFIX, + CP_IMPORT_META_PREFIX, + CP_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"CP import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "client_providers", + } + + try: + r = _get_redis() + r.set( + f"{CP_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CP_IMPORT_REDIS_TTL, + ) + r.set( + f"{CP_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"CP import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"CP import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("CP import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py b/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py new file mode 100644 index 00000000..34fea87c --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py @@ -0,0 +1,500 @@ +""" +Tareas Celery para importación CSV de Clientes y Proveedores. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template +from api.v1.modules.a76.clients_and_providers.models import ( + ClientProvider, + ClientProviderAddress, + ClientOrProviderEnum, +) + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports) +CP_IMPORT_FILE_PREFIX = "cp_import_file:" +CP_IMPORT_META_PREFIX = "cp_import_meta:" +CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:" +CP_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"CP import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cp_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"CP import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CP_IMPORT_FILE_PREFIX}{job_id}", + f"{CP_IMPORT_META_PREFIX}{job_id}", + f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"CP import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: + """Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido.""" + if not val or not str(val).strip(): + return None + v = str(val).strip().lower() + if v in ("client", "cliente", "c"): + return ClientOrProviderEnum.CLIENT + if v in ("provider", "proveedor", "p"): + return ClientOrProviderEnum.PROVIDER + if v in ("both", "ambos", "b", "cliente y proveedor"): + return ClientOrProviderEnum.BOTH + return None + + +def _parse_active(val: Optional[str]) -> bool: + """Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True.""" + if not val or not str(val).strip(): + return True + v = str(val).strip().lower() + if v in ("1", "true", "si", "sí", "yes", "s", "x"): + return True + if v in ("0", "false", "no", "n"): + return False + return True + + +def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Cliente/Proveedor. Retorna error dict o None.""" + rfc = (row.get("RFC") or "").strip() + if not rfc: + return {"line": line_num, "col": "RFC", "msg": "Requerido"} + if len(rfc) > 30: + return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"} + + tipo_raw = (row.get("TIPO") or "").strip() + if tipo_raw and _parse_client_or_provider(tipo_raw) is None: + return { + "line": line_num, + "col": "TIPO", + "msg": "Valor no válido. Use Cliente, Proveedor o Ambos.", + } + + name = (row.get("NOMBRE") or "").strip() + if len(name) > 256: + return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"} + + short_name = (row.get("SHORT_NAME") or "").strip() + if short_name and len(short_name) > 10: + return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"} + + curp = (row.get("CURP") or "").strip() + if curp and len(curp) > 19: + return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"CP import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"CP import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_client_provider(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"CP import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"CP import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider. + Upsert por (tenant_id, company_id, rfc). + """ + logger.info(f"CP import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"CP import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + # Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key + existing_by_rfc: Dict[str, ClientProvider] = {} + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + key = (cp.rfc or "").strip() + existing_by_rfc[key] = cp + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_client_provider(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + rfc = _str_or_none(row_norm.get("RFC"), 30) + if not rfc: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "RFC requerido"}) + continue + + client_or_provider = _parse_client_or_provider(row_norm.get("TIPO")) + if client_or_provider is None: + client_or_provider = ClientOrProviderEnum.BOTH + + is_active = _parse_active(row_norm.get("ACTIVO")) + + existing = existing_by_rfc.get(rfc) + if existing: + existing.name = _str_or_none(row_norm.get("NOMBRE"), 256) + existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10) + existing.curp = _str_or_none(row_norm.get("CURP"), 19) + existing.client_or_provider = client_or_provider + existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80) + existing.position = _str_or_none(row_norm.get("POSICION"), 30) + existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19) + existing.is_active = is_active + session.add(existing) + inserted_count += 1 + else: + new_cp = ClientProvider( + tenant_id=tenant_id, + company_id=company_id, + rfc=rfc, + name=_str_or_none(row_norm.get("NOMBRE"), 256), + short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10), + curp=_str_or_none(row_norm.get("CURP"), 19), + client_or_provider=client_or_provider, + responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80), + position=_str_or_none(row_norm.get("POSICION"), 30), + incoterm=_str_or_none(row_norm.get("INCOTERM"), 19), + is_active=is_active, + ) + session.add(new_cp) + session.flush() + existing_by_rfc[rfc] = new_cp + inserted_count += 1 + + # Opcional: crear dirección si hay email/teléfono/dirección + email = _str_or_none(row_norm.get("EMAIL"), 100) + phone = _str_or_none(row_norm.get("TELEFONO"), 30) + address_str = _str_or_none(row_norm.get("DIRECCION"), 100) + if email or phone or address_str: + addr = ClientProviderAddress( + client_id=new_cp.id, + tenant_id=tenant_id, + company_id=company_id, + streets=address_str, + postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), + city=_str_or_none(row_norm.get("CIUDAD"), 30), + state=_str_or_none(row_norm.get("ESTADO"), 30), + country=_str_or_none(row_norm.get("PAIS"), 3), + phone=phone, + email=email, + contact=_str_or_none(row_norm.get("CONTACTO"), 50), + ) + session.add(addr) + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"CP import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"CP import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"CP import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py new file mode 100644 index 00000000..f0968d9d --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py @@ -0,0 +1,56 @@ +""" +Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "client_providers": [ + {"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]}, + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]}, + {"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]}, + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]}, + {"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]}, + {"canonical": "CURP", "aliases": []}, + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]}, + {"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + {"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]}, + {"canonical": "ESTADO", "aliases": []}, + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + {"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]}, + {"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "INCOTERM", "aliases": []}, + {"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla client_providers.""" + cols = TEMPLATE_COLUMNS.get("client_providers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/clients_and_providers/routes.py b/backend/api/v1/modules/a76/clients_and_providers/routes.py index 6b31d570..c9c67efe 100644 --- a/backend/api/v1/modules/a76/clients_and_providers/routes.py +++ b/backend/api/v1/modules/a76/clients_and_providers/routes.py @@ -20,10 +20,14 @@ from .dto import ( ) from .service import ClientProviderService from .models import ClientProvider +from .imports.routes import router as imports_router # Create main router to add custom endpoints router = APIRouter(prefix="/clients-providers") +# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit) +router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"]) + @router.get("/", response_model=ClientProviderPaginatedResponseDTO) async def get_clients_and_providers( diff --git a/backend/api/v1/modules/a76/csv_templates/__init__.py b/backend/api/v1/modules/a76/csv_templates/__init__.py new file mode 100644 index 00000000..0bb939f9 --- /dev/null +++ b/backend/api/v1/modules/a76/csv_templates/__init__.py @@ -0,0 +1 @@ +# CSV templates: generate CSV from code (no physical XLS/XLSX files) diff --git a/backend/api/v1/modules/a76/csv_templates/registry.py b/backend/api/v1/modules/a76/csv_templates/registry.py new file mode 100644 index 00000000..e6486244 --- /dev/null +++ b/backend/api/v1/modules/a76/csv_templates/registry.py @@ -0,0 +1,147 @@ +""" +Registro central de plantillas CSV: template_id -> lista de cabeceras canónicas. +Construido a partir de los TEMPLATE_COLUMNS de cada módulo de imports. +""" +import csv +import io +from typing import Dict, List, Optional + +# Importar configs de cada módulo +from api.v1.modules.a76.imports.template_config import ( + TEMPLATE_COLUMNS as IMPORTS_TEMPLATE_COLUMNS, + _resolve_template_columns as resolve_imports_template, +) +from api.v1.modules.a76.parts.imports.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS +from api.v1.modules.a76.boms.imports.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS +from api.v1.modules.a76.classes.imports.template_config import TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS +from api.v1.modules.a76.customs_brokers.imports.template_config import ( + TEMPLATE_COLUMNS as CUSTOMS_BROKERS_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.clients_and_providers.imports.template_config import ( + TEMPLATE_COLUMNS as CLIENTS_PROVIDERS_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.general_catalogs.exchange_rate.imports.template_config import ( + TEMPLATE_COLUMNS as EXCHANGE_RATE_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.template_config import ( + TEMPLATE_COLUMNS as US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.pedmientos.imports.template_config import ( + TEMPLATE_COLUMNS as PEDIMENTOS_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.transportation.vehicles.imports.template_config import ( + TEMPLATE_COLUMNS as VEHICLES_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.transportation.drivers.imports.template_config import ( + TEMPLATE_COLUMNS as DRIVERS_TEMPLATE_COLUMNS, +) +from api.v1.modules.a76.transportation.trailers.imports.template_config import ( + TEMPLATE_COLUMNS as TRAILERS_TEMPLATE_COLUMNS, +) + + +def _canonicals_from_columns(cols: Optional[List[Dict]]) -> List[str]: + """Extrae la lista de nombres canónicos en orden a partir de una lista de columnas.""" + if not cols: + return [] + return [item["canonical"] for item in cols] + + +def _build_registry() -> Dict[str, List[str]]: + registry: Dict[str, List[str]] = {} + + # a76/imports (facturas): imp_temp_header, imp_temp_details, imp_def_*, exp_def_* + for tid in ("imp_temp_header", "imp_temp_details", "imp_def_header", "imp_def_details", "exp_def_header", "exp_def_details"): + cols = resolve_imports_template(tid) + registry[tid] = _canonicals_from_columns(cols) + + # part_numbers (parts); "items" usa la misma plantilla + part_cols = PARTS_TEMPLATE_COLUMNS.get("part_numbers") + registry["part_numbers"] = _canonicals_from_columns(part_cols) + registry["items"] = _canonicals_from_columns(part_cols) + + # boms + registry["boms"] = _canonicals_from_columns(BOMS_TEMPLATE_COLUMNS.get("boms")) + + # material_classes + registry["material_classes"] = _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes")) + + # customs_brokers + registry["customs_brokers"] = _canonicals_from_columns(CUSTOMS_BROKERS_TEMPLATE_COLUMNS.get("customs_brokers")) + + # clients_providers + registry["clients_providers"] = _canonicals_from_columns(CLIENTS_PROVIDERS_TEMPLATE_COLUMNS.get("client_providers")) + + # exchange_rates + registry["exchange_rates"] = _canonicals_from_columns(EXCHANGE_RATE_TEMPLATE_COLUMNS.get("exchange_rates")) + + # american_fractions + registry["american_fractions"] = _canonicals_from_columns( + US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS.get("us_tariff_fractions") + ) + + # pedimentos + registry["pedimentos"] = _canonicals_from_columns(PEDIMENTOS_TEMPLATE_COLUMNS.get("pedimentos")) + + # transports (vehicles) + registry["transports"] = _canonicals_from_columns(VEHICLES_TEMPLATE_COLUMNS.get("vehicles")) + + # drivers + registry["drivers"] = _canonicals_from_columns(DRIVERS_TEMPLATE_COLUMNS.get("drivers")) + + # trailers + registry["trailers"] = _canonicals_from_columns(TRAILERS_TEMPLATE_COLUMNS.get("trailers")) + + return registry + + +_TEMPLATE_HEADERS: Dict[str, List[str]] = _build_registry() + +# Nombre de archivo sugerido para descarga (sin path) +TEMPLATE_FILENAMES: Dict[str, str] = { + "customs_brokers": "EstructuraCatAgenteAduanal.csv", + "clients_providers": "EstructuraCatClienteProv.csv", + "exchange_rates": "EstructuraCatTiposCambio.csv", + "american_fractions": "EstructuraCatFraccAme.csv", + "material_classes": "EstructuraCatClasesAF.csv", + "part_numbers": "EstructuraCatPartesAF.csv", + "items": "EstructuraCatPartesAF.csv", + "boms": "EstructuraBOMS.csv", + "pedimentos": "EstructuraCatPedimentos.csv", + "transports": "EstructuraCatTransportes.csv", + "drivers": "EstructuraCatConductor.csv", + "trailers": "EstructuraCatTrailers.csv", + "imp_temp_header": "EstructuraEncFacImpoTemp.csv", + "imp_temp_details": "EstructuraParFacImpoTempAF.csv", + "imp_def_header": "EstructuraEncFacImpoDef.csv", + "imp_def_details": "EstructuraParFacImpoDefAF.csv", + "exp_def_header": "EstructuraEncFacExpoCamReg.csv", + "exp_def_details": "EstructuraParExpoCamReg.csv", +} + + +def get_template_headers(template_id: str) -> Optional[List[str]]: + """Devuelve la lista de cabeceras canónicas para el template_id, o None si no existe.""" + return _TEMPLATE_HEADERS.get(template_id) + + +def get_template_filename(template_id: str) -> str: + """Nombre de archivo sugerido para la descarga.""" + return TEMPLATE_FILENAMES.get(template_id, f"plantilla_{template_id}.csv") + + +def generate_csv_content(template_id: str, include_bom: bool = True) -> Optional[bytes]: + """ + Genera el contenido CSV (solo fila de cabeceras) para el template_id. + UTF-8, opcionalmente con BOM para Excel. + """ + headers = get_template_headers(template_id) + if not headers: + return None + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") + writer.writerow(headers) + content = buf.getvalue().encode("utf-8") + if include_bom: + content = b"\xef\xbb\xbf" + content + return content diff --git a/backend/api/v1/modules/a76/csv_templates/routes.py b/backend/api/v1/modules/a76/csv_templates/routes.py new file mode 100644 index 00000000..26f1e264 --- /dev/null +++ b/backend/api/v1/modules/a76/csv_templates/routes.py @@ -0,0 +1,35 @@ +""" +Rutas para descargar plantillas CSV generadas desde código (sin archivos XLS/XLSX). +""" +from typing import Any, Dict + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import Response + +from core.security import get_current_user + +from .registry import generate_csv_content, get_template_filename + +router = APIRouter() + + +@router.get("/{template_id}", response_class=Response) +async def download_csv_template( + template_id: str, + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Devuelve un CSV con solo la fila de cabeceras para la plantilla indicada. + Las cabeceras son los nombres canónicos definidos en cada template_config. + """ + content = generate_csv_content(template_id, include_bom=True) + if content is None: + raise HTTPException(status_code=404, detail=f"Plantilla desconocida: {template_id}") + filename = get_template_filename(template_id) + return Response( + content=content, + media_type="text/csv; charset=utf-8", + headers={ + "Content-Disposition": f'attachment; filename="{filename}"', + }, + ) diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py new file mode 100644 index 00000000..c1c90d12 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py @@ -0,0 +1,2 @@ +# CSV import flow for Agentes Aduanales (Customs Brokers). +# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py new file mode 100644 index 00000000..99f7c609 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py @@ -0,0 +1,161 @@ +""" +Rutas de importación CSV para Agentes Aduanales. +Mismo flujo que a76.imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CB_IMPORT_FILE_PREFIX, + CB_IMPORT_META_PREFIX, + CB_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"CB import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "customs_brokers", + } + + try: + r = _get_redis() + r.set( + f"{CB_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CB_IMPORT_REDIS_TTL, + ) + r.set( + f"{CB_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CB_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"CB import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"CB import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("CB import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py b/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py new file mode 100644 index 00000000..7d9b548c --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + +class CommitRequest(BaseModel): + pass # no body needed for single model + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py new file mode 100644 index 00000000..ecaed98f --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py @@ -0,0 +1,447 @@ +""" +Tareas Celery para importación CSV de Agentes Aduanales. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con a76.imports) +CB_IMPORT_FILE_PREFIX = "cb_import_file:" +CB_IMPORT_META_PREFIX = "cb_import_meta:" +CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:" +CB_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"CB import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cb_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"CB import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CB_IMPORT_FILE_PREFIX}{job_id}", + f"{CB_IMPORT_META_PREFIX}{job_id}", + f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"CB import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Agente Aduanal. Retorna error dict o None.""" + clave = (row.get("CLAVE") or "").strip() + if not clave: + return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} + if len(clave) > 5: + return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"} + if not re.match(r"^[a-zA-Z0-9]+$", clave): + return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"} + + licencia = (row.get("LICENCIA") or "").strip() + if licencia and (len(licencia) > 4 or not licencia.isdigit()): + return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"CB import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"CB import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_customs_broker(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"CB import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + # Guardar números de línea con error en Redis para insert_valid_rows + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CB_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"CB import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker. + """ + logger.info(f"CB import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"CB import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_key: Dict[str, CustomsBroker] = {} + for b in ( + session.query(CustomsBroker) + .filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ) + .all() + ): + existing_by_key[b.broker_key] = b + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_customs_broker(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + clave = (row_norm.get("CLAVE") or "").strip()[:5] + if not clave: + skipped_invalid += 1 + continue + + existing = existing_by_key.get(clave) + if existing: + existing.type = _str_or_none(row_norm.get("TIPO"), 9) + existing.name = _str_or_none(row_norm.get("NOMBRE"), 80) + existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500) + existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15) + existing.city = _str_or_none(row_norm.get("CIUDAD"), 30) + existing.state = _str_or_none(row_norm.get("ESTADO"), 30) + existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30) + existing.fax = _str_or_none(row_norm.get("FAX"), 30) + existing.email = _str_or_none(row_norm.get("EMAIL"), 100) + existing.country = _str_or_none(row_norm.get("PAIS"), 3) + existing.tax_id = _str_or_none(row_norm.get("RFC"), 30) + existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20) + existing.position = _str_or_none(row_norm.get("POSICION"), 30) + lic = (row_norm.get("LICENCIA") or "").strip() + existing.license = lic[:4] if lic and lic.isdigit() else None + existing.company = _str_or_none(row_norm.get("EMPRESA"), 200) + existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80) + session.add(existing) + inserted_count += 1 + else: + lic = (row_norm.get("LICENCIA") or "").strip() + license_val = lic[:4] if lic and lic.isdigit() else None + new_broker = CustomsBroker( + tenant_id=tenant_id, + company_id=company_id, + broker_key=clave, + type=_str_or_none(row_norm.get("TIPO"), 9), + name=_str_or_none(row_norm.get("NOMBRE"), 80), + address=_str_or_none(row_norm.get("DIRECCION"), 1500), + postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), + city=_str_or_none(row_norm.get("CIUDAD"), 30), + state=_str_or_none(row_norm.get("ESTADO"), 30), + phone=_str_or_none(row_norm.get("TELEFONO"), 30), + fax=_str_or_none(row_norm.get("FAX"), 30), + email=_str_or_none(row_norm.get("EMAIL"), 100), + country=_str_or_none(row_norm.get("PAIS"), 3), + tax_id=_str_or_none(row_norm.get("RFC"), 30), + personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20), + position=_str_or_none(row_norm.get("POSICION"), 30), + license=license_val, + company=_str_or_none(row_norm.get("EMPRESA"), 200), + contact=_str_or_none(row_norm.get("CONTACTO"), 80), + ) + session.add(new_broker) + existing_by_key[clave] = new_broker + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"CB import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"CB import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + # Limpieza + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"CB import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py b/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py new file mode 100644 index 00000000..f4801729 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py @@ -0,0 +1,55 @@ +""" +Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "customs_brokers": [ + {"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]}, + {"canonical": "TIPO"}, + {"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]}, + {"canonical": "DIRECCION", "aliases": ["DOMICILIO"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + {"canonical": "CIUDAD"}, + {"canonical": "ESTADO"}, + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]}, + {"canonical": "FAX"}, + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]}, + {"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]}, + {"canonical": "EMPRESA", "aliases": ["COMPANY"]}, + {"canonical": "CONTACTO", "aliases": ["CONTACT"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla customs_brokers.""" + cols = TEMPLATE_COLUMNS.get("customs_brokers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/customs_brokers/routes.py b/backend/api/v1/modules/a76/customs_brokers/routes.py index 541f41d9..a94eccd1 100644 --- a/backend/api/v1/modules/a76/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/customs_brokers/routes.py @@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from . import dto, services +from .imports.routes import router as imports_router router = APIRouter() +# CSV import (mismo flujo que a76.imports: upload → scan → commit) +router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"]) + customs_broker_crud = TenantCRUDRoutes( service=services.CustomsBrokerService, create_schema=dto.CustomsBrokerCreateDTO, diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py new file mode 100644 index 00000000..4ac8a216 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py @@ -0,0 +1 @@ +# Exchange rate CSV import module diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py new file mode 100644 index 00000000..75a9aef8 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Tipos de Cambio. +Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + ER_IMPORT_FILE_PREFIX, + ER_IMPORT_META_PREFIX, + ER_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"ER import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "exchange_rates", + } + + try: + r = _get_redis() + r.set( + f"{ER_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=ER_IMPORT_REDIS_TTL, + ) + r.set( + f"{ER_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=ER_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"ER import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"ER import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("ER import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py new file mode 100644 index 00000000..c214b80b --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py @@ -0,0 +1,465 @@ +""" +Tareas Celery para importación CSV de Tipos de Cambio. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +ER_IMPORT_FILE_PREFIX = "er_import_file:" +ER_IMPORT_META_PREFIX = "er_import_meta:" +ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:" +ER_IMPORT_REDIS_TTL = 3600 # 1 hour + +DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] +TEMPLATE_ID = "exchange_rates" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"ER import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"er_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"ER import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{ER_IMPORT_FILE_PREFIX}{job_id}", + f"{ER_IMPORT_META_PREFIX}{job_id}", + f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"ER import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _parse_date(val: Optional[str]) -> Optional[datetime]: + """Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc.""" + if not val or not str(val).strip(): + return None + raw = str(val).strip() + for fmt in DATE_FORMATS: + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + continue + return None + + +def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Tipo de Cambio. Retorna error dict o None.""" + fecha_raw = (row.get("FECHA") or "").strip() + if not fecha_raw: + return {"line": line_num, "col": "FECHA", "msg": "Requerido"} + if _parse_date(fecha_raw) is None: + return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} + + valor_raw = (row.get("VALOR") or "").strip() + if not valor_raw: + return {"line": line_num, "col": "VALOR", "msg": "Requerido"} + try: + v = float(valor_raw.replace(",", ".")) + if v <= 0: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} + except ValueError: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} + + local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper() + if local_raw and len(local_raw) > 7: + return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"} + + foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper() + if foreign_raw and len(foreign_raw) > 7: + return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"ER import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"ER import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_exchange_rate(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"ER import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=ER_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"ER import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha). + """ + logger.info(f"ER import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"ER import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_date: Dict[tuple, ExchangeRate] = {} + for er in ( + session.query(ExchangeRate) + .filter( + ExchangeRate.tenant_id == tenant_id, + ExchangeRate.company_id == company_id, + ) + .all() + ): + d = er.date.date() if hasattr(er.date, "date") else er.date + existing_by_date[(tenant_id, company_id, d)] = er + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_exchange_rate(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + parsed_date = _parse_date(row_norm.get("FECHA")) + if not parsed_date: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FECHA: no parseable"}) + continue + + try: + v = float((row_norm.get("VALOR") or "").strip().replace(",", ".")) + value_decimal = Decimal(str(round(v, 6))) + except (ValueError, TypeError): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "VALOR: no numérico"}) + continue + + local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7) + if local_currency: + local_currency = local_currency.upper() + foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7) + if foreign_currency: + foreign_currency = foreign_currency.upper() + + key_date = parsed_date.date() + existing = existing_by_date.get((tenant_id, company_id, key_date)) + + if existing: + existing.value = value_decimal + existing.local_currency = local_currency or None + existing.foreign_currency = foreign_currency or None + session.add(existing) + inserted_count += 1 + else: + new_er = ExchangeRate( + tenant_id=tenant_id, + company_id=company_id, + date=parsed_date, + value=value_decimal, + local_currency=local_currency, + foreign_currency=foreign_currency, + ) + session.add(new_er) + existing_by_date[(tenant_id, company_id, key_date)] = new_er + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"ER import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"ER import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"ER import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py new file mode 100644 index 00000000..c907f9a0 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py @@ -0,0 +1,42 @@ +""" +Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "exchange_rates": [ + {"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]}, + {"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]}, + {"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]}, + {"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla exchange_rates.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py index 22ab8ec9..09086071 100644 --- a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py @@ -34,6 +34,10 @@ crud_router = route_handler.router from fastapi import APIRouter custom_router = APIRouter(prefix="/exchange-rate", tags=[]) +# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered +from .imports.routes import router as imports_router +custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"]) + @custom_router.get("/test-ping") async def test_ping(): return {"message": "pong"} diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py new file mode 100644 index 00000000..cff9f961 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py new file mode 100644 index 00000000..bf5083a3 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Fracción Americana (US Tariff Fractions). +Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + FA_IMPORT_FILE_PREFIX, + FA_IMPORT_META_PREFIX, + FA_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"FA import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "us_tariff_fractions", + } + + try: + r = _get_redis() + r.set( + f"{FA_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=FA_IMPORT_REDIS_TTL, + ) + r.set( + f"{FA_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=FA_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"FA import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"FA import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("FA import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py new file mode 100644 index 00000000..edac5df2 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py @@ -0,0 +1,474 @@ +""" +Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions). +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +FA_IMPORT_FILE_PREFIX = "fa_import_file:" +FA_IMPORT_META_PREFIX = "fa_import_meta:" +FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:" +FA_IMPORT_REDIS_TTL = 3600 # 1 hour + +TEMPLATE_ID = "us_tariff_fractions" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"FA import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"fa_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"FA import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{FA_IMPORT_FILE_PREFIX}{job_id}", + f"{FA_IMPORT_META_PREFIX}{job_id}", + f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"FA import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _normalize_code(raw: Optional[str]) -> str: + """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" + if not raw: + return "" + s = str(raw).strip().replace(".", "").replace("-", "") + return s[:16] if len(s) > 16 else s + + +def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Fracción Americana. Retorna error dict o None.""" + code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip() + if not code_raw: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} + code_norm = _normalize_code(code_raw) + if not code_norm: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} + if len(code_norm) > 16: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"} + + prefix_raw = (row.get("PREFIJO") or "").strip() + if prefix_raw and len(prefix_raw) > 10: + return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"} + + um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip() + if um_raw and len(um_raw) > 10: + return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"} + + tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip() + if tipo_raw and len(tipo_raw) > 10: + return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"} + + adv_pct = row.get("ADVALOREM_PCT") + if adv_pct is not None and str(adv_pct).strip(): + try: + v = float(str(adv_pct).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"} + + adv_dlls = row.get("ADVALOREM_DLLS") + if adv_dlls is not None and str(adv_dlls).strip(): + try: + v = float(str(adv_dlls).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"FA import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"FA import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_us_tariff_fraction(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"FA import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=FA_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"FA import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _parse_float(val: Any) -> Optional[float]: + if val is None or str(val).strip() == "": + return None + try: + return float(str(val).strip().replace(",", ".")) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code). + """ + logger.info(f"FA import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"FA import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_us_tariff_fraction(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA")) + if not code: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"}) + continue + + prefix = _str_or_none(row_norm.get("PREFIJO"), 10) + unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10) + description = _str_or_none(row_norm.get("DESCRIPCION")) + type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10) + ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT")) + fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS")) + fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None + + existing = ( + session.query(USTariffFraction) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + USTariffFraction.code == code, + ) + .first() + ) + + if existing: + existing.prefix = prefix + existing.type_code = type_code + existing.ad_valorem = ad_valorem + existing.fixed_cost = fixed_cost + existing.unit_of_measure = unit_of_measure + existing.description = description + session.add(existing) + inserted_count += 1 + else: + new_row = USTariffFraction( + tenant_id=tenant_id, + company_id=company_id, + code=code, + prefix=prefix, + type_code=type_code, + ad_valorem=ad_valorem, + fixed_cost=fixed_cost, + unit_of_measure=unit_of_measure, + description=description, + ) + session.add(new_row) + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"FA import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"FA import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path_clean = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path_clean): + os.remove(meta_path_clean) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"FA import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py new file mode 100644 index 00000000..aa5a5e7f --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py @@ -0,0 +1,47 @@ +""" +Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "us_tariff_fractions": [ + {"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]}, + {"canonical": "PREFIJO"}, + {"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]}, + {"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]}, + {"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla us_tariff_fractions.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template( + row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions" +) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py index 158bf765..61907ffa 100644 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py @@ -17,23 +17,28 @@ from .dto import ( ) from .service import USTariffFractionService -# Create router using TenantCRUDRoutes factory for basic CRUD operations +# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router) crud_router = TenantCRUDRoutes( service=USTariffFractionService, create_schema=USTariffFractionCreateDTO, update_schema=USTariffFractionUpdateDTO, response_schema=USTariffFractionResponseDTO, - prefix="/us-tariff-fractions", + prefix="", tags=["a76 / general catalogs / us tariff fractions"], resource_name="US Tariff Fraction", id_name="id", enable_list=False, # We implement our custom list endpoint ) -router = crud_router.router +# Master router with prefix so all routes live under /us-tariff-fractions +from .imports.routes import router as imports_router +main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"]) +main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"]) +main_router.include_router(crud_router.router) -# Custom list endpoint with search filter -@router.get( + +# Custom list endpoint with search filter (under /us-tariff-fractions/) +@main_router.get( "/", response_model=Dict[str, Any], summary="List US Tariff Fractions", @@ -65,3 +70,6 @@ async def list_us_tariff_fractions( "page_size": page_size, "pages": (total + page_size - 1) // page_size, } + + +router = main_router diff --git a/backend/api/v1/modules/a76/imports/routes.py b/backend/api/v1/modules/a76/imports/routes.py index 1ea76a74..f3c3d9ee 100644 --- a/backend/api/v1/modules/a76/imports/routes.py +++ b/backend/api/v1/modules/a76/imports/routes.py @@ -1,5 +1,6 @@ from datetime import datetime from uuid import uuid4 +import base64 import os import json import logging @@ -12,24 +13,39 @@ from core.config import settings from core.database import get_core_db from core.security import get_current_user, validate_access_to_resource -from .tasks import scan_file, insert_valid_rows +from .tasks import ( + scan_file, + insert_valid_rows, + IMPORT_FILE_KEY_PREFIX, + IMPORT_META_KEY_PREFIX, + IMPORT_REDIS_TTL, +) from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest router = APIRouter() logger = logging.getLogger(__name__) + +def _get_redis(): + """Redis client (same broker as Celery so worker can read).""" + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + @router.post("/upload/{model_target}", response_model=ImportJobResponse) async def upload_import_file( model_target: Literal["invoice_header", "invoice_details"], file: UploadFile = File(...), footer_config: Optional[str] = Form(None), # JSON string with settings - company_id: int = Query(..., description="Company ID"), # Required for context + template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas + company_id: int = Query(..., description="Company ID"), # Required for context operation_type: Optional[str] = Query("imp"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): """ Step 1: Upload CSV, save to temp, trigger scan task. + Si se envía template_id, solo se leen las columnas de esa plantilla. """ # 1. Validate Access & Get Tenant try: @@ -40,40 +56,51 @@ async def upload_import_file( if not file.filename.endswith(".csv"): raise HTTPException(status_code=400, detail="Only .csv files allowed") - + job_id = str(uuid4()) - - # Ensure directory exists (Safety check) - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") - os.makedirs(upload_dir, exist_ok=True) - - file_path = os.path.join(upload_dir, f"{job_id}.csv") - meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") - + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "footer_config": footer_config, + "operation_type": operation_type, + "template_id": template_id, + } + + # Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed) try: - # Save CSV - contents = await file.read() + redis_client = _get_redis() + redis_client.set( + f"{IMPORT_FILE_KEY_PREFIX}{job_id}", + base64.b64encode(contents), + ex=IMPORT_REDIS_TTL, + ) + redis_client.set( + f"{IMPORT_META_KEY_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Redis store error: {e}") + raise HTTPException(status_code=500, detail="Failed to queue file for processing.") + + # Optional: also write to local disk (e.g. for same-machine worker or debugging) + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"{job_id}.csv") + meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") with open(file_path, "wb") as f: f.write(contents) - - # Save Metadata (Context) - meta_data = { - "tenant_id": tenant_id, - "company_id": company_id, - "user_id": current_user.get("id"), - "footer_config": footer_config, - "operation_type": operation_type, - } with open(meta_path, "w") as f: json.dump(meta_data, f) - except Exception as e: - logger.error(f"File save error: {e}") - raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}") + logger.warning(f"Local file save failed (worker will use Redis): {e}") - # Trigger Celery Task (Async) - # Use our job_id as the Celery task_id for easier tracking - scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id) + # Trigger Celery Task (Async). Worker loads file from Redis. + scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id) return ImportJobResponse( job_id=job_id, @@ -84,24 +111,56 @@ async def upload_import_file( @router.get("/{job_id}/status") async def get_import_status(job_id: str): """ - Poll this endpoint to get % progress or final report. + Poll to get progress or final report. Always returns an object with "status". """ - # In a real app, query Redis or DB. - # For MVP, we might mock or use Celery AsyncResult if backend shares Redis. task_result = celery_app.AsyncResult(job_id) - - if task_result.state == 'PENDING': + + if task_result.state == "PENDING": return {"status": "processing", "progress": 0} - elif task_result.state == 'PROGRESS': + if task_result.state == "PROGRESS": return { - "status": "processing", - "progress": task_result.info.get('current', 0), - "total": task_result.info.get('total', 0) + "status": "processing", + "progress": (task_result.info or {}).get("current", 0), + "total": (task_result.info or {}).get("total", 0), } - elif task_result.state == 'SUCCESS': - return task_result.result # Should return the report - else: - return {"status": task_result.state, "error": str(task_result.info)} + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + # FAILURE: obtener mensaje real (traceback, result o get(propagate=False)) + logger.warning("Import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb: + logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg and len(lines) > 1: + err_msg = lines[-2] + " " + (lines[-1] or "") + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg: + result = getattr(task_result, "result", None) + info = getattr(task_result, "info", None) + if result is not None and not isinstance(result, dict): + err_msg = str(result) + elif isinstance(result, dict) and (result.get("error") or result.get("message")): + err_msg = result.get("error") or result.get("message") + if not err_msg and isinstance(info, str): + err_msg = info + elif not err_msg and isinstance(info, dict) and "error" in info: + err_msg = str(info["error"]) + if not err_msg: + err_msg = "Task failed" + return {"status": "failed", "error": err_msg} @router.post("/{job_id}/commit") diff --git a/backend/api/v1/modules/a76/imports/tasks.py b/backend/api/v1/modules/a76/imports/tasks.py index 58ef97e3..b6e8e8d9 100644 --- a/backend/api/v1/modules/a76/imports/tasks.py +++ b/backend/api/v1/modules/a76/imports/tasks.py @@ -1,4 +1,5 @@ import os +import base64 from datetime import datetime from decimal import Decimal import csv @@ -6,9 +7,12 @@ import json import logging import re import unicodedata -from celery import shared_task -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app from core.database import CoreSessionLocal + +from .template_config import row_from_template # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process # We'll need schemas for validation @@ -17,6 +21,78 @@ from core.database import CoreSessionLocal logger = logging.getLogger(__name__) +# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem) +IMPORT_FILE_KEY_PREFIX = "import_file:" +IMPORT_META_KEY_PREFIX = "import_meta:" +IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" +IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + """Redis client using same URL as Celery broker (worker and API can share data).""" + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + """Directory on the worker for temp CSV and meta (same structure as API, but local to worker).""" + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + """ + Load file content from Redis and write to worker's upload dir. + Returns local file_path if successful, None otherwise. + """ + redis_client = _get_redis() + key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}" + data = redis_client.get(key) + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Failed to decode import file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + """Load meta from Redis and write to worker's meta file. Returns True if meta was found and written.""" + redis_client = _get_redis() + key = f"{IMPORT_META_KEY_PREFIX}{job_id}" + data = redis_client.get(key) + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Failed to decode import meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + """Remove file, meta and error lines from Redis after commit (cleanup).""" + try: + r = _get_redis() + r.delete( + f"{IMPORT_FILE_KEY_PREFIX}{job_id}", + f"{IMPORT_META_KEY_PREFIX}{job_id}", + f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Failed to delete import keys from Redis: {e}") + class ForeignKeyValidator: def __init__(self, session, tenant_id, company_id): self.session = session @@ -40,26 +116,90 @@ class ForeignKeyValidator: self.cache[key] = exists return exists -@shared_task(bind=True) -def scan_file(self, job_id: str, file_path: str, model_target: str, config: str = None): + +TRANSPORT_TYPE_VALUES = { + "none", + "transport", + "box", + "licence plates", + "truck", + "vessel", + "rail_barge", + "container", + "airplane", + "gondola", + "flatbed", +} + + +def normalize_public_code(value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = str(value).strip().upper() + return text or None + + +def validate_public_code( + validator: ForeignKeyValidator, + model, + value: Optional[str], + line_num: int, + col_name: str, + field_name: str = "code", + required: bool = False, +) -> Optional[Dict[str, Any]]: + code = normalize_public_code(value) + if not code: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, code, field_name=field_name, is_public=True): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + + +def validate_tenant_fk_id( + validator: ForeignKeyValidator, + model, + value: Optional[int], + line_num: int, + col_name: str, + required: bool = False, +) -> Optional[Dict[str, Any]]: + if value is None: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, value): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, model_target: str, config: str = None): """ Pass 1: Read CSV, Validate types, Write Errors to JSONL. + File content is loaded from Redis (written by API on upload) so worker does not need shared filesystem. """ logger.info(f"Starting scan for job {job_id} target {model_target}") - - # 1. Setup Error Log + + # 1. Get file from Redis and write to worker local disk + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + # 2. Setup Error Log error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") os.makedirs(os.path.dirname(error_path), exist_ok=True) - + total_rows = 0 error_count = 0 processed_rows = 0 - - # 2. Count Total (Quick Pass) or just estimate - # For better progress, we can get file line count first + + # 3. Count Total (Quick Pass) or just estimate try: with open(file_path, 'r', encoding='utf-8-sig') as f: - total_rows = sum(1 for _ in f) - 1 # Minus header + total_rows = sum(1 for _ in f) - 1 # Minus header except Exception as e: return {"status": "failed", "error": f"Cannot read file: {e}"} @@ -70,10 +210,65 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + tenant_id = None + company_id = None + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f_meta: + meta = json.load(f_meta) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception as e: + logger.warning(f"Failed to read meta for job {job_id}: {e}") + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Missing context (tenant/company)"} + + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + + inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM") + if not inv_type_value: + inv_type_value = "TEM" try: - with open(file_path, 'r', encoding='utf-8-sig') as f_in, \ + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, + ) + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm + from api.v1.modules.a76.parts.models import Part + + models = { + "InvoiceHeader": InvoiceHeader, + "InvoiceType": InvoiceType, + "ClientProvider": ClientProvider, + "CustomsBroker": CustomsBroker, + "RegimenPedimento": RegimenPedimento, + "CodePedimentoRegimen": CodePedimentoRegimen, + "PedimentoCode": PedimentoCode, + "CurrencyType": CurrencyType, + "CustomsSection": CustomsSection, + "Incoterm": Incoterm, + "Part": Part, + } + + with CoreSessionLocal() as session, \ + open(file_path, 'r', encoding='utf-8-sig') as f_in, \ open(error_path, 'w', encoding='utf-8') as f_err: + validator = ForeignKeyValidator(session, tenant_id, company_id) + invoice_id_cache: Dict[str, Optional[int]] = {} # Detect Delimiter sample = f_in.read(2048) @@ -94,9 +289,18 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str 'errors': error_count }) - # Validation (Phase 1: Minimal) - row_norm = normalize_row(row) - errors = validate_row_phase_1(row_norm, model_target, i, date_format) + # Solo columnas de la plantilla (respetar plantilla tal cual) + row_norm = row_from_template(row, template_id, normalize_header) + errors = validate_row_strict( + row_norm, + model_target, + i, + date_format, + validator, + inv_type_value, + invoice_id_cache, + models, + ) if errors: error_count += 1 @@ -109,14 +313,45 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str logger.error(f"Scan failed: {e}") return {"status": "failed", "error": str(e)} - # 4. Result + # 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker) + error_lines_list = [] + errors_detail: List[Dict[str, Any]] = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + if len(errors_detail) < 500: + errors_detail.append( + { + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + } + ) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Failed to store error lines in Redis: {e}") + + # 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV) return { "status": "waiting_confirmation", "job_id": job_id, "total_rows": processed_rows, "error_count": error_count, "valid_rows": processed_rows - error_count, - "error_file": error_path + "errors": errors_detail, } def validate_row_phase_1( @@ -124,13 +359,61 @@ def validate_row_phase_1( target: str, line_num: int, date_format: Optional[str], -) -> Dict[str, Any]: +) -> Optional[Dict[str, Any]]: """ - Minimal validation: Unique IDs and Dates. + Validation: Unique IDs, Dates, and Numeric constraint checks. Target: 'invoice_header' or 'invoice_details' """ - errors = {} - + def check_decimal(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_decimal(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"} + return None + + def check_int(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_int(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"} + return None + + def check_date(col_name): + date_str = row.get(col_name) + if date_str and str(date_str).strip(): + if not is_valid_date(date_str, date_format): + expected = display_date_format(date_format) + return { + "line": line_num, + "col": col_name, + "msg": f"Formato de fecha inválido ({expected})", + } + return None + + def check_weight(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_weight_unit(val) is None: + return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"} + return None + + def check_currency(col_name): + val = row.get(col_name) + if val and str(val).strip(): + parsed_currency = parse_currency(val, None) + val_norm = normalize_header(val) + # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"} + return None + + def check_transport_type(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES: + return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"} + return None + # A. Invoice Header if target == 'invoice_header': # 1. Unique ID @@ -138,33 +421,180 @@ def validate_row_phase_1( return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} # 2. Date Format - date_str = row.get('FECHA FACTURA') - if date_str: - if not is_valid_date(date_str, date_format): - expected = display_date_format(date_format) - return { - "line": line_num, - "col": "FECHA FACTURA", - "msg": f"Formato inválido ({expected})", - } - else: + date_str = row.get('FECHA FACTURA') or row.get('FECHA') + if not date_str or not str(date_str).strip(): return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"} + + err = check_date('FECHA FACTURA') or check_date('FECHA') + if err: return err + + err = check_date('FECHA EMISION') + if err: return err + + # 3. Numeric Fields + for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']: + err = check_decimal(col) + if err: return err + + # 4. Integer FKs + for col in ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA']: + err = check_int(col) + if err: return err + + # 5. Enums + for col in ['TIPO PESO']: + err = check_weight(col) + if err: return err + + for col in ['TIPO MONEDA']: + err = check_currency(col) + if err: return err + + for col in ['TIPO TRANSPORTE']: + err = check_transport_type(col) + if err: return err # B. Invoice Details (Parts) elif target == 'invoice_details': # 1. Line Number - if not row.get('LINEA'): + if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'): return {"line": line_num, "col": "LINEA", "msg": "Requerido"} # 2. Parent Link (Invoice Number) if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')): return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} - - # 2. Parent Link (Simplified for now, we assume parent exists or is in same batch) - # In a real scenario, we'd check if the invoice exists. - pass + + # 3. Numeric Fields + for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']: + err = check_decimal(col) + if err: return err + + for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']: + err = check_int(col) + if err: return err - return errors if errors else None + return None + + +def validate_row_strict( + row: Dict[str, Any], + target: str, + line_num: int, + date_format: Optional[str], + validator: ForeignKeyValidator, + inv_type_value: str, + invoice_id_cache: Dict[str, Optional[int]], + models: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + err = validate_row_phase_1(row, target, line_num, date_format) + if err: + return err + + InvoiceHeader = models["InvoiceHeader"] + InvoiceType = models["InvoiceType"] + ClientProvider = models["ClientProvider"] + CustomsBroker = models["CustomsBroker"] + RegimenPedimento = models["RegimenPedimento"] + CurrencyType = models["CurrencyType"] + CustomsSection = models["CustomsSection"] + Incoterm = models["Incoterm"] + Part = models["Part"] + + if target == "invoice_header": + if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): + return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"} + + provider_id = parse_int(row.get("CLAVE PROVEEDOR")) + err = validate_tenant_fk_id(validator, ClientProvider, provider_id, line_num, "CLAVE PROVEEDOR", required=True) + if err: + return err + + sold_to_id = parse_int(row.get("CLAVE VENDIDO A")) + err = validate_tenant_fk_id(validator, ClientProvider, sold_to_id, line_num, "CLAVE VENDIDO A", required=True) + if err: + return err + + shipped_to_id = parse_int(row.get("CLAVE ENVIADO A")) + err = validate_tenant_fk_id(validator, ClientProvider, shipped_to_id, line_num, "CLAVE ENVIADO A", required=True) + if err: + return err + + broker_id = parse_int(row.get("AGENTE ADUANAL")) + err = validate_tenant_fk_id(validator, CustomsBroker, broker_id, line_num, "AGENTE ADUANAL") + if err: + return err + + err = validate_public_code( + validator, + RegimenPedimento, + row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"), + line_num, + "CLAVEDOCUMENTO", + ) + if err: + return err + + err = validate_public_code( + validator, + CustomsSection, + row.get("ADUANA DE CRUCE"), + line_num, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + return err + + err = validate_public_code( + validator, + CurrencyType, + row.get("CLAVE MONEDA"), + line_num, + "CLAVE MONEDA", + ) + if err: + return err + + err = validate_public_code( + validator, + Incoterm, + row.get("CLAVE INCOTERM"), + line_num, + "CLAVE INCOTERM", + ) + if err: + return err + + elif target == "invoice_details": + invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() + if not invoice_number: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} + + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] + else: + invoice_id = ( + validator.session.query(InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == validator.tenant_id, + InvoiceHeader.company_id == validator.company_id, + InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, + ) + .scalar() + ) + invoice_id_cache[cache_key] = invoice_id + if not invoice_id: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"} + + part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip() + if not part_num: + return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} + if not validator.check_exists(Part, part_num, field_name="part_number"): + return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"} + + return None def parse_footer_config(config: Optional[str]) -> Dict[str, Any]: if not config: @@ -247,6 +677,16 @@ def parse_decimal(value: Any) -> Optional[Decimal]: return None +def decimal_or_zero(value: Any) -> Decimal: + """Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default).""" + return parse_decimal(value) or Decimal("0") + + +def int_or_zero(value: Any) -> int: + """Return parsed int or 0 for CSV nulls/empty (vanilla default).""" + return parse_int(value) if parse_int(value) is not None else 0 + + def parse_currency(value: Optional[str], currency_type: Optional[str]): from api.v1.modules.a76.invoices.models import Currency if value: @@ -319,13 +759,24 @@ def resolve_public_code( cache[normalized] = normalized if exists is not None else None return cache[normalized] -@shared_task(bind=True) +@celery_app.task(bind=True) def insert_valid_rows(self, job_id: str, model_target: str): """ Pass 2: Re-read CSV, Skip Errors, Bulk Insert. + File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem. """ logger.info(f"Starting Commit for {job_id} target {model_target}") - + + # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + upload_dir = _worker_upload_dir() + file_path = os.path.join(upload_dir, f"{job_id}.csv") + if not os.path.exists(file_path): + return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + try: from api.v1.modules.a76.invoices.models import ( InvoiceHeader, @@ -334,16 +785,18 @@ def insert_valid_rows(self, job_id: str, model_target: str): InvoiceLogistics, InvoiceSalesDetails, OperationType, + TransportType, WeightUnit, ) from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.line_financials.models import LineFinancial @@ -352,30 +805,37 @@ def insert_valid_rows(self, job_id: str, model_target: str): from api.v1.modules.a76.items.line_descriptions.models import LineDescription from api.v1.modules.a76.parts.models import Part - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") - file_path = os.path.join(upload_dir, f"{job_id}.csv") error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") - - # 1. Load Error Line Numbers + + # 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file) error_lines = set() - if os.path.exists(error_path): + try: + r = _get_redis() + raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): with open(error_path, 'r', encoding='utf-8') as f: for line in f: try: err = json.loads(line) error_lines.add(err['line']) - except: pass + except Exception: + pass # Load Metadata (Context) - meta_path = file_path.replace("temp", "temp").replace(".csv", ".meta.json") + meta_path = file_path.replace(".csv", ".meta.json") tenant_id = None company_id = None footer_config = {} + meta = {} if os.path.exists(meta_path): try: with open(meta_path, 'r') as f: - meta = json.load(f) + meta = json.load(f) or {} tenant_id = meta.get('tenant_id') company_id = meta.get('company_id') operation_type_raw = meta.get('operation_type', 'imp') @@ -407,7 +867,7 @@ def insert_valid_rows(self, job_id: str, model_target: str): # Default types from config or fallback op_type_value = OperationType(meta.get('operation_type', 'imp').lower()) - inv_type_value = footer_config.get('invoice_type', 'TEM') + inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM' logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") @@ -436,13 +896,17 @@ def insert_valid_rows(self, job_id: str, model_target: str): reader = csv.DictReader(f, dialect=dialect) + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + for i, row in enumerate(reader, start=1): if i in error_lines: continue - row_norm = normalize_row(row) + row_norm = row_from_template(row, template_id, normalize_header) - # Mapping Logic + # Mapping Logic (solo campos que acepta el modelo de facturas) if model_target == 'invoice_header': invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format) @@ -461,25 +925,148 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue - - # 2. Client/Provider (Tenant) + provider_id = parse_int(row_norm.get('CLAVE PROVEEDOR')) - if provider_id and not validator.check_exists(ClientProvider, provider_id): - skipped_missing_fk += 1 - reason = f"Proveedor ID '{provider_id}' no existe" + err = validate_tenant_fk_id( + validator, + ClientProvider, + provider_id, + i, + "CLAVE PROVEEDOR", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue - # 3. Customs Broker (Tenant) - broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) - if broker_id and not validator.check_exists(CustomsBroker, broker_id): - skipped_missing_fk += 1 - reason = f"Agente Aduanal ID '{broker_id}' no existe" + sold_to_id = parse_int(row_norm.get('CLAVE VENDIDO A')) + err = validate_tenant_fk_id( + validator, + ClientProvider, + sold_to_id, + i, + "CLAVE VENDIDO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue + shipped_to_id = parse_int(row_norm.get('CLAVE ENVIADO A')) + err = validate_tenant_fk_id( + validator, + ClientProvider, + shipped_to_id, + i, + "CLAVE ENVIADO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) + err = validate_tenant_fk_id( + validator, + CustomsBroker, + broker_id, + i, + "AGENTE ADUANAL", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + RegimenPedimento, + row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'), + i, + "CLAVEDOCUMENTO", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CustomsSection, + row_norm.get('ADUANA DE CRUCE'), + i, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CurrencyType, + row_norm.get('CLAVE MONEDA'), + i, + "CLAVE MONEDA", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + Incoterm, + row_norm.get('CLAVE INCOTERM'), + i, + "CLAVE INCOTERM", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + transport_type_val = row_norm.get('TIPO TRANSPORTE') + if transport_type_val and str(transport_type_val).strip().lower() not in TRANSPORT_TYPE_VALUES: + skipped_invalid += 1 + reason = "TIPO TRANSPORTE: Tipo de transporte invalido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + currency_val = row_norm.get('TIPO MONEDA') + if currency_val and str(currency_val).strip(): + parsed_currency = parse_currency(currency_val, None) + val_norm = normalize_header(currency_val) + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + skipped_invalid += 1 + reason = "TIPO MONEDA: Moneda invalida" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + # 2. Client/Provider and broker checks are handled above + # --- 4. Check for Existing Invoice (Upsert Logic) --- existing_header = None if invoice_number: @@ -608,12 +1195,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): financials = InvoiceFinancials( currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type), currency_type=financials_currency_type, - exchange_rate=parse_decimal(row_norm.get('TIPO DE CAMBIO')), - freight=parse_decimal(row_norm.get('FLETES')), - insurance_value=parse_decimal(row_norm.get('VALOR SEGUROS')), - insurance=parse_decimal(row_norm.get('SEGUROS')), - packaging=parse_decimal(row_norm.get('EMBALAJES')), - other_increments=parse_decimal(row_norm.get('OTROS INCREMENTABLES')), + exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')), + freight=decimal_or_zero(row_norm.get('FLETES')), + insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')), + insurance=decimal_or_zero(row_norm.get('SEGUROS')), + packaging=decimal_or_zero(row_norm.get('EMBALAJES')), + other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')), tenant_id=tenant_id, company_id=company_id, ) @@ -621,10 +1208,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) logistics = None if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): + raw_transport = (row_norm.get('TIPO TRANSPORTE') or "none") + transport_str = str(raw_transport).strip().lower() or "none" + try: + transport_type = TransportType(transport_str) + except ValueError: + transport_type = TransportType.NONE logistics = InvoiceLogistics( carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), - transport_type=str(row_norm.get('TIPO TRANSPORTE') or "none").lower(), + transport_type=transport_type, transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), weight_type=weight_type or WeightUnit.KGS, seal_number=(row_norm.get('PRECINTO') or None), @@ -647,8 +1240,9 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_invalid += 1 continue - if invoice_number in invoice_id_cache: - invoice_id = invoice_id_cache[invoice_number] + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] else: invoice_id = ( session.query(InvoiceHeader.id) @@ -656,10 +1250,11 @@ def insert_valid_rows(self, job_id: str, model_target: str): InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, ) .scalar() ) - invoice_id_cache[invoice_number] = invoice_id + invoice_id_cache[cache_key] = invoice_id if not invoice_id: logger.warning( @@ -670,6 +1265,20 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_missing_invoice += 1 continue + part_num = (row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or '').strip() + if not part_num: + skipped_invalid += 1 + reason = "NUMPARTE: Requerido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + if not validator.check_exists(Part, part_num, field_name="part_number"): + skipped_missing_fk += 1 + reason = f"NUMPARTE '{part_num}' no existe" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + # --- Prevent Duplicates: Clear existing items for this invoice (Once per job) --- if invoice_id not in cleared_invoices: logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates") @@ -688,19 +1297,17 @@ def insert_valid_rows(self, job_id: str, model_target: str): # --- NEW LOGIC: Expanded Anexo 76 Structure --- # A. Find/Cache Part - part_num = (row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or '').strip() part_id = None - if part_num: - part_id = part_cache.get(part_num) - if part_id is None: - p = session.query(Part.id).filter( - Part.part_number == part_num, - Part.tenant_id == tenant_id, - Part.company_id == company_id - ).first() - if p: - part_id = p.id - part_cache[part_num] = part_id + part_id = part_cache.get(part_num) + if part_id is None: + p = session.query(Part.id).filter( + Part.part_number == part_num, + Part.tenant_id == tenant_id, + Part.company_id == company_id + ).first() + if p: + part_id = p.id + part_cache[part_num] = part_id line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA')) line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) @@ -727,23 +1334,23 @@ def insert_valid_rows(self, job_id: str, model_target: str): session.add(line) session.flush() # Need line.id - # 3. Financial Data + # 3. Financial Data (vanilla: nulls from CSV -> 0) price = parse_decimal(row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO')) val_com = parse_decimal(row_norm.get('VALOR COMERCIAL') or row_norm.get('VALORCOMERCIAL')) qty = parse_decimal(row_norm.get('CANTIDAD')) - + commercial_total = val_com or (price * qty if price and qty else None) + session.add(LineFinancial( item_line_id=line.id, - unit_price=price, - commercial_value=val_com or (price * qty if price and qty else None), + unit_cost_capture=decimal_or_zero(price), + total_commercial_value=decimal_or_zero(commercial_total), )) - # 4. Quantities - if qty: - session.add(LineQuantity( - item_line_id=line.id, - quantity=qty, - )) + # 4. Quantities (vanilla: nulls -> 0 so we always have a quantity row) + session.add(LineQuantity( + item_line_id=line.id, + quantity=decimal_or_zero(qty), + )) # 5. Customs/Fraction origin = row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') @@ -763,12 +1370,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): description_spanish=desc, )) - # 7. Legacy Sales Details (For specific audit/UI fields) + # 7. Legacy Sales Details (For specific audit/UI fields; vanilla: nulls -> 0) detail = InvoiceSalesDetails( invoice_id=invoice_id, line_number=line_num, sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), - line_bundles=parse_int(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), + line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), tenant_id=tenant_id, company_id=company_id, ) @@ -850,15 +1457,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): logger.error(traceback.format_exc()) return {"status": "failed", "error": str(e)} - # 5. Cleanup + # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely try: - if os.path.exists(file_path): + if file_path and os.path.exists(file_path): os.remove(file_path) if os.path.exists(error_path): os.remove(error_path) - except: - logger.warning("Failed to cleanup temp files") - + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) + # Ensure response is defined (fallback in case of unexpected errors) if response is None: logger.error(f"Unexpected error: response not set for job {job_id}") diff --git a/backend/api/v1/modules/a76/imports/template_config.py b/backend/api/v1/modules/a76/imports/template_config.py new file mode 100644 index 00000000..1a3fc274 --- /dev/null +++ b/backend/api/v1/modules/a76/imports/template_config.py @@ -0,0 +1,118 @@ +""" +Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean. +La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora. +Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models). +""" + +from typing import Dict, List, Any, Optional + +# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV). +# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que +# espera la lógica de validación e insert (tasks.py). +# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA). + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + # --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) --- + "imp_temp_header": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, + {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, + {"canonical": "FECHA EMISION"}, + {"canonical": "CLAVE PROVEEDOR"}, + {"canonical": "CLAVE VENDIDO A"}, + {"canonical": "CLAVE ENVIADO A"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, + {"canonical": "ADUANA DE CRUCE"}, + {"canonical": "CLAVE MONEDA"}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "TIPO PESO"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "REMESA"}, + {"canonical": "AGENTE ADUANAL"}, + {"canonical": "FLETES"}, + {"canonical": "VALOR SEGUROS"}, + {"canonical": "SEGUROS"}, + {"canonical": "EMBALAJES"}, + {"canonical": "OTROS INCREMENTABLES"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "OBSERVACIONES E"}, + {"canonical": "OBSERVACIONES I"}, + {"canonical": "E DOCUMENT"}, + {"canonical": "NUM OPERACION"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "PRECINTO"}, + ], + # --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura --- + "imp_def_header": None, # se resuelve igual que imp_temp_header + # --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura --- + "exp_def_header": None, + # --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) --- + "imp_temp_details": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]}, + {"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]}, + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]}, + {"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]}, + {"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]}, + {"canonical": "CANTIDAD"}, + {"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]}, + {"canonical": "FRACCION"}, + {"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]}, + ], + # --- Partidas: Impo Def y Expo - misma estructura --- + "imp_def_details": None, + "exp_def_details": None, +} + + +def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]: + cols = TEMPLATE_COLUMNS.get(template_id) + if cols is not None: + return cols + if template_id in ("imp_def_header", "exp_def_header"): + return TEMPLATE_COLUMNS.get("imp_temp_header") + if template_id in ("imp_def_details", "exp_def_details"): + return TEMPLATE_COLUMNS.get("imp_temp_details") + return None + + +def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]: + """ + Construye un diccionario: normalized_header -> canonical_name. + normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos). + """ + cols = _resolve_template_columns(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]: + """ + A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict + solo con las columnas de la plantilla, usando nombres canónicos. + Así la plantilla se respeta: solo entran columnas definidas en la plantilla. + """ + lookup = build_normalized_lookup(template_id, normalize_header_fn) + if not lookup: + # Sin template definido: comportamiento legacy (normalizar todo) + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/invoices/catalog_service.py b/backend/api/v1/modules/a76/invoices/catalog_service.py index 2dee90a7..2c7feb47 100644 --- a/backend/api/v1/modules/a76/invoices/catalog_service.py +++ b/backend/api/v1/modules/a76/invoices/catalog_service.py @@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.public.reference_data.transport_types.models import TransportType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection +from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.transport_modes.models import TransportMode diff --git a/backend/api/v1/modules/a76/invoices/schemas.py b/backend/api/v1/modules/a76/invoices/schemas.py index f117377e..55124a29 100644 --- a/backend/api/v1/modules/a76/invoices/schemas.py +++ b/backend/api/v1/modules/a76/invoices/schemas.py @@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel): operation_type: Optional[OperationType] = Field( ..., description="Operation type: imp/exp/sm/ctm" ) + + @field_validator("operation_type", mode="before") + @classmethod + def normalize_operation_type(cls, v): + """Accept DB string (e.g. 'IMP') and coerce to enum value ('imp').""" + if isinstance(v, str): + return v.lower() if v else v + return v invoice_type: Optional[str] = Field( None, max_length=5, description="Invoice type key" ) @@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel): class InvoiceFinancialsBase(BaseModel): """Base fields for Financials""" - currency: Currency = Field(None, max_length=7, description="Currency code") + currency: Optional[Currency] = Field(None, max_length=7, description="Currency code") currency_type: Optional[str] = Field("USD", description="Currency type") - exchange_rate: Decimal = Field(0.00, description="Exchange rate") + exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate") exchange_rate_mm: Optional[Decimal] = Field( None, description="Exchange rate currency to currency" ) diff --git a/backend/api/v1/modules/a76/invoices/services.py b/backend/api/v1/modules/a76/invoices/services.py index f232366d..90e6a84a 100644 --- a/backend/api/v1/modules/a76/invoices/services.py +++ b/backend/api/v1/modules/a76/invoices/services.py @@ -67,11 +67,13 @@ class InvoiceService: # Apply filters if provided if filters: - if filters.get("status"): + if filters.get("status") is not None: query = query.filter(models.InvoiceHeader.is_updated == filters["status"]) if filters.get("operation_type"): + ot = filters["operation_type"] + ot_val = ot.value if hasattr(ot, "value") else ot query = query.filter( - models.InvoiceHeader.operation_type == filters["operation_type"] + models.InvoiceHeader.operation_type == ot_val ) if filters.get("invoice_type"): query = query.filter( @@ -89,10 +91,9 @@ class InvoiceService: f"%{filters['pedimento']}%" ) ) - if ( - not filters.get("invoice_type") - and filters.get("operation_type") == "exp" - ): + ot_exp = filters.get("operation_type") + ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp + if not filters.get("invoice_type") and ot_exp_val == "exp": query = query.filter(models.InvoiceHeader.operation_type != "REPAR") if filters.get("manifest_number"): diff --git a/backend/api/v1/modules/a76/parts/imports/__init__.py b/backend/api/v1/modules/a76/parts/imports/__init__.py new file mode 100644 index 00000000..025db976 --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Parts (Números de parte) diff --git a/backend/api/v1/modules/a76/parts/imports/routes.py b/backend/api/v1/modules/a76/parts/imports/routes.py new file mode 100644 index 00000000..bffb87ac --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para Números de Parte. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + PART_IMPORT_FILE_PREFIX, + PART_IMPORT_META_PREFIX, + PART_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Parts import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "part_numbers", + } + + try: + r = _get_redis() + r.set( + f"{PART_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=PART_IMPORT_REDIS_TTL, + ) + r.set( + f"{PART_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=PART_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Parts import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Parts import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Parts import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/parts/imports/schemas.py b/backend/api/v1/modules/a76/parts/imports/schemas.py new file mode 100644 index 00000000..7827dae6 --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/parts/imports/tasks.py b/backend/api/v1/modules/a76/parts/imports/tasks.py new file mode 100644 index 00000000..c87bca7c --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/tasks.py @@ -0,0 +1,639 @@ +""" +Tareas Celery para importación CSV de Números de Parte. +Flujo: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +PART_IMPORT_FILE_PREFIX = "part_import_file:" +PART_IMPORT_META_PREFIX = "part_import_meta:" +PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:" +PART_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Parts import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"part_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Parts import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{PART_IMPORT_FILE_PREFIX}{job_id}", + f"{PART_IMPORT_META_PREFIX}{job_id}", + f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Parts import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_part( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_currency_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + part_number = (row.get("NUMPARTE") or "").strip() + if not part_number: + return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} + if len(part_number) > 70: + return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"} + + commercial = (row.get("NUMPARTECOM") or "").strip() + if commercial and len(commercial) > 70: + return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"} + + desc_es = (row.get("DESCRIPCIONE") or "").strip() + if desc_es and len(desc_es) > 500: + return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} + desc_en = (row.get("DESCRIPCIONI") or "").strip() + if desc_en and len(desc_en) > 500: + return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} + + part_class = (row.get("CLASE") or "").strip() + if part_class and len(part_class) > 8: + return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} + # Si CLASE no existe en catálogo se guardará null (no se rechaza la fila) + + uom = (row.get("UNIMED") or "").strip() + if uom and len(uom) > 5: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} + # Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila) + + currency_key = (row.get("MONEDA") or "").strip() + if currency_key and len(currency_key) > 3: + return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"} + # Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila) + + unit_cost = row.get("COSTOUNIT") + if unit_cost is not None and unit_cost != "": + try: + Decimal(str(unit_cost)) + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"} + + unit_weight = row.get("PESOUNIT") + if unit_weight is not None and unit_weight != "": + try: + Decimal(str(unit_weight)) + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"} + + fraction = (row.get("FRACCION") or "").strip() + if fraction and len(fraction) > 10: + return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"} + us_fraction = (row.get("FRACCIONAME") or "").strip() + if us_fraction and len(us_fraction) > 16: + return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} + fda_key = (row.get("FDAKEY") or "").strip() + if fda_key and len(fda_key) > 20: + return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"} + fcc_key = (row.get("FCCKEY") or "").strip() + if fcc_key and len(fcc_key) > 30: + return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"} + license_code = (row.get("LICENCIA") or "").strip() + if license_code and len(license_code) > 3: + return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"} + eccn = (row.get("ECCN") or "").strip() + if eccn and len(eccn) > 20: + return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"} + export_code = (row.get("EXPORTCODE") or "").strip() + if export_code and len(export_code) > 2: + return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"} + exclusion = (row.get("EXCLUSION") or "").strip() + if exclusion and len(exclusion) > 19: + return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"} + weight_type = (row.get("TIPOPESO") or "").strip() + if weight_type and len(weight_type) > 6: + return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Parts import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Parts import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_class_codes: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_currency_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + for c in ( + session.query(Class.class_code) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + valid_class_codes.add(c[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + for cur in session.query(CurrencyType.code).all(): + valid_currency_codes.add(cur[0]) + except Exception as e: + logger.warning(f"Parts import: could not load FK sets: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Parts import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=PART_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Parts import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _bool_from_row(val: Any) -> bool: + if val is None or val == "": + return True + s = str(val).strip().upper() + if s in ("0", "F", "FALSE", "NO", "N"): + return False + return True + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Parts import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Parts import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.parts.models import Part + + valid_class_codes: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_currency_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + for c in ( + session.query(Class.class_code) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + valid_class_codes.add(c[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + for cur in session.query(CurrencyType.code).all(): + valid_currency_codes.add(cur[0]) + except Exception as e: + logger.warning(f"Parts import: could not load FK sets: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_part_number: Dict[str, Part] = {} + for p in ( + session.query(Part) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + existing_by_part_number[p.part_number] = p + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + part_number = _str_or_none(row_norm.get("NUMPARTE"), 70) + if not part_number: + skipped_invalid += 1 + continue + + commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + part_class = _str_or_none(row_norm.get("CLASE"), 8) + if part_class and part_class not in valid_class_codes: + part_class = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + currency_key = _str_or_none(row_norm.get("MONEDA"), 3) + if currency_key and currency_key not in valid_currency_codes: + currency_key = None + + unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT")) + currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None + unit_weight = _decimal_or_none(row_norm.get("PESOUNIT")) + weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6) + fraction = _str_or_none(row_norm.get("FRACCION"), 10) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + fda_key = _str_or_none(row_norm.get("FDAKEY"), 20) + fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30) + license_code = _str_or_none(row_norm.get("LICENCIA"), 3) + eccn = _str_or_none(row_norm.get("ECCN"), 20) + export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2) + exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19) + is_active = _bool_from_row(row_norm.get("ACTIVO")) + + existing = existing_by_part_number.get(part_number) + if existing: + existing.commercial_part_number = commercial + existing.description_spanish = desc_es + existing.description_english = desc_en + existing.part_class = part_class + existing.unit_of_measure = unit_of_measure + existing.unit_cost = unit_cost + existing.currency_type = currency_type + existing.currency_key = currency_key + existing.unit_weight = unit_weight + existing.weight_type = weight_type + existing.fraction = fraction + existing.us_fraction = us_fraction + existing.fda_key = fda_key + existing.fcc_key = fcc_key + existing.license_code = license_code + existing.eccn = eccn + existing.export_code = export_code + existing.exclusion_symbol = exclusion_symbol + existing.is_active = is_active + session.add(existing) + inserted_count += 1 + else: + new_part = Part( + tenant_id=tenant_id, + company_id=company_id, + client_id=company_id, + part_number=part_number, + commercial_part_number=commercial, + description_spanish=desc_es, + description_english=desc_en, + part_class=part_class, + unit_of_measure=unit_of_measure, + unit_cost=unit_cost, + currency_type=currency_type, + currency_key=currency_key, + unit_weight=unit_weight, + weight_type=weight_type, + fraction=fraction, + us_fraction=us_fraction, + fda_key=fda_key, + fcc_key=fcc_key, + license_code=license_code, + eccn=eccn, + export_code=export_code, + exclusion_symbol=exclusion_symbol, + is_active=is_active, + ) + session.add(new_part) + existing_by_part_number[part_number] = new_part + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Parts import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Parts import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Parts import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/parts/imports/template_config.py b/backend/api/v1/modules/a76/parts/imports/template_config.py new file mode 100644 index 00000000..7cc240c4 --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/template_config.py @@ -0,0 +1,54 @@ +""" +Configuración de plantilla CSV para Números de Parte (EstructuraCatPartesAF.xls). +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "part_numbers": [ + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE", "PART NUMBER", "NUM PARTE"]}, + {"canonical": "NUMPARTECOM", "aliases": ["NUMERO PARTE COMERCIAL", "COMMERCIAL PART", "PARTE COMERCIAL"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES", "DESC ESPANOL"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION", "DESC INGLES"]}, + {"canonical": "CLASE", "aliases": ["CLASS", "CLASE MATERIAL", "PART CLASS"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM", "UNIT OF MEASURE"]}, + {"canonical": "COSTOUNIT", "aliases": ["COSTO UNITARIO", "UNIT COST", "COSTO"]}, + {"canonical": "MONEDA", "aliases": ["CURRENCY", "MONEDA CLAVE", "CURRENCY KEY"]}, + {"canonical": "PESOUNIT", "aliases": ["PESO UNITARIO", "UNIT WEIGHT", "PESO"]}, + {"canonical": "TIPOPESO", "aliases": ["WEIGHT TYPE", "TIPO PESO"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, + {"canonical": "FDAKEY", "aliases": ["FDA", "FDA KEY"]}, + {"canonical": "FCCKEY", "aliases": ["FCC", "FCC KEY"]}, + {"canonical": "LICENCIA", "aliases": ["LICENSE CODE", "LICENSE"]}, + {"canonical": "ECCN", "aliases": ["ECCN CODE"]}, + {"canonical": "EXPORTCODE", "aliases": ["EXPORT CODE", "CODIGO EXPORT"]}, + {"canonical": "EXCLUSION", "aliases": ["EXCLUSION SYMBOL", "SIMBOLO EXCLUSION"]}, + {"canonical": "ACTIVO", "aliases": ["IS ACTIVE", "ACTIVE", "ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("part_numbers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/parts/routes.py b/backend/api/v1/modules/a76/parts/routes.py index 033ad472..17bf937a 100644 --- a/backend/api/v1/modules/a76/parts/routes.py +++ b/backend/api/v1/modules/a76/parts/routes.py @@ -2,21 +2,31 @@ Endpoints API para gestión de partes (SCAII) """ -from api.v1.common.tenant_crud_routes import TenantCRUDRoutes +from fastapi import APIRouter + +from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import PartCreateDTO, PartResponseDTO, PartUpdateDTO from .service import PartService +from .imports.routes import router as imports_router +router = APIRouter() -router = TenantCRUDRoutes( - service=PartService, - create_schema=PartCreateDTO, - update_schema=PartUpdateDTO, - response_schema=PartResponseDTO, - prefix="/parts", - tags=["a76 / parts"], - resource_name="Part", - id_name="part_id", - enable_list=True, - enable_filters=True, -).router \ No newline at end of file +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/parts/imports", tags=["a76 / parts / csv_import"]) + +# CRUD +router.include_router( + TenantCRUDRoutes( + service=PartService, + create_schema=PartCreateDTO, + update_schema=PartUpdateDTO, + response_schema=PartResponseDTO, + prefix="/parts", + tags=["a76 / parts"], + resource_name="Part", + id_name="part_id", + enable_list=True, + enable_filters=True, + ).router +) \ No newline at end of file diff --git a/backend/api/v1/modules/a76/pedmientos/imports/__init__.py b/backend/api/v1/modules/a76/pedmientos/imports/__init__.py new file mode 100644 index 00000000..8378bf88 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Pedimentos (upload → scan → commit) diff --git a/backend/api/v1/modules/a76/pedmientos/imports/routes.py b/backend/api/v1/modules/a76/pedmientos/imports/routes.py new file mode 100644 index 00000000..f624e5dc --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Pedimentos. +Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + PED_IMPORT_FILE_PREFIX, + PED_IMPORT_META_PREFIX, + PED_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Pedimentos import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "pedimentos", + } + + try: + r = _get_redis() + r.set( + f"{PED_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=PED_IMPORT_REDIS_TTL, + ) + r.set( + f"{PED_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=PED_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Pedimentos import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Pedimentos import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/pedmientos/imports/schemas.py b/backend/api/v1/modules/a76/pedmientos/imports/schemas.py new file mode 100644 index 00000000..6fe69385 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: Optional[str] = None + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/pedmientos/imports/tasks.py b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py new file mode 100644 index 00000000..82b630d2 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py @@ -0,0 +1,563 @@ +""" +Tareas Celery para importación CSV de Pedimentos. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from datetime import datetime +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con otros imports) +PED_IMPORT_FILE_PREFIX = "ped_import_file:" +PED_IMPORT_META_PREFIX = "ped_import_meta:" +PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" +PED_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"ped_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{PED_IMPORT_FILE_PREFIX}{job_id}", + f"{PED_IMPORT_META_PREFIX}{job_id}", + f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_pedimento( + row: Dict[str, Any], + line_num: int, + valid_client_ids: Set[int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], +) -> Optional[Dict[str, Any]]: + """Valida una fila para Pedimento. Retorna error dict o None.""" + # Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN + year = (row.get("AÑO") or "").strip() + if not year: + return {"line": line_num, "col": "AÑO", "msg": "Requerido"} + if len(year) > 2: + return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"} + + customs_office = (row.get("ADUANA") or "").strip() + if not customs_office: + return {"line": line_num, "col": "ADUANA", "msg": "Requerido"} + if len(customs_office) > 3: + return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"} + + license_val = (row.get("PATENTE") or "").strip() + if not license_val: + return {"line": line_num, "col": "PATENTE", "msg": "Requerido"} + if len(license_val) > 4: + return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"} + + pedimento_number = (row.get("NUMERO") or "").strip() + if not pedimento_number: + return {"line": line_num, "col": "NUMERO", "msg": "Requerido"} + if len(pedimento_number) > 7: + return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"} + + client_id_str = (row.get("CLIENTE_ID") or "").strip() + if not client_id_str: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"} + try: + client_id = int(client_id_str) + except ValueError: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"} + if client_id not in valid_client_ids: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"} + + pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip() + if not pedimento_code: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"} + if len(pedimento_code) > 2: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"} + if pedimento_code not in valid_pedimento_codes: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"} + + regime = (row.get("REGIMEN") or "").strip() + if not regime: + return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"} + if len(regime) > 3: + return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"} + if regime not in valid_regimes: + return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"} + + # Optional numeric/string fields - validate format if present + status = (row.get("ESTATUS") or "").strip() + if status and len(status) > 30: + return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"} + + for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]: + val = (row.get(col) or "").strip() + if not val: + continue + try: + Decimal(val.replace(",", ".")) + except (InvalidOperation, ValueError): + return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} + + operation_type = (row.get("TIPO_OPERACION") or "").strip().lower() + if operation_type and operation_type not in ("imp", "exp", ""): + return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"} + + pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower() + if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""): + return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"} + + return None + + +def _parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + return Decimal(str(val).strip().replace(",", ".")) + except (InvalidOperation, ValueError): + return None + + +def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]: + """Build dict for PedimentosCreate from normalized CSV row (canonical names).""" + year = (row.get("AÑO") or "").strip()[:2] + customs_office = (row.get("ADUANA") or "").strip()[:3] + license_val = (row.get("PATENTE") or "").strip()[:4] + pedimento_number = (row.get("NUMERO") or "").strip()[:7] + client_id_str = (row.get("CLIENTE_ID") or "").strip() + client_id = int(client_id_str) if client_id_str else None + pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2] + regime = (row.get("REGIMEN") or "").strip()[:3] + + data = { + "year": year, + "customs_office": customs_office, + "license": license_val, + "pedimento_number": pedimento_number, + "client_id": client_id, + "pedimento_code": pedimento_code, + "regime": regime, + } + + op = (row.get("TIPO_OPERACION") or "").strip().lower() + if op in ("imp", "exp"): + data["operation_type"] = op + + ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower() + if ptype in ("normal", "consolidated", "complementary", "automobile"): + data["pedimento_type"] = ptype + + status = (row.get("ESTATUS") or "").strip() + if status: + data["status"] = status[:30] + + data["usd_value"] = _parse_decimal(row.get("VALOR_USD")) + data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO")) + data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO")) + data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO")) + + obs = (row.get("OBSERVACIONES") or "").strip() + if obs: + data["observations"] = obs + + return data + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"Pedimentos import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Pedimentos import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + # Load valid FK sets for validation + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + for cp in session.query(ClientProvider).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + except Exception as e: + logger.error(f"Pedimentos import: failed to load FK sets: {e}") + return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, "pedimentos") + err = _validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Pedimentos import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=PED_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create. + """ + logger.info(f"Pedimentos import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + # Reload FK sets for commit-time validation + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + for cp in session.query(ClientProvider).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + except Exception as e: + logger.error(f"Pedimentos import: failed to load FK sets: {e}") + return {"status": "failed", "error": str(e)} + + from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate + from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate + from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService + + inserted_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, "pedimentos") + err = _validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + try: + data = _row_to_pedimentos_create(row_norm) + # Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL + if "pedimento_dates" not in data or data.get("pedimento_dates") is None: + data["pedimento_dates"] = PedimentoDatesCreate( + entry_date=datetime.now(), + end_date=datetime.now(), + ) + create_data = PedimentosCreate(**data) + PedimentosService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except ValueError as ve: + if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): + skipped_duplicate += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + else: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + except Exception as e: + logger.warning(f"Pedimentos import line {i}: {e}") + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(e)}) + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Pedimentos import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/pedmientos/imports/template_config.py b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py new file mode 100644 index 00000000..06fea167 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py @@ -0,0 +1,53 @@ +""" +Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "pedimentos": [ + {"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]}, + {"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]}, + {"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]}, + {"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]}, + {"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]}, + {"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]}, + {"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]}, + {"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]}, + {"canonical": "REGIMEN", "aliases": ["REGIME"]}, + {"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]}, + {"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]}, + {"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]}, + {"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]}, + {"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]}, + {"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla pedimentos.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/pedmientos/router.py b/backend/api/v1/modules/a76/pedmientos/router.py index f294cba4..2b987b46 100644 --- a/backend/api/v1/modules/a76/pedmientos/router.py +++ b/backend/api/v1/modules/a76/pedmientos/router.py @@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import ( from .routes.pedimento_transport_means import router as pedimento_transport_means_router from .routes.pedimento_validation import router as pedimento_validation_router from .routes.pedimentos import router as pedimentos_router +from .imports.routes import router as pedimentos_imports_router router = APIRouter() @@ -117,3 +118,8 @@ router.include_router( router.include_router( pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"] ) +router.include_router( + pedimentos_imports_router, + prefix="/pedimentos/imports", + tags=["a76 / pedimentos / csv_import"], +) diff --git a/backend/api/v1/modules/a76/router.py b/backend/api/v1/modules/a76/router.py index b8d0e610..eafc4d2b 100644 --- a/backend/api/v1/modules/a76/router.py +++ b/backend/api/v1/modules/a76/router.py @@ -12,9 +12,10 @@ from .general_catalogs.router import router as general_catalogs_router from .invoices.routes import router as invoices_router from .items.routes import router as items_router from .classes import router as classes_router -from .classes import router as classes_router + from .clients_and_providers import router as client_and_provider_router from .imports.routes import router as imports_router +from .csv_templates.routes import router as csv_templates_router from .invoice_settings.routes import router as invoice_settings_router from .item_presets.routes import router as item_presets_router from .general_catalogs.company import router as company_router @@ -23,6 +24,7 @@ from .transportation.drivers.routes import router as drivers_router from .doc_types_dig.routes import router as doc_types_dig_router from .fraction_rule_octave.routes import router as fraction_rule_octave_router from .parts import router as parts_router +from .boms import router as boms_router from .pedmientos.router import router as pedimentos_router from .permission_rule_oct.routes import router as permission_rule_oct_router from .transportation.trailers.routes import router as trailers_router @@ -54,12 +56,14 @@ router.include_router(general_catalogs_router, prefix="/a76", tags=["a76 / gener router.include_router(invoices_router, prefix="/a76", tags=["a76 / invoices"]) router.include_router(items_router, prefix="/a76", tags=["a76 / items"]) router.include_router(imports_router, prefix="/a76/imports", tags=["a76 / imports"]) +router.include_router(csv_templates_router, prefix="/a76/csv-templates", tags=["a76 / csv_templates"]) router.include_router(invoice_settings_router) router.include_router(item_presets_router, prefix="/a76/item-presets", tags=["a76 / item_presets"]) router.include_router(pedimentos_router, prefix="/a76") router.include_router(client_and_provider_router, prefix="/a76", tags=["a76 / clients_and_providers"]) router.include_router(classes_router, prefix="/a76/classes", tags=["a76 / classes"]) router.include_router(parts_router, prefix="/a76", tags=["a76 / parts"]) +router.include_router(boms_router, prefix="/a76/boms", tags=["a76 / boms"]) router.include_router(permission_rule_oct_router, prefix="/a76", tags=["a76 / permission_rule_oct"]) router.include_router(fraction_rule_octave_router, prefix="/a76", tags=["a76 / fraction_rule_octave"]) router.include_router(country_rule_oct_router, prefix="/a76", tags=["a76 / country_rule_oct"]) diff --git a/backend/api/v1/modules/a76/transportation/drivers/dto.py b/backend/api/v1/modules/a76/transportation/drivers/dto.py index f07e4b4c..bfb3d8fc 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/dto.py +++ b/backend/api/v1/modules/a76/transportation/drivers/dto.py @@ -6,28 +6,28 @@ from pydantic import BaseModel class DriverBaseDTO(BaseModel): transporter_key: str line: int - driver_name: Optional[str] - license_number: Optional[str] - express_line_id: Optional[str] - ace_id: Optional[str] - birth_date: Optional[int] - gender: Optional[str] - birth_country: Optional[str] - hazardous_material_auth: Optional[str] - hazardous_material_state: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - id_key1: Optional[str] - id_number1: Optional[str] - id_state1: Optional[str] - id_country1: Optional[str] - id_key2: Optional[str] - id_number2: Optional[str] - id_state2: Optional[str] - id_country2: Optional[str] - badge_number: Optional[str] - class_type: Optional[str] - unique_badge_number: Optional[str] + driver_name: Optional[str] = None + license_number: Optional[str] = None + express_line_id: Optional[str] = None + ace_id: Optional[str] = None + birth_date: Optional[int] = None + gender: Optional[str] = None + birth_country: Optional[str] = None + hazardous_material_auth: Optional[str] = None + hazardous_material_state: Optional[str] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + id_key1: Optional[str] = None + id_number1: Optional[str] = None + id_state1: Optional[str] = None + id_country1: Optional[str] = None + id_key2: Optional[str] = None + id_number2: Optional[str] = None + id_state2: Optional[str] = None + id_country2: Optional[str] = None + badge_number: Optional[str] = None + class_type: Optional[str] = None + unique_badge_number: Optional[str] = None company_id: int tenant_id: int diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py b/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py new file mode 100644 index 00000000..98bef45c --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Drivers / Conductores (upload -> scan -> status -> commit). diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py b/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py new file mode 100644 index 00000000..ea5d2eb4 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py @@ -0,0 +1,188 @@ +""" +Rutas de importacion CSV para Conductores. +Flujo: upload -> scan -> status (polling) -> commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + DRV_IMPORT_FILE_PREFIX, + DRV_IMPORT_META_PREFIX, + DRV_IMPORT_STATUS_PREFIX, + DRV_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Drivers import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "drivers", + } + + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=DRV_IMPORT_REDIS_TTL, + ) + r.set( + f"{DRV_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Drivers import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Drivers import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception(f"Drivers import: background scan failed: {e}") + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + try: + r = _get_redis() + raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug(f"Drivers import: could not read status from Redis: {e}") + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug(f"Drivers import: could not write processing status: {e}") + + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception(f"Drivers import: background commit failed: {e}") + + threading.Thread(target=run_commit_background, daemon=True).start() + + return { + "status": "committing", + "message": "Insercion iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py b/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py new file mode 100644 index 00000000..c5d2c2cc --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + updated: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py b/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py new file mode 100644 index 00000000..9234776e --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py @@ -0,0 +1,640 @@ +""" +Tareas Celery para importacion CSV de Conductores. +Flujo: scan_file (validacion) -> insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +DRV_IMPORT_FILE_PREFIX = "drv_import_file:" +DRV_IMPORT_META_PREFIX = "drv_import_meta:" +DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:" +DRV_IMPORT_STATUS_PREFIX = "drv_import_status:" +DRV_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Drivers import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"drv_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Drivers import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{DRV_IMPORT_FILE_PREFIX}{job_id}", + f"{DRV_IMPORT_META_PREFIX}{job_id}", + f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Drivers import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +_MAX = { + "transporter_key": 5, + "driver_name": 80, + "license_number": 29, + "express_line_id": 17, + "ace_id": 20, + "gender": 1, + "birth_country": 3, + "hazardous_material_auth": 2, + "hazardous_material_state": 30, + "first_name": 20, + "last_name": 20, + "id_key1": 40, + "id_number1": 20, + "id_state1": 30, + "id_country1": 3, + "id_key2": 40, + "id_number2": 20, + "id_state2": 30, + "id_country2": 3, + "badge_number": 20, + "class_type": 1, + "unique_badge_number": 100, +} + + +def _parse_int(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d+$", s): + return int(s) + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + return None + + +def _parse_birth_date(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + return None + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + return None + break + return None + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _dedupe_headers(headers: List[str]) -> List[str]: + counts: Dict[str, int] = {} + unique: List[str] = [] + for header in headers: + name = str(header or "").strip() or "COL" + count = counts.get(name, 0) + 1 + counts[name] = count + if count == 1: + unique.append(name) + else: + unique.append(f"{name} {count}") + return unique + + +def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + transporter_key = (row.get("TRANSPORTISTA") or "").strip() + if not transporter_key: + return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"} + if len(transporter_key) > _MAX["transporter_key"]: + return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"} + + line_val = _parse_int(row.get("LINEA")) + if line_val is None: + return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"} + if line_val <= 0: + return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"} + + driver_name = (row.get("CLAVE CONDUCTOR") or "").strip() + if driver_name and len(driver_name) > _MAX["driver_name"]: + return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"} + + for col, max_len in [ + ("LICENCIA", _MAX["license_number"]), + ("PERMISO LINEA EXPRESS", _MAX["express_line_id"]), + ("IDENTIFICACION ACE", _MAX["ace_id"]), + ("SEXO", _MAX["gender"]), + ("PAIS NACIMIENTO", _MAX["birth_country"]), + ("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]), + ("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]), + ("NOMBRE(S)", _MAX["first_name"]), + ("APELLIDO PATERNO", _MAX["last_name"]), + ("FORMA IDENTIFICACION 1", _MAX["id_key1"]), + ("NUM. IDENTIFICACION 1", _MAX["id_number1"]), + ("ESTADO", _MAX["id_state1"]), + ("PAIS", _MAX["id_country1"]), + ("FORMA IDENTIFICACION 2", _MAX["id_key2"]), + ("NUM. IDENTIFICACION 2", _MAX["id_number2"]), + ("ESTADO 2", _MAX["id_state2"]), + ("PAIS 2", _MAX["id_country2"]), + ]: + val = (row.get(col) or "").strip() + if val and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"} + + fecha = row.get("FECHA NACIMIENTO") + if fecha is not None and str(fecha).strip(): + if _parse_birth_date(fecha) is None: + return { + "line": line_num, + "col": "FECHA NACIMIENTO", + "msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)", + } + + return None + + +def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: + transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"]) + line = _parse_int(row.get("LINEA")) + if not transporter_key or line is None: + return {} + data = { + "transporter_key": transporter_key, + "line": line, + "driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]), + "license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]), + "express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]), + "ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]), + "birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")), + "gender": _str_or_none(row.get("SEXO"), _MAX["gender"]), + "birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]), + "hazardous_material_auth": _str_or_none( + row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"] + ), + "hazardous_material_state": _str_or_none( + row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"] + ), + "first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]), + "last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]), + "id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]), + "id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]), + "id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]), + "id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]), + "id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]), + "id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]), + "id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]), + "id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]), + "company_id": company_id, + "tenant_id": tenant_id, + } + return data + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Drivers import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f_in, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_driver(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Drivers import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store error lines: {e}") + + result = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + return result + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Drivers import: starting scan for job {job_id}") + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state( + state="PROGRESS", + meta={"current": current, "total": total, "errors": errors}, + ) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Drivers import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.transportation.drivers.services import DriverService + from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_driver(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", + "invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + data = _row_to_driver_dto(row_norm, tenant_id, company_id) + if not data or not data.get("transporter_key") or data.get("line") is None: + skipped_invalid += 1 + continue + + key = f"{data['transporter_key']}:{data['line']}" + if key in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append( + { + "line": i, + "driver_key": key, + "invoice": key, + "reason": "Clave duplicada en el archivo (se usa la primera)", + } + ) + continue + seen_keys_in_file[key] = i + + existing = DriverService.get_driver_by_key_and_line( + session, data["transporter_key"], data["line"], str(company_id), tenant_id + ) + try: + if existing: + update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")} + for field, value in update_fields.items(): + setattr(existing, field, value) + session.add(existing) + updated_count += 1 + else: + create_data = DriverCreateDTO(**data) + DriverService.create_driver(session, create_data) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append( + {"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)} + ) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Drivers import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.error(f"Drivers import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Drivers import cleanup failed: {cleanup_err}") + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros validos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Drivers import: starting commit for job {job_id}") + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") + return result diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py b/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py new file mode 100644 index 00000000..cd9a3194 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py @@ -0,0 +1,56 @@ +""" +Configuracion de plantilla CSV para Conductores (EstructuraCatConductor.xls). +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "drivers": [ + {"canonical": "TRANSPORTISTA", "aliases": ["TRANSPORTISTA CLAVE", "CLAVE TRANSPORTISTA", "TRANSPORTER"]}, + {"canonical": "LINEA", "aliases": ["LINE", "LINEA CONDUCTOR"]}, + {"canonical": "CLAVE CONDUCTOR", "aliases": ["CONDUCTOR", "DRIVER", "NOMBRE CONDUCTOR"]}, + {"canonical": "LICENCIA", "aliases": ["LICENSE", "LICENCE"]}, + {"canonical": "PERMISO LINEA EXPRESS", "aliases": ["LINEA EXPRESS", "LINEA EXPRESS ID", "EXPRESS LINE"]}, + {"canonical": "IDENTIFICACION ACE", "aliases": ["ACE", "ACE ID", "IDENTIFICACION ACE ID"]}, + {"canonical": "FECHA NACIMIENTO", "aliases": ["FECHA NAC", "BIRTH DATE", "BIRTHDATE"]}, + {"canonical": "SEXO", "aliases": ["GENERO", "GENDER"]}, + {"canonical": "PAIS NACIMIENTO", "aliases": ["PAIS Nacimiento", "BIRTH COUNTRY"]}, + {"canonical": "TRANSPORTA MAT. PELIGROSO?", "aliases": ["MATERIAL PELIGROSO", "HAZMAT", "HAZARDOUS MATERIAL"]}, + {"canonical": "PERMISO MAT. PELIGROSO", "aliases": ["PERMISO MAT PELIGROSO", "HAZMAT PERMIT"]}, + {"canonical": "NOMBRE(S)", "aliases": ["NOMBRE", "NOMBRES", "FIRST NAME"]}, + {"canonical": "APELLIDO PATERNO", "aliases": ["APELLIDO", "LAST NAME", "APELLIDO P"]}, + {"canonical": "FORMA IDENTIFICACION 1", "aliases": ["FORMA IDENTIFICACION", "ID TIPO 1", "ID KEY 1"]}, + {"canonical": "NUM. IDENTIFICACION 1", "aliases": ["NUM IDENTIFICACION 1", "ID NUMERO 1", "ID NUMBER 1"]}, + {"canonical": "ESTADO", "aliases": ["ESTADO 1", "STATE 1"]}, + {"canonical": "PAIS", "aliases": ["PAIS 1", "COUNTRY 1"]}, + {"canonical": "FORMA IDENTIFICACION 2", "aliases": ["ID TIPO 2", "ID KEY 2"]}, + {"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]}, + {"canonical": "ESTADO 2", "aliases": ["STATE 2"]}, + {"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("drivers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/transportation/drivers/routes.py b/backend/api/v1/modules/a76/transportation/drivers/routes.py index 981ab353..a07b5c55 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/routes.py +++ b/backend/api/v1/modules/a76/transportation/drivers/routes.py @@ -8,9 +8,13 @@ from sqlalchemy.orm import Session from .dto import DriverCreateDTO, DriverResponseDTO from .models import Driver from .services import DriverService +from .imports.routes import router as imports_router router = APIRouter(prefix="/drivers") +# CSV import (upload -> scan -> status -> commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / drivers / csv_import"]) + @router.get("/", response_model=Dict[str, Any]) async def list_drivers( diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py b/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py new file mode 100644 index 00000000..ef25c98a --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py @@ -0,0 +1 @@ +# Trailers CSV import: upload -> scan -> status -> commit diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py b/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py new file mode 100644 index 00000000..7e51aca1 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py @@ -0,0 +1,188 @@ +""" +Rutas de importación CSV para Trailers y Cajas. +Flujo: upload -> scan -> status (polling) -> commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + TRL_IMPORT_FILE_PREFIX, + TRL_IMPORT_META_PREFIX, + TRL_IMPORT_STATUS_PREFIX, + TRL_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Trailers import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "trailers", + } + + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=TRL_IMPORT_REDIS_TTL, + ) + r.set( + f"{TRL_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Trailers import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Trailers import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception(f"Trailers import: background scan failed: {e}") + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + try: + r = _get_redis() + raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug(f"Trailers import: could not read status from Redis: {e}") + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug(f"Trailers import: could not write processing status: {e}") + + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception(f"Trailers import: background commit failed: {e}") + + threading.Thread(target=run_commit_background, daemon=True).start() + + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py b/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py new file mode 100644 index 00000000..c5d2c2cc --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + updated: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py b/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py new file mode 100644 index 00000000..c43c93cf --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py @@ -0,0 +1,535 @@ +""" +Tareas Celery para importación CSV de Trailers y Cajas. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Upsert por trailer_number usando TrailerService. +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +TRL_IMPORT_FILE_PREFIX = "trl_import_file:" +TRL_IMPORT_META_PREFIX = "trl_import_meta:" +TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:" +TRL_IMPORT_STATUS_PREFIX = "trl_import_status:" +TRL_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Trailers import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"trl_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Trailers import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{TRL_IMPORT_FILE_PREFIX}{job_id}", + f"{TRL_IMPORT_META_PREFIX}{job_id}", + f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Trailers import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +# Max lengths from Trailer model (a76.trailer) +_MAX = { + "trailer_number": 20, + "ace_trailer_number": 10, + "trailer_type_key": 2, + "seal": 15, + "entity_code": 1, + "plate_number": 17, + "state": 30, + "country": 3, + "container_key": 3, +} + + +def _dedupe_headers(headers: List[str]) -> List[str]: + counts: Dict[str, int] = {} + unique: List[str] = [] + for header in headers: + name = str(header or "").strip() or "COL" + count = counts.get(name, 0) + 1 + counts[name] = count + if count == 1: + unique.append(name) + else: + unique.append(f"{name} {count}") + return unique + + +def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Trailer. Retorna error dict o None.""" + trailer_number = (row.get("NUMERO TRAILER") or "").strip() + if not trailer_number: + return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"} + if len(trailer_number) > _MAX["trailer_number"]: + return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"} + + for col, max_len in [ + ("CLAVE ACE", _MAX["ace_trailer_number"]), + ("TIPO TRAILER", _MAX["trailer_type_key"]), + ("PRECINTO", _MAX["seal"]), + ("CODIGO ENTIDAD", _MAX["entity_code"]), + ("PLACAS", _MAX["plate_number"]), + ("ESTADO", _MAX["state"]), + ("PAIS", _MAX["country"]), + ("CLAVE CONTENEDOR", _MAX["container_key"]), + ]: + val = (row.get(col) or "").strip() + if val and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + + return None + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: + """Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row.""" + trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"]) + if not trailer_number: + return {} + return { + "trailer_number": trailer_number, + "ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]), + "trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]), + "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), + "entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]), + "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), + "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), + "country": _str_or_none(row.get("PAIS"), _MAX["country"]), + "container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]), + } + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Trailers import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f_in, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_trailer(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Trailers import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store error lines: {e}") + + result = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + return result + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Trailers import: starting scan for job {job_id}") + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state( + state="PROGRESS", + meta={"current": current, "total": total, "errors": errors}, + ) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Trailers import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.transportation.trailers.services import TrailerService + from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_trailer(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", + "invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + data = _row_to_trailer_dto(row_norm, tenant_id, company_id) + if not data or not data.get("trailer_number"): + skipped_invalid += 1 + continue + + tn = data["trailer_number"] + if tn in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append( + { + "line": i, + "trailer_number": tn, + "invoice": tn, + "reason": "Clave duplicada en el archivo (se usa la primera)", + } + ) + continue + seen_keys_in_file[tn] = i + + existing = TrailerService.get_by_id(session, tn, tenant_id, company_id) + try: + if existing: + update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"}) + TrailerService.update(session, tn, tenant_id, update_data, company_id) + updated_count += 1 + else: + create_data = TrailerCreateDTO(**data) + TrailerService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append( + {"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)} + ) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Trailers import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.error(f"Trailers import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Trailers import cleanup failed: {cleanup_err}") + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Trailers import: starting commit for job {job_id}") + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") + return result diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py b/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py new file mode 100644 index 00000000..be0eb4a7 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py @@ -0,0 +1,45 @@ +""" +Configuración de plantilla CSV para Trailers / Cajas (EstructuraCatTrailers.xls). +Mapeo: NUMERO TRAILER → trailer_number, CLAVE ACE → ace_trailer_number, etc. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "trailers": [ + {"canonical": "NUMERO TRAILER", "aliases": ["CLAVE TRAILER", "TRAILER NUMBER", "TRAILER", "NUMERO"]}, + {"canonical": "CLAVE ACE", "aliases": ["ACE", "NUMERO ACE", "ACE TRAILER"]}, + {"canonical": "TIPO TRAILER", "aliases": ["TIPO", "TRAILER TYPE", "TIPO CAJA"]}, + {"canonical": "PRECINTO", "aliases": ["SEAL"]}, + {"canonical": "CODIGO ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE", "CODIGO DE ENTIDAD"]}, + {"canonical": "PLACAS", "aliases": ["PLACA", "PLATE NUMBER", "PLATE"]}, + {"canonical": "ESTADO", "aliases": ["STATE"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]}, + {"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("trailers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/transportation/trailers/routes.py b/backend/api/v1/modules/a76/transportation/trailers/routes.py index ad108e46..4dcea2e3 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/routes.py +++ b/backend/api/v1/modules/a76/transportation/trailers/routes.py @@ -1,11 +1,19 @@ +from fastapi import APIRouter + from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO from .services import TrailerService +from .imports.routes import router as imports_router -# Create router using TenantCRUDRoutes factory -# Note: trailer_number is a string (not int) and is used as the primary key -router = TenantCRUDRoutes( +# Main router: trailers CRUD + CSV imports +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/trailers/imports", tags=["a76 / trailers / csv_import"]) + +# CRUD routes +crud_router = TenantCRUDRoutes( service=TrailerService, create_schema=TrailerCreateDTO, update_schema=TrailerUpdateDTO, @@ -13,10 +21,11 @@ router = TenantCRUDRoutes( prefix="/trailers", tags=[], resource_name="Trailer", - id_name="trailer_number", # Using trailer_number instead of numeric ID - id_type=str, # Specify that the ID is a string - enable_list=True, # Enable GET /trailers with pagination - enable_filters=True, # Enable filtering by plate_number and trailer_type_key + id_name="trailer_number", + id_type=str, + enable_list=True, + enable_filters=True, default_page_size=50, max_page_size=100, ).router +router.include_router(crud_router) diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py new file mode 100644 index 00000000..87b6b8b2 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Vehicles / Transportes (upload → scan → status → commit). diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py new file mode 100644 index 00000000..20960753 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py @@ -0,0 +1,192 @@ +""" +Rutas de importación CSV para Vehículos (Transportes). +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + VEHL_IMPORT_FILE_PREFIX, + VEHL_IMPORT_META_PREFIX, + VEHL_IMPORT_STATUS_PREFIX, + VEHL_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Vehicles import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "vehicles", + } + + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=VEHL_IMPORT_REDIS_TTL, + ) + r.set( + f"{VEHL_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Vehicles import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Vehicles import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + # Fallback sin worker: ejecutar scan en un hilo y guardar resultado en Redis + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception(f"Vehicles import: background scan failed: {e}") + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + # Si el resultado del scan está en Redis (worker o hilo de fallback), usarlo + try: + r = _get_redis() + raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug(f"Vehicles import: could not read status from Redis: {e}") + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + # Escribir "processing" en Redis para que el polling muestre estado hasta que termine el commit + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug(f"Vehicles import: could not write processing status: {e}") + + # Ejecutar commit en hilo (mismo fallback que el scan: no depende del worker de Celery) + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception(f"Vehicles import: background commit failed: {e}") + + threading.Thread(target=run_commit_background, daemon=True).start() + + # Mismo job_id para que el front siga haciendo polling y reciba el resultado de Redis + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py new file mode 100644 index 00000000..7827dae6 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py new file mode 100644 index 00000000..7e9df06c --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py @@ -0,0 +1,607 @@ +""" +Tareas Celery para importación CSV de Vehículos (Transportes). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key. +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +VEHL_IMPORT_FILE_PREFIX = "veh_import_file:" +VEHL_IMPORT_META_PREFIX = "veh_import_meta:" +VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:" +VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:" +VEHL_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Vehicles import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"veh_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", + f"{VEHL_IMPORT_META_PREFIX}{job_id}", + f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +# Max lengths from Vehicle model (a76.vehicle) +_MAX = { + "vehicle_key": 14, + "ace_vehicle_key": 10, + "transporter_key": 23, + "transport_identifier": 30, + "transport_type": 2, + "entity_code": 1, + "transponder_number": 16, + "dot_number": 8, + "plate_number": 17, + "city": 30, + "state": 30, + "country": 3, + "seal": 49, + "insurance_company_name": 30, + "insurance_number": 20, + "series": 30, +} + + +def _parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + s = str(val).strip().replace(",", "") + return Decimal(s) + except Exception: + return None + + +def _parse_insurance_date(val: Any) -> Optional[int]: + """Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD.""" + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if not s: + return None + # Already integer-like + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + pass + # Try DD/MM/YYYY or similar + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + pass + break + return None + + +def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Vehículo. Retorna error dict o None.""" + clave = (row.get("CLAVE") or "").strip() + if not clave: + return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} + if len(clave) > _MAX["vehicle_key"]: + return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"} + + # Optional fields: max lengths only + for col, max_len in [ + ("CLAVE ACE", _MAX["ace_vehicle_key"]), + ("CLAVE TRANSPORTE", _MAX["transporter_key"]), + ("VIN", _MAX["series"]), + ("TIPO TRANSPORTE", _MAX["transport_type"]), + ("CODIGO DE ENTIDAD", _MAX["entity_code"]), + ("TRANSPONDEDOR", _MAX["transponder_number"]), + ("NUMERO DOT", _MAX["dot_number"]), + ("PLACAS", _MAX["plate_number"]), + ("CIUDAD", _MAX["city"]), + ("ESTADO", _MAX["state"]), + ("PAIS", _MAX["country"]), + ("PRECINTO", _MAX["seal"]), + ("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]), + ("NUM. ASEGURADORA", _MAX["insurance_number"]), + ]: + val = (row.get(col) or "").strip() + if val and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + + # MONTO ASEGURADO: must be numeric if present + monto = row.get("MONTO ASEGURADO") or row.get("MONTO") + if monto is not None and str(monto).strip(): + if _parse_decimal(monto) is None: + return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"} + + # FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None) + # Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either + # reject or set None. We reject invalid date to keep data quality. + fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA") + if fecha is not None and str(fecha).strip(): + if _parse_insurance_date(fecha) is None: + return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} + + return None + + +def _do_scan( + job_id: str, + progress_callback: Optional[Any] = None, +) -> Dict[str, Any]: + """ + Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync. + progress_callback(current, total, error_count) opcional. + """ + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Vehicles import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_vehicle(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Vehicles import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}") + + result = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + return result + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + """ + Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis. + Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery. + """ + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"Vehicles import: starting scan for job {job_id}") + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state( + state="PROGRESS", + meta={"current": current, "total": total, "errors": errors}, + ) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") + return result + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]: + """Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row.""" + vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"]) + if not vehicle_key: + return {} + data = { + "vehicle_key": vehicle_key, + "ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]), + "transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]), + "series": _str_or_none(row.get("VIN"), _MAX["series"]), + "transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]), + "entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]), + "transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]), + "dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]), + "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), + "city": _str_or_none(row.get("CIUDAD"), _MAX["city"]), + "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), + "country": _str_or_none(row.get("PAIS"), _MAX["country"]), + "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), + "insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]), + "insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]), + "insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")), + "insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")), + } + return data + + +def _do_commit(job_id: str) -> Dict[str, Any]: + """ + Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync. + """ + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Vehicles import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.transportation.vehicles.services import VehicleService + from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_vehicle(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-", + "invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-", + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + data = _row_to_vehicle_dto(row_norm) + if not data or not data.get("vehicle_key"): + skipped_invalid += 1 + continue + + vk = data["vehicle_key"] + if vk in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append( + {"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"} + ) + continue + seen_keys_in_file[vk] = i + + existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) + try: + if existing: + update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) + VehicleService.update(session, vk, tenant_id, update_data, company_id) + updated_count += 1 + else: + create_data = VehicleCreateDTO(**data) + VehicleService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append( + {"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)} + ) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Vehicles import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.error(f"Vehicles import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Vehicles import cleanup failed: {cleanup_err}") + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + """ + Ejecuta el commit en el proceso actual y guarda el resultado en Redis. + Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery. + """ + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService. + """ + logger.info(f"Vehicles import: starting commit for job {job_id}") + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") + return result diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py new file mode 100644 index 00000000..3474ae50 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py @@ -0,0 +1,53 @@ +""" +Configuración de plantilla CSV para Vehículos / Transportes (EstructuraCatTransportes.xls). +Mapeo: CLAVE → vehicle_key, CLAVE ACE → ace_vehicle_key, etc. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "vehicles": [ + {"canonical": "CLAVE", "aliases": ["CLAVE VEHICULO", "VEHICLE KEY", "KEY"]}, + {"canonical": "CLAVE ACE", "aliases": ["ACE", "CLAVEACE"]}, + {"canonical": "CLAVE TRANSPORTE", "aliases": ["TRANSPORTE", "TRANSPORTISTA"]}, + {"canonical": "VIN", "aliases": ["SERIE", "SERIES", "NUMERO SERIE"]}, + {"canonical": "TIPO TRANSPORTE", "aliases": ["TIPO", "TRANSPORT TYPE"]}, + {"canonical": "CODIGO DE ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE"]}, + {"canonical": "TRANSPONDEDOR", "aliases": ["TRANSPONDER", "TRANSPONDER NUMBER"]}, + {"canonical": "NUMERO DOT", "aliases": ["DOT", "NUM DOT"]}, + {"canonical": "PLACAS", "aliases": ["PLACA", "PLATE", "PLATE NUMBER"]}, + {"canonical": "CIUDAD", "aliases": ["CITY"]}, + {"canonical": "ESTADO", "aliases": ["STATE"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]}, + {"canonical": "PRECINTO", "aliases": ["SEAL"]}, + {"canonical": "EMPRESA ASEGURADORA", "aliases": ["ASEGURADORA", "INSURANCE COMPANY"]}, + {"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]}, + {"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]}, + {"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("vehicles") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/transportation/vehicles/routes.py b/backend/api/v1/modules/a76/transportation/vehicles/routes.py index 1afc20ed..ce823259 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/routes.py +++ b/backend/api/v1/modules/a76/transportation/vehicles/routes.py @@ -1,11 +1,19 @@ +from fastapi import APIRouter + from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO from .services import VehicleService +from .imports.routes import router as imports_router -# Create router using TenantCRUDRoutes factory -# Note: vehicle_key is a string (not int) and is used as the primary key -router = TenantCRUDRoutes( +# Main router: vehicles CRUD + CSV imports +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/vehicles/imports", tags=["a76 / vehicles / csv_import"]) + +# CRUD routes +crud_router = TenantCRUDRoutes( service=VehicleService, create_schema=VehicleCreateDTO, update_schema=VehicleUpdateDTO, @@ -13,10 +21,11 @@ router = TenantCRUDRoutes( prefix="/vehicles", tags=[], resource_name="Vehicle", - id_name="vehicle_key", # Using vehicle_key instead of numeric ID - id_type=str, # Specify that the ID is a string - enable_list=True, # Enable GET /vehicles with pagination - enable_filters=True, # Enable filtering by plate_number and transport_type + id_name="vehicle_key", + id_type=str, + enable_list=True, + enable_filters=True, default_page_size=50, max_page_size=100, ).router +router.include_router(crud_router) diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 9b292943..4af76815 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -1,6 +1,13 @@ import os from celery import Celery +# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen (mapper) +from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode +from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento +from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, +) + # Import models in correct order for SQLAlchemy relationship resolution # CRITICAL: FaLineItem must be imported BEFORE LineItem from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem # noqa: F401 @@ -26,6 +33,15 @@ celery_app.conf.update( "api.v1.modules.a76.reports.movements.invoices.tasks", "api.v1.modules.a76.reports.exportacion.descargo.task", "api.v1.modules.a76.imports.tasks", + "api.v1.modules.a76.customs_brokers.imports.tasks", + "api.v1.modules.a76.clients_and_providers.imports.tasks", + "api.v1.modules.a76.pedmientos.imports.tasks", + "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", + "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", + "api.v1.modules.a76.classes.imports.tasks", + "api.v1.modules.a76.parts.imports.tasks", + "api.v1.modules.a76.boms.imports.tasks", + "api.v1.modules.a76.transportation.vehicles.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/backend/core/error_handlers.py b/backend/core/error_handlers.py index d3ffc998..c3c4c90c 100644 --- a/backend/core/error_handlers.py +++ b/backend/core/error_handlers.py @@ -11,11 +11,23 @@ from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from .config import settings from .exceptions import BaseAPIException logger = logging.getLogger(__name__) +def _cors_headers(request: Request) -> Dict[str, str]: + """CORS headers for error responses so browser does not block on 4xx/5xx.""" + origin = request.headers.get("origin") + if not origin or origin not in settings.cors_origins_list: + return {} + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + } + + async def base_exception_handler( request: Request, exc: BaseAPIException, @@ -36,10 +48,13 @@ async def base_exception_handler( if hasattr(exc, "errors") and exc.errors: logger.warning(f"Validation errors details: {exc.errors}") - return JSONResponse( + response = JSONResponse( status_code=exc.status_code, content=jsonable_encoder(exc.to_dict()), ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def validation_exception_handler( @@ -65,7 +80,7 @@ async def validation_exception_handler( extra={"errors": errors}, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, content={ "error": "VALIDATION_ERROR", @@ -74,6 +89,9 @@ async def validation_exception_handler( "errors": errors, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def integrity_error_handler( @@ -105,7 +123,7 @@ async def integrity_error_handler( else: error_message = "Error de integridad en la base de datos" - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_409_CONFLICT, content={ "error": "DATABASE_INTEGRITY_ERROR", @@ -113,6 +131,9 @@ async def integrity_error_handler( "status_code": status.HTTP_409_CONFLICT, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def sqlalchemy_error_handler( @@ -131,7 +152,7 @@ async def sqlalchemy_error_handler( exc_info=True, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "error": "DATABASE_ERROR", @@ -139,6 +160,9 @@ async def sqlalchemy_error_handler( "status_code": status.HTTP_500_INTERNAL_SERVER_ERROR, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def http_exception_handler( @@ -174,7 +198,7 @@ async def general_exception_handler( exc_info=True, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "error": "INTERNAL_SERVER_ERROR", @@ -182,6 +206,9 @@ async def general_exception_handler( "status_code": status.HTTP_500_INTERNAL_SERVER_ERROR, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response def register_exception_handlers(app) -> None: diff --git a/backend/main.py b/backend/main.py index c8fffaa3..b50f92a0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -17,8 +17,13 @@ from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.material_types.models import MaterialType from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod +# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen para que +# SQLAlchemy resuelva los nombres en relationship() al configurar el mapper from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento +from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, +) from api.v1.modules.public.reference_data.sectors.models import Sector from api.v1.modules.public.reference_data.states.models import State from api.v1.modules.public.reference_data.transport_modes.models import TransportMode @@ -134,6 +139,20 @@ logger = logging.getLogger(__name__) register_exception_handlers(app) +def _cors_headers_for_request(request: Request): + """Return CORS headers if request Origin is allowed (so error responses don't get blocked by browser).""" + origin = request.headers.get("origin") + if not origin: + return {} + allowed = settings.cors_origins_list + if origin in allowed: + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + } + return {} + + # Add validation error handler @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): @@ -141,12 +160,29 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE f"Validation error for {request.method} {request.url.path}: {exc.errors()}" ) logger.error(f"Request body: {await request.body()}") - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_400_BAD_REQUEST, content={"detail": exc.errors(), "body": exc.body}, ) + for k, v in _cors_headers_for_request(request).items(): + response.headers[k] = v + return response +# Add HTTP exception handler +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + logger.error( + f"HTTP {exc.status_code} for {request.method} {request.url.path}: {exc.detail}" + ) + response = JSONResponse( + status_code=exc.status_code, + content={"detail": exc.detail}, + ) + for k, v in _cors_headers_for_request(request).items(): + response.headers[k] = v + return response + def run_migrations(): subprocess.run(["alembic", "upgrade", "head"], check=True) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9b422bfa..2c828760 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -173,10 +173,7 @@ services: - CORS_ORIGINS=${CORS_ORIGINS:-https://anexo76-dev.aduanasoft.com,http://localhost:3000} - SITAR_API_URL=${SITAR_API_URL} - SITAR_API_USER=${SITAR_API_USER} - - SITAR_API_PASSWORD=${SITAR_API_PASSWORD} - - EXTERNAL_API_URL=${EXTERNAL_API_URL} - - EXTERNAL_API_USER=${EXTERNAL_API_USER} - - EXTERNAL_API_PASSWORD=${EXTERNAL_API_PASSWORD} + - SITAR_API_PASSWORD=${SITAR_API_PASSWORD} - VALKEY_URL=${VALKEY_URL:-redis://valkey:6379/0} - CENTRAL_SERVER_URL=${CENTRAL_SERVER_URL:-""} - SYNC_SECRET_TOKEN=${SYNC_SECRET_TOKEN:-change-this-sync-token-in-production} diff --git a/docker-compose.yml b/docker-compose.yml index 5ae2e394..6a61948f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -178,9 +178,6 @@ services: - SITAR_API_URL=${SITAR_API_URL} - SITAR_API_USER=${SITAR_API_USER} - SITAR_API_PASSWORD=${SITAR_API_PASSWORD} - - EXTERNAL_API_URL=${EXTERNAL_API_URL} - - EXTERNAL_API_USER=${EXTERNAL_API_USER} - - EXTERNAL_API_PASSWORD=${EXTERNAL_API_PASSWORD} - VALKEY_URL=${VALKEY_URL:-redis://valkey:6379/0} - CENTRAL_SERVER_URL=${CENTRAL_SERVER_URL:-""} - SYNC_SECRET_TOKEN=${SYNC_SECRET_TOKEN:-change-this-sync-token-in-production} @@ -268,12 +265,7 @@ services: options: max-size: "10m" max-file: "3" - deploy: - resources: - limits: - memory: 1G - reservations: - memory: 512M + # celery celery_worker: diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 00000000..d02723d3 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12723 @@ + + + + + + Anexo76 - Gestión de Comercio Exterior + + + + + + +

Anexo76

Gestión de Comercio Exterior

Bienvenido a Anexo76

Plataforma SaaS para gestión de comercio exterior conforme a Anexos 24, 30 y 22 del SAT. + Ideal para maquilas, empresas IMMEX y agentes aduanales.

Registrarse

Características principales

Multi-tenant

Arquitectura híbrida con BD compartida o dedicada según necesidades

Seguridad

Autenticación con Keycloak y control de acceso basado en roles

Licencias

Planes flexibles desde Free hasta Enterprise con features personalizadas

© 2025 Anexo76. Desarrollado para la industria de comercio exterior mexicana.

+ + +
+ + diff --git a/frontend/package.json b/frontend/package.json index b21c873b..e7cd1887 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -64,5 +64,6 @@ "lucide-svelte": "^0.553.0", "marked": "^12.0.0", "svelte-sonner": "^1.0.7" - } -} \ No newline at end of file + }, + "packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017" +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index baa8f75c..a525d200 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -40,94 +40,49 @@ function onTokenRefreshed(token: string) { } /** - * Intenta refrescar el token usando el refresh token + * Refresca el token silenciosamente usando el endpoint server-side. + * + * El servidor lee el refresh_token desde la cookie HttpOnly, + * llama a Keycloak, actualiza las cookies y devuelve el nuevo access_token. + * El refresh_token NUNCA es leído por este código JavaScript. */ async function refreshToken(): Promise { if (!browser) return null; - let refreshTokenValue = localStorage.getItem('refresh_token'); - - // Si no está en localStorage, intentar obtenerlo de las cookies - if (!refreshTokenValue) { - const getCookie = (name: string): string | null => { - const value = `; ${document.cookie}`; - const parts = value.split(`; ${name}=`); - if (parts.length === 2) return parts.pop()?.split(';').shift() || null; - return null; - }; - - refreshTokenValue = getCookie('refresh_token'); - if (refreshTokenValue) { - localStorage.setItem('refresh_token', refreshTokenValue); - } - } - - if (!refreshTokenValue) { - console.error('❌ [API] No hay refresh token disponible'); - return null; - } - try { - const response = await fetch(`${API_BASE_URL}/v1/auth/refresh`, { + const response = await fetch('/api-sveltekit/auth/silent-refresh', { method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ refresh_token: refreshTokenValue }), - credentials: 'include' + credentials: 'include', // Envía cookies HttpOnly automáticamente + headers: { 'Content-Type': 'application/json' } }); if (!response.ok) { - console.error('❌ [API] Refresh token expirado o inválido, status:', response.status); - // Si el refresh token también está expirado, limpiar todo - localStorage.removeItem('access_token'); - localStorage.removeItem('refresh_token'); - // Limpiar cookies también + console.error('❌ [API] Silent refresh falló, status:', response.status); + // Limpiar la cookie del access_token (no HttpOnly) para forzar re-login document.cookie = 'access_token=; path=/; expires=Thu, 01 Jan 1970 00:00:00 UTC'; - document.cookie = 'refresh_token=; path=/; expires=Thu, 01 Jan 1970 00:00:00 UTC'; - // Redirigir al login después de un pequeño delay para que el usuario vea el mensaje - setTimeout(() => { - if (browser) { - window.location.href = '/login'; - } - }, 2000); + setTimeout(() => { window.location.href = '/login'; }, 1500); return null; } - const data = await response.json(); + const data = await response.json() as { access_token?: string }; - // Guardar los nuevos tokens if (data.access_token) { - localStorage.setItem('access_token', data.access_token); + // Actualizar cookie no-HttpOnly del access_token + const secure = window.location.protocol === 'https:' ? '; Secure' : ''; + document.cookie = `access_token=${data.access_token}; path=/; max-age=${60 * 60 * 24 * 7}; SameSite=Lax${secure}`; - if (data.refresh_token) { - localStorage.setItem('refresh_token', data.refresh_token); - } - - // Actualizar también las cookies - const isSecure = window.location.protocol === 'https:'; - const secureFlag = isSecure ? '; Secure' : ''; - - document.cookie = `access_token=${data.access_token}; path=/; max-age=${60 * 60 * 24 * 7}; SameSite=Lax${secureFlag}`; - if (data.refresh_token) { - document.cookie = `refresh_token=${data.refresh_token}; path=/; max-age=${60 * 60 * 24 * 30}; SameSite=Lax${secureFlag}`; - } - - // Actualizar el authStore si está disponible + // Actualizar authStore en memoria try { const { authStore } = await import('./auth'); authStore.setToken(data.access_token); - } catch (e) { - // Si no se puede importar authStore, no es crítico - console.warn('⚠️ [API] No se pudo actualizar authStore:', e); - } + } catch {} return data.access_token; } return null; } catch (error) { - console.error('❌ [API] Error refreshing token:', error); + console.error('❌ [API] Error en silent refresh:', error); return null; } } @@ -315,6 +270,35 @@ export const api = { delete: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, { method: 'DELETE', ...options }), + /** + * Download CSV template by template_id (generated from code, no static file). + * Returns blob and suggested filename for the browser download. + */ + async getCsvTemplateDownload( + templateId: string + ): Promise<{ blob: Blob; filename: string }> { + const token = getToken(); + const headers: Record = {}; + if (token) headers['Authorization'] = `Bearer ${token}`; + const response = await fetch(`${API_BASE_URL}/v1/a76/csv-templates/${templateId}`, { + method: 'GET', + headers, + credentials: 'include' + }); + if (!response.ok) { + const msg = response.status === 404 ? 'Plantilla no encontrada' : `Error ${response.status}`; + throw new Error(msg); + } + const blob = await response.blob(); + let filename = `plantilla_${templateId}.csv`; + const disposition = response.headers.get('Content-Disposition'); + if (disposition) { + const match = /filename="?([^";\n]+)"?/.exec(disposition); + if (match) filename = match[1].trim(); + } + return { blob, filename }; + }, + // Endpoints específicos auth: { login: (credentials: { username: string; password: string; tenant_slug: string }) => @@ -341,6 +325,215 @@ export const api = { validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`) }, + imports: { + upload: ( + file: File, + modelTarget: string, + footerConfig: any, + companyId: number, + operationType: string, + templateId?: string + ) => { + const formData = new FormData(); + formData.append('file', file); + if (footerConfig) { + formData.append('footer_config', JSON.stringify(footerConfig)); + } + if (templateId) { + formData.append('template_id', templateId); + } + + const queryParams = new URLSearchParams({ + company_id: String(companyId), + operation_type: operationType || 'imp' + }).toString(); + + return fetchApi(`/v1/a76/imports/upload/${modelTarget}?${queryParams}`, { + method: 'POST', + body: formData + }); + }, + status: (jobId: string) => api.get(`/v1/a76/imports/${jobId}/status`), + commit: (jobId: string, modelTarget: string) => + api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget }) + }, + + // CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports) + customsBrokerImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/customs-brokers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/customs-brokers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {}) + }, + + // CSV import for Clientes y Proveedores (flujo propio en clients_and_providers/imports) + clientProviderImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/clients-providers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/clients-providers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {}) + }, + + // CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports) + exchangeRateImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/exchange-rate/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {}) + }, + + // CSV import for Fracción Americana (us_tariff_fractions/imports) + americanFractionImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/us-tariff-fractions/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/us-tariff-fractions/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {}) + }, + + // CSV import for Pedimentos (pedimentos/imports) + pedimentosImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/pedimentos/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/pedimentos/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {}) + }, + + // CSV import for Clases de Materiales (classes/imports) + materialClassImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/classes/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/classes/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/classes/imports/${jobId}/commit`, {}) + }, + + // CSV import for Vehículos / Transportes (transportation/vehicles/imports) + vehicleImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/transportation/vehicles/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/transportation/vehicles/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/transportation/vehicles/imports/${jobId}/commit`, {}) + }, + + // CSV import for Conductores (drivers/imports) + driverImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/drivers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/drivers/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/drivers/imports/${jobId}/commit`, {}) + }, + + // CSV import for Trailers y Cajas (transportation/trailers/imports) + trailerImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/transportation/trailers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/transportation/trailers/imports/${jobId}/commit`, {}) + }, + + // CSV import for Transportistas (transporters/imports) + transporterImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/transporters/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/transporters/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/transporters/imports/${jobId}/commit`, {}) + }, + + // CSV import for Números de parte (parts/imports) + partNumberImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/parts/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/parts/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/parts/imports/${jobId}/commit`, {}) + }, + + // CSV import for BOMs (boms/imports) + bomImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/boms/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/boms/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/boms/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/lib/auth.ts b/frontend/src/lib/auth.ts index e75c29ff..f35153e4 100644 --- a/frontend/src/lib/auth.ts +++ b/frontend/src/lib/auth.ts @@ -1,11 +1,21 @@ /** * Servicio de autenticación con Keycloak + * + * Seguridad de tokens: + * - access_token → en memoria (authStore) + cookie no-HttpOnly (para SSR) + * - refresh_token → cookie HttpOnly únicamente (JS nunca lo lee directamente) + * - El refresh se hace server-side via /api-sveltekit/auth/silent-refresh + * - NO se usa localStorage para tokens */ + import Keycloak from 'keycloak-js'; import { writable, derived } from 'svelte/store'; import { browser } from '$app/environment'; +// ───────────────────────────────────────────────────────── // Tipos +// ───────────────────────────────────────────────────────── + export interface User { id: string; username: string; @@ -22,51 +32,51 @@ export interface AuthState { token: string | null; } +// ───────────────────────────────────────────────────────── // Configuración de Keycloak +// ───────────────────────────────────────────────────────── + const keycloakConfig = { url: import.meta.env.VITE_KEYCLOAK_URL, realm: import.meta.env.VITE_KEYCLOAK_REALM, clientId: import.meta.env.VITE_KEYCLOAK_CLIENT_ID }; -// Instancia de Keycloak let keycloakInstance: Keycloak | null = null; -// Helper para obtener cookies +// ───────────────────────────────────────────────────────── +// Cookie helpers (solo para access_token no-HttpOnly) +// ───────────────────────────────────────────────────────── + +/** Lee el valor de una cookie no-HttpOnly */ const getCookie = (name: string): string | null => { if (!browser) return null; const value = `; ${document.cookie}`; const parts = value.split(`; ${name}=`); - if (parts.length === 2) return parts.pop()?.split(';').shift() || null; + if (parts.length === 2) return parts.pop()?.split(';').shift() ?? null; return null; }; -// Helper para establecer cookies con las opciones correctas según el entorno +/** Escribe una cookie no-HttpOnly */ const setCookie = (name: string, value: string, days: number = 7) => { if (!browser) return; - const expirationDate = new Date(); - expirationDate.setDate(expirationDate.getDate() + days); - - // En desarrollo (localhost), no usar Secure flag - const isSecure = window.location.protocol === 'https:'; - const secureFlag = isSecure ? '; Secure' : ''; - - const cookieString = `${name}=${value}; path=/; expires=${expirationDate.toUTCString()}; SameSite=Lax${secureFlag}`; - document.cookie = cookieString; - - // Verificar que se estableció - const verification = getCookie(name); + const exp = new Date(); + exp.setDate(exp.getDate() + days); + const secure = window.location.protocol === 'https:' ? '; Secure' : ''; + document.cookie = `${name}=${value}; path=/; expires=${exp.toUTCString()}; SameSite=Lax${secure}`; }; -// Helper para eliminar cookies +/** Elimina una cookie */ const deleteCookie = (name: string) => { if (!browser) return; - const isSecure = window.location.protocol === 'https:'; - const secureFlag = isSecure ? '; Secure' : ''; - document.cookie = `${name}=; path=/; expires=Thu, 01 Jan 1970 00:00:00 UTC; SameSite=Lax${secureFlag}`; + const secure = window.location.protocol === 'https:' ? '; Secure' : ''; + document.cookie = `${name}=; path=/; expires=Thu, 01 Jan 1970 00:00:00 UTC; SameSite=Lax${secure}`; }; -// Store de autenticación +// ───────────────────────────────────────────────────────── +// Auth store (tokens solo en memoria) +// ───────────────────────────────────────────────────────── + const createAuthStore = () => { const { subscribe, set, update } = writable({ isAuthenticated: false, @@ -78,16 +88,16 @@ const createAuthStore = () => { return { subscribe, setAuthenticated: (authenticated: boolean) => - update((state) => ({ ...state, isAuthenticated: authenticated })), + update((s) => ({ ...s, isAuthenticated: authenticated })), setLoading: (loading: boolean) => - update((state) => ({ ...state, isLoading: loading })), - setUser: (user: User | null) => update((state) => ({ ...state, user })), - setToken: (token: string | null) => update((state) => ({ ...state, token })), - setTokens: (accessToken: string, refreshToken?: string) => { - update((state) => ({ ...state, token: accessToken })); - if (browser && refreshToken) { - localStorage.setItem('refresh_token', refreshToken); - } + update((s) => ({ ...s, isLoading: loading })), + setUser: (user: User | null) => update((s) => ({ ...s, user })), + setToken: (token: string | null) => update((s) => ({ ...s, token })), + /** ⚠️ Los tokens ya NO se guardan en localStorage; solo en memoria. */ + setTokens: (accessToken: string, _refreshToken?: string) => { + update((s) => ({ ...s, token: accessToken })); + // El refresh_token llega en cookie HttpOnly desde el servidor; + // el cliente no lo almacena ni lo lee en ningún momento. }, reset: () => set({ @@ -101,14 +111,17 @@ const createAuthStore = () => { export const authStore = createAuthStore(); -// Derived store para verificar si está autenticado -export const isAuthenticated = derived(authStore, ($auth) => $auth.isAuthenticated); +export const isAuthenticated = derived(authStore, ($a) => $a.isAuthenticated); +export const currentUser = derived(authStore, ($a) => $a.user); -// Derived store para obtener el usuario -export const currentUser = derived(authStore, ($auth) => $auth.user); +// ───────────────────────────────────────────────────────── +// Inicialización +// ───────────────────────────────────────────────────────── /** - * Inicializa la autenticación (Keycloak o token-based) + * Inicializa el estado de autenticación en el cliente. + * - Si hay un access_token en la cookie no-HttpOnly, lo usa. + * - En cualquier caso intenta inicializar Keycloak JS (para el flujo SSO). */ export const initAuth = async (): Promise => { if (!browser) return false; @@ -116,38 +129,28 @@ export const initAuth = async (): Promise => { try { authStore.setLoading(true); - // Primero intentar restaurar sesión desde localStorage - const token = localStorage.getItem('access_token'); - if (token) { - authStore.setToken(token); + // Restaurar token desde cookie no-HttpOnly (password login flow) + const cookieToken = getCookie('access_token'); + if (cookieToken) { + authStore.setToken(cookieToken); authStore.setAuthenticated(true); - - // Sincronizar con cookies si no existe - const cookieToken = getCookie('access_token'); - if (!cookieToken) { - setCookie('access_token', token); - } - - await loadUserInfo(token); + await loadUserInfo(cookieToken).catch(() => {}); authStore.setLoading(false); return true; } - // Si no hay token local, intentar con Keycloak - await initKeycloak(); - + // Sin token local, intentar Keycloak JS (flujo SSO) + const authenticated = await initKeycloak(); authStore.setLoading(false); - return false; - } catch (error) { - console.error('Error inicializando autenticación:', error); + return authenticated; + } catch (err) { + console.error('[auth] Error en initAuth:', err); authStore.setLoading(false); return false; } }; -/** - * Inicializa Keycloak - */ +/** Inicializa Keycloak JS para el flujo SSO con PKCE */ export const initKeycloak = async (): Promise => { if (!browser) return false; @@ -163,22 +166,18 @@ export const initKeycloak = async (): Promise => { if (authenticated) { await updateAuthState(); - setupTokenRefresh(); + setupKeycloakTokenHooks(); } return authenticated; - } catch (error) { - console.error('Error inicializando Keycloak:', error); + } catch (err) { + console.error('[auth] Error inicializando Keycloak:', err); return false; } }; -// Variable para rastrear el tenant anterior let previousTenantId: number | undefined = undefined; -/** - * Actualiza el estado de autenticación con los datos de Keycloak - */ const updateAuthState = async () => { if (!keycloakInstance?.authenticated) { authStore.reset(); @@ -187,22 +186,22 @@ const updateAuthState = async () => { try { const profile = await keycloakInstance.loadUserProfile(); - const token = keycloakInstance.token || null; - const tokenParsed = keycloakInstance.tokenParsed as any; + const token = keycloakInstance.token ?? null; + const parsed = keycloakInstance.tokenParsed as any; - const roles = tokenParsed?.realm_access?.roles || []; - const tenantId = tokenParsed?.tenant_id || tokenParsed?.attributes?.tenant_id; - const newTenantId = tenantId ? parseInt(tenantId) : undefined; + const roles: string[] = parsed?.realm_access?.roles ?? []; + const tenantId: number | undefined = parsed?.tenant_id + ? parseInt(parsed.tenant_id) + : undefined; - // Detectar si cambió el tenant - const tenantChanged = previousTenantId !== undefined && previousTenantId !== newTenantId; + const tenantChanged = previousTenantId !== undefined && previousTenantId !== tenantId; const user: User = { - id: profile.id || '', - username: profile.username || '', + id: profile.id ?? '', + username: profile.username ?? '', email: profile.email, - name: `${profile.firstName || ''} ${profile.lastName || ''}`.trim(), - tenantId: newTenantId, + name: `${profile.firstName ?? ''} ${profile.lastName ?? ''}`.trim(), + tenantId, roles }; @@ -210,71 +209,83 @@ const updateAuthState = async () => { authStore.setUser(user); authStore.setToken(token); - // Si cambió el tenant, limpiar el store de compañías if (tenantChanged && browser) { try { const { companyStore } = await import('./stores/company.svelte'); companyStore.clear(); - } catch (error) { - console.error('Error al limpiar store de compañías:', error); - } + } catch {} } - // Actualizar el tenant anterior - previousTenantId = newTenantId; - } catch (error) { - console.error('Error actualizando estado de autenticación:', error); + previousTenantId = tenantId; + } catch (err) { + console.error('[auth] Error actualizando estado:', err); authStore.reset(); } }; +// ───────────────────────────────────────────────────────── +// Keycloak JS token hooks (solo para el flujo SSO) +// ───────────────────────────────────────────────────────── + /** - * Configura el refresh automático del token + * Configura los callbacks de Keycloak JS para notificar al SessionManager + * sobre cambios de token y eventos de sesión SSO. */ -const setupTokenRefresh = () => { +const setupKeycloakTokenHooks = () => { if (!keycloakInstance) return; - // Refrescar token cada 60 segundos si está cerca de expirar keycloakInstance.onTokenExpired = () => { keycloakInstance ?.updateToken(70) .then((refreshed) => { - if (refreshed) { - authStore.setToken(keycloakInstance?.token || null); + if (refreshed && keycloakInstance?.token) { + authStore.setToken(keycloakInstance.token); + import('./session-manager') + .then(({ getSessionManager }) => { + getSessionManager()?.updateToken(keycloakInstance!.token!); + }) + .catch(() => {}); } }) .catch(() => { - console.error('Error refrescando token'); - logout(); + console.error('[auth] No se pudo refrescar el token de Keycloak'); + void logout(); }); }; + + keycloakInstance.onAuthRefreshSuccess = () => { + if (keycloakInstance?.token) authStore.setToken(keycloakInstance.token); + }; + + keycloakInstance.onAuthRefreshError = () => { + console.error('[auth] Error en refresh de Keycloak — cerrando sesión'); + void logout(); + }; + + keycloakInstance.onAuthLogout = () => { + authStore.reset(); + }; }; -/** - * Inicia sesión con Keycloak (OAuth flow) - */ +// ───────────────────────────────────────────────────────── +// Login +// ───────────────────────────────────────────────────────── + +/** Inicia sesión con Keycloak (OAuth redirect flow) */ export const loginWithKeycloak = async (tenantSlug?: string) => { if (!keycloakInstance) { - console.error('Keycloak no está inicializado'); + console.error('[auth] Keycloak no está inicializado'); return; } - - const options: any = { - redirectUri: window.location.origin + '/callback' - }; - - if (tenantSlug) { - options.loginHint = tenantSlug; - } - + const options: any = { redirectUri: window.location.origin + '/callback' }; + if (tenantSlug) options.loginHint = tenantSlug; await keycloakInstance.login(options); }; /** - * Inicia sesión con credenciales (username/password) - * Nota: Esta función ya no se usa directamente desde el login form, - * el login ahora se hace mediante form actions del servidor. - * Se mantiene para compatibilidad con SSO y otros flujos. + * Login con usuario/contraseña (legacy — el login principal es via form action del servidor). + * Los tokens se guardan en cookies (vía setCookie) y en memoria (authStore). + * NO se guardan en localStorage. */ export const login = async (credentials: { username: string; @@ -282,235 +293,185 @@ export const login = async (credentials: { tenant_slug: string; }): Promise<{ success: boolean; error?: string; data?: any }> => { try { - // Usar la API centralizada const { api } = await import('./api'); const response = await api.auth.login(credentials); - // Si hay error en la respuesta if (response.error) { - return { - success: false, - error: response.error - }; + return { success: false, error: response.error }; } - // Guardar tokens y actualizar estado const loginData = response.data; if (loginData?.access_token) { + // Guardar en memoria y en cookie no-HttpOnly para SSR authStore.setToken(loginData.access_token); authStore.setAuthenticated(true); - - // Guardar también en localStorage para persistencia - if (browser) { - localStorage.setItem('access_token', loginData.access_token); - if (loginData.refresh_token) { - localStorage.setItem('refresh_token', loginData.refresh_token); - } - - // Guardar en cookies para que el servidor pueda acceder - setCookie('access_token', loginData.access_token); - if (loginData.refresh_token) { - setCookie('refresh_token', loginData.refresh_token); - } - } - - // Cargar información del usuario + setCookie('access_token', loginData.access_token); + // El refresh_token llega en cookie HttpOnly desde el servidor. + // NO lo guardamos en JS. await loadUserInfo(loginData.access_token); } - return { - success: true, - data: loginData - }; - } catch (error) { - console.error('Error en login:', error); - return { - success: false, - error: 'Error de conexión con el servidor' - }; + return { success: true, data: loginData }; + } catch (err) { + console.error('[auth] Error en login:', err); + return { success: false, error: 'Error de conexión con el servidor' }; } }; -/** - * Carga la información del usuario desde el token - */ +// ───────────────────────────────────────────────────────── +// User info +// ───────────────────────────────────────────────────────── + const loadUserInfo = async (token: string) => { try { - // Guardar temporalmente el token para que api.ts lo use authStore.setToken(token); - - // Usar la API centralizada const { api } = await import('./api'); const response = await api.auth.me(); - if (response.data) { - const data = response.data; - const user: User = { - id: data.sub || '', - username: data.preferred_username || data.username || '', - email: data.email, - name: data.name, - tenantId: data.tenant_id, - roles: data.realm_access?.roles || [] - }; - authStore.setUser(user); + const d = response.data; + authStore.setUser({ + id: d.sub ?? '', + username: d.preferred_username ?? d.username ?? '', + email: d.email, + name: d.name, + tenantId: d.tenant_id, + roles: d.realm_access?.roles ?? [] + }); } - } catch (error) { - console.error('Error cargando información del usuario:', error); + } catch (err) { + console.error('[auth] Error cargando info del usuario:', err); } }; -/** - * Cierra sesión - */ +// ───────────────────────────────────────────────────────── +// Logout +// ───────────────────────────────────────────────────────── + export const logout = async () => { if (!browser) return; try { - // Capturar tokens antes de limpiar nada - const refreshToken = localStorage.getItem('refresh_token'); - const accessToken = localStorage.getItem('access_token'); + // Detener el SessionManager + try { + const { destroySessionManager } = await import('./session-manager'); + destroySessionManager(); + } catch {} // Limpiar store de compañías try { const { companyStore } = await import('./stores/company.svelte'); companyStore.clear(); - } catch (error) { - console.error('Error al limpiar store de compañías:', error); - } + } catch {} - // Limpiar estado local + // Limpiar estado en memoria authStore.reset(); - localStorage.removeItem('access_token'); - localStorage.removeItem('refresh_token'); + + // Eliminar cookie no-HttpOnly del access_token deleteCookie('access_token'); - deleteCookie('refresh_token'); + // La cookie HttpOnly del refresh_token la limpia el servidor - // Si hay instancia de Keycloak, hacer logout de Keycloak + // Logout de Keycloak JS si estaba autenticado con SSO if (keycloakInstance?.authenticated) { - // Primero notificamos al servidor para limpieza de cookies (SvelteKit) try { - await fetch('/logout', { - method: 'POST' - }); - } catch (e) { - console.error("Error calling server logout:", e); - } - + await fetch('/logout', { method: 'POST' }); + } catch {} await keycloakInstance.logout({ redirectUri: window.location.origin + '/login' }); return; } - // Llamar al endpoint del servidor para limpiar cookies de SvelteKit - // Usar un formulario para hacer POST y permitir la redirección + // Para login con password: POST al logout route del servidor const form = document.createElement('form'); form.method = 'POST'; form.action = '/logout'; - - - document.body.appendChild(form); form.submit(); - - } catch (error) { - console.error('Error durante logout:', error); - // Asegurar que se redirija al login aunque haya error + } catch (err) { + console.error('[auth] Error durante logout:', err); window.location.href = '/login'; } }; -/** - * Verifica si el usuario tiene un rol específico - */ +// ───────────────────────────────────────────────────────── +// Token accessors +// ───────────────────────────────────────────────────────── + export const hasRole = (role: string): boolean => { if (!keycloakInstance?.authenticated) return false; return keycloakInstance.hasRealmRole(role); }; -/** - * Obtiene el token de acceso actual - */ +/** Obtiene el access token desde memoria (Keycloak JS o authStore) */ export const getToken = (): string | null => { - // Intentar obtener de Keycloak primero - if (keycloakInstance?.token) { - return keycloakInstance.token; - } + // Prioridad 1: Keycloak JS en memoria + if (keycloakInstance?.token) return keycloakInstance.token; - // Si no, intentar de localStorage - if (browser) { - let token = localStorage.getItem('access_token'); + // Prioridad 2: authStore en memoria + let token: string | null = null; + const unsub = authStore.subscribe((s) => { token = s.token; }); + unsub(); + if (token) return token; - // Si no hay token en localStorage, intentar de las cookies - if (!token) { - token = getCookie('access_token'); - // Si lo encontramos en cookies, sincronizarlo a localStorage - if (token) { - localStorage.setItem('access_token', token); - } - } - - return token; - } + // Prioridad 3: cookie no-HttpOnly (fallback para acceso inicial antes del onMount) + if (browser) return getCookie('access_token'); return null; }; /** - * Refresca el access token usando el refresh token + * Refresca el access token usando el endpoint server-side seguro. + * El servidor lee el refresh_token de la cookie HttpOnly. + * @returns true si el refresh fue exitoso */ export const refreshAccessToken = async (): Promise => { if (!browser) return false; - const refreshToken = localStorage.getItem('refresh_token'); - if (!refreshToken) { - return false; + // Con Keycloak JS activo, usar su mecanismo nativo + if (keycloakInstance?.authenticated) { + try { + const refreshed = await keycloakInstance.updateToken(70); + if (refreshed || keycloakInstance.token) { + authStore.setToken(keycloakInstance.token ?? null); + return true; + } + } catch { + await logout(); + return false; + } } + // Flujo de contraseña: usar el endpoint server-side seguro try { - const { api } = await import('./api'); - const response = await api.auth.refresh(refreshToken); + const resp = await fetch('/api-sveltekit/auth/silent-refresh', { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' } + }); - if (response.error || !response.data) { - console.error('Failed to refresh token:', response.error); - // Si falla el refresh, hacer logout + if (!resp.ok) { await logout(); return false; } - // Actualizar tokens - const newAccessToken = response.data.access_token; - const newRefreshToken = response.data.refresh_token; - - authStore.setToken(newAccessToken); - localStorage.setItem('access_token', newAccessToken); - - if (newRefreshToken) { - localStorage.setItem('refresh_token', newRefreshToken); + const data = await resp.json() as { access_token?: string }; + if (data.access_token) { + authStore.setToken(data.access_token); + setCookie('access_token', data.access_token); + return true; } - - // Actualizar también la cookie - setCookie('access_token', newAccessToken); - return true; - - } catch (error) { - await logout(); - return false; + } catch (err) { + console.error('[auth] Error en refreshAccessToken:', err); } + + await logout(); + return false; }; -/** - * Obtiene el refresh token - */ +/** @deprecated El refresh_token ya no se expone en JS. */ export const getRefreshToken = (): string | null => { - if (!browser) return null; - return localStorage.getItem('refresh_token'); + console.warn('[auth] getRefreshToken() está deprecado — el refresh_token no se expone en JS.'); + return null; }; -/** - * Obtiene la instancia de Keycloak - */ -export const getKeycloakInstance = (): Keycloak | null => { - return keycloakInstance; -}; +export const getKeycloakInstance = (): Keycloak | null => keycloakInstance; diff --git a/frontend/src/lib/components/dashboard/PrerequisitesModal.svelte b/frontend/src/lib/components/dashboard/PrerequisitesModal.svelte new file mode 100644 index 00000000..57cb8128 --- /dev/null +++ b/frontend/src/lib/components/dashboard/PrerequisitesModal.svelte @@ -0,0 +1,74 @@ + + + + + +
+ + Aviso +
+ +

{message}

+

+ Puedes registrarlos en + + Agentes Aduanales + + y + + Clientes y Proveedores + . +

+ + + + Cancelar + Aceptar + + + diff --git a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte index 7375b3ab..bd7bac46 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte @@ -143,17 +143,53 @@ {#if scanResults.error_count > 0}

Se detectaron problemas en el archivo

- Las filas con errores serán omitidas automáticamente. Solo se importarán los - registros válidos. + Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para + importar solo las filas válidas (las erróneas se omitirán).

+ {#if scanResults.errors && scanResults.errors.length > 0} +
+
+
+ Detalle de errores (para corregir en el CSV) +
+ + {scanResults.errors.length} error(es) + +
+
+ + + + + + + + + + {#each scanResults.errors as err} + + + + + + {/each} + +
LíneaColumnaMensaje
{err.line}{err.col || '-'}{err.msg || '-'}
+
+
+ {/if} {:else}
diff --git a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte index 8084f135..35d1f218 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte @@ -4,6 +4,7 @@ import { UploadCloud, Lock } from 'lucide-svelte'; import { cn } from '$lib/utils'; import { toast } from 'svelte-sonner'; + import { api } from '$lib/api'; let { items, @@ -88,23 +89,31 @@ } } - function handleContextMenu(e: MouseEvent, item: CsvUploadItem) { + async function handleContextMenu(e: MouseEvent, item: CsvUploadItem) { if (item.disabled) { e.preventDefault(); return; } - if (!item.templateUrl) return; e.preventDefault(); - const link = document.createElement('a'); - link.href = item.templateUrl; - link.download = item.templateUrl.split('/').pop() || 'plantilla.xls'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); + if (!item.templateId) return; - toast.info(`Descargando plantilla para ${item.title}...`); + try { + toast.info(`Descargando plantilla para ${item.title}...`); + const { blob, filename } = await api.getCsvTemplateDownload(item.templateId); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); + toast.success(`Plantilla descargada: ${filename}`); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Error al descargar la plantilla'); + } } @@ -123,7 +132,7 @@ ondragover={(e) => handleDragOver(e, item.disabled)} ondrop={(e) => handleDrop(e, item)} oncontextmenu={(e) => handleContextMenu(e, item)} - roles="button" + role="button" tabindex={item.disabled ? -1 : 0} onclick={() => handleClick(item.id, item.disabled)} onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)} diff --git a/frontend/src/lib/components/dashboard/customs_brokers/columns.ts b/frontend/src/lib/components/dashboard/customs_brokers/columns.ts index 3ea92e34..add7af3f 100644 --- a/frontend/src/lib/components/dashboard/customs_brokers/columns.ts +++ b/frontend/src/lib/components/dashboard/customs_brokers/columns.ts @@ -8,6 +8,20 @@ export type { CustomsBroker }; export function createColumns(onSuccess?: () => void): ColumnDef[] { return [ + { + accessorKey: "id", + header: "ID", + cell: ({ row }) => { + const idSnippet = createRawSnippet<[{ id: number }]>((getId) => { + const { id } = getId(); + return { + render: () => + `${id}` + }; + }); + return renderSnippet(idSnippet, { id: row.original.id }); + } + }, { accessorKey: "broker_key", header: "Clave", diff --git a/frontend/src/lib/components/dashboard/customs_brokers/create-dialog.svelte b/frontend/src/lib/components/dashboard/customs_brokers/create-dialog.svelte index 44bca734..672f019a 100644 --- a/frontend/src/lib/components/dashboard/customs_brokers/create-dialog.svelte +++ b/frontend/src/lib/components/dashboard/customs_brokers/create-dialog.svelte @@ -275,7 +275,7 @@
- + { formData.postal_code = e.currentTarget.value.replace(/[^a-zA-Z0-9]/g, ''); }} />
diff --git a/frontend/src/lib/components/dashboard/customs_brokers/edit-dialog.svelte b/frontend/src/lib/components/dashboard/customs_brokers/edit-dialog.svelte index 147df888..294c7515 100644 --- a/frontend/src/lib/components/dashboard/customs_brokers/edit-dialog.svelte +++ b/frontend/src/lib/components/dashboard/customs_brokers/edit-dialog.svelte @@ -318,6 +318,7 @@ placeholder="C.P." maxlength={15} disabled={loading} + oninput={(e) => { formData.postal_code = e.currentTarget.value.replace(/[^a-zA-Z0-9]/g, ''); }} />
diff --git a/frontend/src/lib/components/dashboard/exchange_rate/create-edit-dialog.svelte b/frontend/src/lib/components/dashboard/exchange_rate/create-edit-dialog.svelte index f19312c0..de77a2d1 100644 --- a/frontend/src/lib/components/dashboard/exchange_rate/create-edit-dialog.svelte +++ b/frontend/src/lib/components/dashboard/exchange_rate/create-edit-dialog.svelte @@ -119,7 +119,14 @@ const companyId = companyStore.activeCompany?.id; if (!companyId) throw new Error('No hay una compañía seleccionada'); if (!formData.date) throw new Error('La fecha es requerida'); - if (formData.value === null) throw new Error('El valor es requerido'); + if ( + formData.value === null || + formData.value === undefined || + String(formData.value).trim() === '' + ) + throw new Error('El tipo de cambio es requerido'); + if (Number(formData.value) <= 0) + throw new Error('El tipo de cambio debe ser un valor mayor a 0'); showConfirmation = true; } catch (e) { diff --git a/frontend/src/lib/components/dashboard/invoices/create-edit-dialog.svelte b/frontend/src/lib/components/dashboard/invoices/create-edit-dialog.svelte index c6a4e708..4237c5e9 100644 --- a/frontend/src/lib/components/dashboard/invoices/create-edit-dialog.svelte +++ b/frontend/src/lib/components/dashboard/invoices/create-edit-dialog.svelte @@ -181,6 +181,20 @@ return; } + // Validar tipo de cambio + if ( + formData.exchange_rate === null || + formData.exchange_rate === undefined || + String(formData.exchange_rate).trim() === '' + ) { + error = 'El tipo de cambio es requerido (pestaña Financieros)'; + return; + } + if (Number(formData.exchange_rate) <= 0) { + error = 'El tipo de cambio debe ser mayor a 0 (pestaña Financieros)'; + return; + } + loading = true; error = null; diff --git a/frontend/src/lib/components/help/HelpDrawer.svelte b/frontend/src/lib/components/help/HelpDrawer.svelte index 6a557aed..e2de2644 100644 --- a/frontend/src/lib/components/help/HelpDrawer.svelte +++ b/frontend/src/lib/components/help/HelpDrawer.svelte @@ -121,7 +121,7 @@ - + + + + + + diff --git a/frontend/src/lib/components/sidebar/nav-main.svelte b/frontend/src/lib/components/sidebar/nav-main.svelte index e21cdf95..ff5a24f7 100644 --- a/frontend/src/lib/components/sidebar/nav-main.svelte +++ b/frontend/src/lib/components/sidebar/nav-main.svelte @@ -101,7 +101,7 @@ {#if item.icon} {:else} -
+
{/if} {item.title} @@ -127,7 +127,7 @@ {#if item.icon} {:else} -
+
{/if}
diff --git a/frontend/src/lib/components/ui/select/select-content.svelte b/frontend/src/lib/components/ui/select/select-content.svelte index 8293adf1..9bed7978 100644 --- a/frontend/src/lib/components/ui/select/select-content.svelte +++ b/frontend/src/lib/components/ui/select/select-content.svelte @@ -85,7 +85,6 @@ bind:this={searchInputRef} value={searchQuery} placeholder={searchPlaceholder} - autofocus={autoFocusSearch} class="placeholder:text-muted-foreground flex h-8 w-full rounded-md border border-input bg-background px-3 py-1 text-sm outline-none focus:border-ring focus:ring-1 focus:ring-ring" oninput={(event) => updateQuery((event.currentTarget as HTMLInputElement).value)} onclick={(e) => e.stopPropagation()} diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index 656b0459..c51ceaae 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -1,366 +1,388 @@ -import { - User, - Users, - FileText, - Truck, - Container, - Ship, - Plane, - Package, - Briefcase, - Globe, - CreditCard, - DollarSign, - Calendar, - Hash, - MapPin, - ShieldCheck, - FileDigit, - Scale, -} from 'lucide-svelte'; - -// --- Interfaces --- - -export interface CsvUploadItem { - id: string; - title: string; - icon: any; - group?: string; // For grouping within a tab - modelTarget?: string; // The backend model this maps to - description?: string; - templateUrl?: string; // Path to the template file in static/ - disabled?: boolean; // New property to mark items as "Coming Soon" -} - -export interface CsvUploadField { - name: string; - label: string; - type: 'text' | 'select' | 'boolean' | 'date' | 'radio'; - options?: { label: string; value: string | boolean | number }[]; - required?: boolean; - defaultValue?: any; -} - -// Map of Tab ID -> Array of Fields -export const tabSettings: Record = { - catalogos: [ - { - name: 'mode', - label: 'Modo de Carga', - type: 'radio', - options: [ - { label: 'Actualizar', value: 'update' }, - { label: 'Reemplazar', value: 'replace' } - ], - defaultValue: 'update' - } - ], - transportes: [ - { - name: 'mode', - label: 'Modo de Carga', - type: 'radio', - options: [ - { label: 'Actualizar', value: 'update' }, - { label: 'Reemplazar', value: 'replace' } - ], - defaultValue: 'update' - } - ], - importacion: [ - { - name: 'autonumber_remesas', - label: 'Autonumerar Remesas', - type: 'boolean', - defaultValue: false - }, - { - name: 'recalculate_dates', - label: 'Recalcular Fechas', - type: 'boolean', - defaultValue: false - }, - { - name: 'dateFormat', - label: 'Formato de Fecha', - type: 'select', - options: [ - { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, - { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, - { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } - ], - defaultValue: 'dd/mm/yyyy' - } - ], - exportacion: [ - { - name: 'invoice_type', - label: 'Tipo de Factura', - type: 'select', - options: [ - { label: 'AFIJO', value: 'AFIJO' }, - { label: 'NORMAL', value: 'NORMAL' }, - ], - defaultValue: 'AFIJO', - }, - { - name: 'is_regime_change', - label: 'Es Cambio de Régimen', - type: 'boolean', - defaultValue: false, - }, - { - name: 'dateFormat', - label: 'Formato de Fecha', - type: 'select', - options: [ - { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, - { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, - { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } - ], - defaultValue: 'dd/mm/yyyy' - } - ] -}; - -// --- DATA DEFINITIONS (Items only, no config) --- - -export const catalogosConfig: CsvUploadItem[] = [ - { - id: 'customs_brokers', - title: 'Agentes Aduanales', - icon: User, - modelTarget: 'CustomsBroker', - templateUrl: '/csv/EstructuraCatAgenteAduanal.xls' - }, - { - id: 'clients_providers', - title: 'Clientes y Proveedores', - icon: Users, - modelTarget: 'ClientProvider', - templateUrl: '/csv/EstructuraCatClienteProv.xls' - }, - { - id: 'exchange_rates', - title: 'Tipo de Cambios', - icon: DollarSign, - modelTarget: 'ExchangeRate', - templateUrl: '/csv/EstructuraCatTiposCambio.xls' - }, - { - id: 'american_fractions', - title: 'Fracc. Ame.', - icon: Globe, - modelTarget: 'AmericanFraction', - templateUrl: '/csv/EstructuraCatFraccAme.xls' - }, - { - id: 'material_classes', - title: 'Clases de Materiales', - icon: Package, - modelTarget: 'MaterialClass', - templateUrl: '/csv/EstructuraCatClasesAF.xls' - }, - { - id: 'items', - title: 'Partidas (Permisos)', - icon: FileText, - group: 'Permisos', - modelTarget: 'ItemPermission', - templateUrl: '/csv/EstructuraCatPartesAF.xls' - }, - { - id: 'headers', - title: 'Encabezados (Permisos)', - icon: FileText, - group: 'Permisos', - modelTarget: 'HeaderPermission', - disabled: true, - }, - { - id: 'historical_fractions', - title: 'Fracciones Históricas', - icon: Calendar, - modelTarget: 'HistoricalFraction', - disabled: true, - }, - { - id: 'pedimentos', - title: 'Pedimentos', - icon: FileDigit, - modelTarget: 'Pedimento', - templateUrl: '/csv/EstructuraCatPedimentos.xls' - }, -]; - -export const transportesConfig: CsvUploadItem[] = [ - { - id: 'transports', - title: 'Transportes', - icon: Truck, - modelTarget: 'Transport', - templateUrl: '/csv/EstructuraCatTransportes.xls' - }, - { - id: 'drivers', - title: 'Conductores', - icon: User, - modelTarget: 'Driver', - templateUrl: '/csv/EstructuraCatConductor.xls' - }, - { - id: 'trailers', - title: 'Trailers y Cajas', - icon: Container, - modelTarget: 'Trailer', - templateUrl: '/csv/EstructuraCatTrailers.xls' - }, -]; - -export const importacionConfig: CsvUploadItem[] = [ - // Impo Temp - { - id: 'imp_temp_header', - title: 'Encabezado', - icon: FileText, - group: 'Impo. Temp.', - modelTarget: 'invoice_header', - templateUrl: '/csv/EstructuraEncFacImpoTemp.xls' - }, - { - id: 'imp_temp_details', - title: 'Partidas', - icon: Package, - group: 'Impo. Temp.', - modelTarget: 'invoice_details', - templateUrl: '/csv/EstructuraParFacImpoTempAF.xls' - }, - { - id: 'imp_temp_series', - title: 'Series', - icon: Hash, - group: 'Impo. Temp.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - // Impo Def - { - id: 'imp_def_header', - title: 'Encabezado', - icon: FileText, - group: 'Impo. Def.', - modelTarget: 'InvoiceHeader', - templateUrl: '/csv/EstructuraEncFacImpoDef.xls' - }, - { - id: 'imp_def_details', - title: 'Partidas', - icon: Package, - group: 'Impo. Def.', - modelTarget: 'InvoiceSalesDetails', - templateUrl: '/csv/EstructuraParFacImpoDefAF.xls' - }, - { - id: 'imp_def_series', - title: 'Series', - icon: Hash, - group: 'Impo. Def.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - // Compras Mex - { - id: 'comp_mex_header', - title: 'Encabezado', - icon: FileText, - group: 'Compras Mex.', - modelTarget: 'InvoiceHeader', - disabled: true, - }, - { - id: 'comp_mex_details', - title: 'Partidas', - icon: Package, - group: 'Compras Mex.', - modelTarget: 'InvoiceSalesDetails', - disabled: true, - }, - { - id: 'comp_mex_series', - title: 'Series', - icon: Hash, - group: 'Compras Mex.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, -]; - -export const exportacionConfig: CsvUploadItem[] = [ - // Expo Def / Cam. Reg. - { - id: 'exp_def_header', - title: 'Encabezado', - icon: FileText, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceHeader', - templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls' - }, - { - id: 'exp_def_details', - title: 'Partidas', - icon: Package, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceSalesDetails', - templateUrl: '/csv/EstructuraParExpoCamReg.xls' - }, - { - id: 'exp_def_series', - title: 'Series', - icon: Hash, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - { - id: 'exp_def_nodes', - title: 'NODES', - icon: Briefcase, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'Nodes', - disabled: true, - }, - // Expo Rep - { - id: 'exp_rep_header', - title: 'Encabezado', - icon: FileText, - group: 'Expo. Rep.', - modelTarget: 'InvoiceHeader', - disabled: true, - }, - { - id: 'exp_rep_details', - title: 'Partidas', - icon: Package, - group: 'Expo. Rep.', - modelTarget: 'InvoiceSalesDetails', - disabled: true, - }, - { - id: 'exp_rep_series', - title: 'Series', - icon: Hash, - group: 'Expo. Rep.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - // Manifiesto - { - id: 'manifest_header', - title: 'Encabezado', - icon: FileText, - group: 'Manifiesto', - modelTarget: 'Manifest', - disabled: true, - }, -]; +import { + User, + Users, + FileText, + Truck, + Container, + Ship, + Plane, + Package, + Briefcase, + Globe, + CreditCard, + DollarSign, + Calendar, + Hash, + MapPin, + ShieldCheck, + FileDigit, + Scale, +} from 'lucide-svelte'; + +// --- Interfaces --- + +export interface CsvUploadItem { + id: string; + title: string; + icon: any; + group?: string; // For grouping within a tab + modelTarget?: string; // The backend model this maps to + description?: string; + /** Backend template id for CSV download (e.g. customs_brokers, part_numbers). No physical file. */ + templateId?: string; + disabled?: boolean; // New property to mark items as "Coming Soon" +} + +export interface CsvUploadField { + name: string; + label: string; + type: 'text' | 'select' | 'boolean' | 'date' | 'radio'; + options?: { label: string; value: string | boolean | number }[]; + required?: boolean; + defaultValue?: any; +} + +// Map of Tab ID -> Array of Fields +export const tabSettings: Record = { + catalogos: [ + { + name: 'mode', + label: 'Modo de Carga', + type: 'radio', + options: [ + { label: 'Actualizar', value: 'update' }, + { label: 'Reemplazar', value: 'replace' } + ], + defaultValue: 'update' + } + ], + transportes: [ + { + name: 'mode', + label: 'Modo de Carga', + type: 'radio', + options: [ + { label: 'Actualizar', value: 'update' }, + { label: 'Reemplazar', value: 'replace' } + ], + defaultValue: 'update' + } + ], + importacion: [ + { + name: 'autonumber_remesas', + label: 'Autonumerar Remesas', + type: 'boolean', + defaultValue: false + }, + { + name: 'recalculate_dates', + label: 'Recalcular Fechas', + type: 'boolean', + defaultValue: false + }, + { + name: 'dateFormat', + label: 'Formato de Fecha', + type: 'select', + options: [ + { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, + { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, + { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } + ], + defaultValue: 'dd/mm/yyyy' + } + ], + exportacion: [ + { + name: 'invoice_type', + label: 'Tipo de Factura', + type: 'select', + options: [ + { label: 'AFIJO', value: 'AFIJO' }, + { label: 'NORMAL', value: 'NORMAL' }, + ], + defaultValue: 'AFIJO', + }, + { + name: 'is_regime_change', + label: 'Es Cambio de Régimen', + type: 'boolean', + defaultValue: false, + }, + { + name: 'dateFormat', + label: 'Formato de Fecha', + type: 'select', + options: [ + { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, + { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, + { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } + ], + defaultValue: 'dd/mm/yyyy' + } + ] +}; + +// --- DATA DEFINITIONS (Items only, no config) --- + +export const catalogosConfig: CsvUploadItem[] = [ + { + id: 'customs_brokers', + title: 'Agentes Aduanales', + icon: User, + modelTarget: 'CustomsBroker', + templateId: 'customs_brokers' + }, + { + id: 'clients_providers', + title: 'Clientes y Proveedores', + icon: Users, + modelTarget: 'ClientProvider', + templateId: 'clients_providers' + }, + { + id: 'exchange_rates', + title: 'Tipo de Cambios', + icon: DollarSign, + modelTarget: 'ExchangeRate', + templateId: 'exchange_rates' + }, + { + id: 'american_fractions', + title: 'Fracc. Ame.', + icon: Globe, + modelTarget: 'AmericanFraction', + templateId: 'american_fractions' + }, + { + id: 'material_classes', + title: 'Clases de Materiales', + icon: Package, + modelTarget: 'MaterialClass', + templateId: 'material_classes' + }, + { + id: 'part_numbers', + title: 'Números de parte', + icon: Hash, + modelTarget: 'Part', + templateId: 'part_numbers', + }, + { + id: 'boms', + title: 'BOMs', + icon: Briefcase, + modelTarget: 'Bom', + templateId: 'boms', + }, + { + id: 'items', + title: 'Partidas (Permisos)', + icon: FileText, + group: 'Permisos', + modelTarget: 'ItemPermission', + templateId: 'part_numbers' + }, + { + id: 'headers', + title: 'Encabezados (Permisos)', + icon: FileText, + group: 'Permisos', + modelTarget: 'HeaderPermission', + disabled: true, + }, + { + id: 'historical_fractions', + title: 'Fracciones Históricas', + icon: Calendar, + modelTarget: 'HistoricalFraction', + disabled: true, + }, + { + id: 'pedimentos', + title: 'Pedimentos', + icon: FileDigit, + modelTarget: 'Pedimento', + templateId: 'pedimentos' + }, +]; + +export const transportesConfig: CsvUploadItem[] = [ + { + id: 'transporters', + title: 'Transportistas', + icon: Ship, + modelTarget: 'Transporter', + // No templateId: backend transporters/imports not implemented yet + }, + { + id: 'transports', + title: 'Transportes', + icon: Truck, + modelTarget: 'Transport', + templateId: 'transports' + }, + { + id: 'drivers', + title: 'Conductores', + icon: User, + modelTarget: 'Driver', + templateId: 'drivers' + }, + { + id: 'trailers', + title: 'Trailers y Cajas', + icon: Container, + modelTarget: 'Trailer', + templateId: 'trailers' + }, +]; + +export const importacionConfig: CsvUploadItem[] = [ + // Impo Temp + { + id: 'imp_temp_header', + title: 'Encabezado', + icon: FileText, + group: 'Impo. Temp.', + modelTarget: 'invoice_header', + templateId: 'imp_temp_header' + }, + { + id: 'imp_temp_details', + title: 'Partidas', + icon: Package, + group: 'Impo. Temp.', + modelTarget: 'invoice_details', + templateId: 'imp_temp_details' + }, + { + id: 'imp_temp_series', + title: 'Series', + icon: Hash, + group: 'Impo. Temp.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + // Impo Def + { + id: 'imp_def_header', + title: 'Encabezado', + icon: FileText, + group: 'Impo. Def.', + modelTarget: 'invoice_header', + templateId: 'imp_def_header' + }, + { + id: 'imp_def_details', + title: 'Partidas', + icon: Package, + group: 'Impo. Def.', + modelTarget: 'invoice_details', + templateId: 'imp_def_details' + }, + { + id: 'imp_def_series', + title: 'Series', + icon: Hash, + group: 'Impo. Def.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + // Compras Mex + { + id: 'comp_mex_header', + title: 'Encabezado', + icon: FileText, + group: 'Compras Mex.', + modelTarget: 'invoice_header', + disabled: true, + }, + { + id: 'comp_mex_details', + title: 'Partidas', + icon: Package, + group: 'Compras Mex.', + modelTarget: 'invoice_details', + disabled: true, + }, + { + id: 'comp_mex_series', + title: 'Series', + icon: Hash, + group: 'Compras Mex.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, +]; + +export const exportacionConfig: CsvUploadItem[] = [ + // Expo Def / Cam. Reg. + { + id: 'exp_def_header', + title: 'Encabezado', + icon: FileText, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'invoice_header', + templateId: 'exp_def_header' + }, + { + id: 'exp_def_details', + title: 'Partidas', + icon: Package, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'invoice_details', + templateId: 'exp_def_details' + }, + { + id: 'exp_def_series', + title: 'Series', + icon: Hash, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + { + id: 'exp_def_nodes', + title: 'NODES', + icon: Briefcase, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'Nodes', + disabled: true, + }, + // Expo Rep + { + id: 'exp_rep_header', + title: 'Encabezado', + icon: FileText, + group: 'Expo. Rep.', + modelTarget: 'InvoiceHeader', + disabled: true, + }, + { + id: 'exp_rep_details', + title: 'Partidas', + icon: Package, + group: 'Expo. Rep.', + modelTarget: 'InvoiceSalesDetails', + disabled: true, + }, + { + id: 'exp_rep_series', + title: 'Series', + icon: Hash, + group: 'Expo. Rep.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + // Manifiesto + { + id: 'manifest_header', + title: 'Encabezado', + icon: FileText, + group: 'Manifiesto', + modelTarget: 'Manifest', + disabled: true, + }, +]; diff --git a/frontend/src/lib/server/api.ts b/frontend/src/lib/server/api.ts index a338dc06..a68524d0 100644 --- a/frontend/src/lib/server/api.ts +++ b/frontend/src/lib/server/api.ts @@ -41,6 +41,10 @@ export function getAuthTokens(cookies: Cookies) { /** * Establece los tokens de autenticación en las cookies + * + * Política de seguridad: + * - access_token → NO HttpOnly (el cliente necesita incluirlo en el header Authorization) + * - refresh_token → HttpOnly=true (JS nunca lo lee; el servidor lo maneja via /api-sveltekit/auth/silent-refresh) */ export function setAuthTokens( cookies: Cookies, @@ -49,7 +53,7 @@ export function setAuthTokens( ) { cookies.set('access_token', accessToken, { path: '/', - httpOnly: false, + httpOnly: false, // El cliente JS necesita leerlo para el header Bearer sameSite: 'lax', secure: process.env.NODE_ENV === 'production', maxAge: 60 * 60 * 24 * 7 // 7 días @@ -58,7 +62,7 @@ export function setAuthTokens( if (refreshToken) { cookies.set('refresh_token', refreshToken, { path: '/', - httpOnly: false, + httpOnly: true, // *** HttpOnly: JS nunca lee el refresh_token *** sameSite: 'lax', secure: process.env.NODE_ENV === 'production', maxAge: 60 * 60 * 24 * 30 // 30 días diff --git a/frontend/src/lib/session-manager.ts b/frontend/src/lib/session-manager.ts new file mode 100644 index 00000000..b9157e89 --- /dev/null +++ b/frontend/src/lib/session-manager.ts @@ -0,0 +1,505 @@ +/** + * Gestor de sesión SSO para Keycloak + * + * Responsabilidades: + * - Refresh silencioso del access token SOLO cuando el usuario está activo. + * - Detección de actividad del usuario (evita polling innecesario cuando está idle). + * - Idle timeout: si el usuario está inactivo, no refrescar → el token y la sesión + * expiran en Keycloak de forma natural → siguiente petición 401 → logout. + * - Logout automático cuando el refresh falla (sesión SSO terminada por Keycloak, + * admin forzado, max session alcanzado, etc.). + * - Verificación de sesión SSO usando el iframe silencioso de Keycloak JS. + * + * Flujo de tokens: + * - Access token: en memoria (authStore) + cookie no-HttpOnly (password login) + * o en instancia Keycloak JS (SSO flow) + * - Refresh token: cookie HttpOnly únicamente (el JS nunca lo toca) + * - El refresh se hace server-side via /api-sveltekit/auth/silent-refresh + */ + +import { browser } from '$app/environment'; +import type Keycloak from 'keycloak-js'; + +// ───────────────────────────────────────────────────────── +// Eventos DOM personalizados +// ───────────────────────────────────────────────────────── + +/** Se emite cuando la sesión está próxima a expirar por inactividad */ +export const SESSION_WARNING_EVENT = 'session:warning'; +/** Se emite cuando la sesión ha expirado (idle, max-session o refresh fallido) */ +export const SESSION_EXPIRED_EVENT = 'session:expired'; +/** Se emite cuando el usuario extiende la sesión desde el diálogo de advertencia */ +export const SESSION_EXTENDED_EVENT = 'session:extended'; +/** Se emite después de un refresh silencioso exitoso */ +export const SESSION_TOKEN_REFRESHED_EVENT = 'session:token-refreshed'; + +// ───────────────────────────────────────────────────────── +// Tipos +// ───────────────────────────────────────────────────────── + +export type SessionExpiredReason = 'idle' | 'refresh_failed' | 'keycloak_session_ended' | 'error' | 'manual'; + +export interface SessionExpiredDetail { + reason: SessionExpiredReason; +} + +export interface SessionWarningDetail { + remainingMs: number; +} + +export interface SessionManagerOptions { + /** + * Segundos antes de la expiración del token para intentar el refresh. + * Default: 60 + */ + refreshBeforeExpirySeconds?: number; + + /** + * Tiempo de inactividad (ms) después del cual NO se refresca el token, + * permitiendo que la sesión de Keycloak expire de forma natural. + * Default: 30 minutos (1_800_000 ms) + */ + idleTimeoutMs?: number; + + /** + * Milisegundos antes del idle timeout para mostrar el diálogo de advertencia. + * Default: 5 minutos (300_000 ms) + */ + warningBeforeIdleMs?: number; + + /** + * Intervalo (ms) para verificar silenciosamente la sesión SSO de Keycloak. + * Solo se usa cuando hay una instancia de Keycloak JS autenticada. + * Default: 5 minutos (300_000 ms). 0 para deshabilitar. + */ + ssoCheckIntervalMs?: number; + + /** Función para obtener la instancia de Keycloak JS (si usa el SSO flow) */ + getKeycloakInstance?: () => Keycloak | null; + + /** Callback cuando el token se refresca exitosamente */ + onTokenRefreshed?: (newToken: string) => void; + + /** Callback cuando la sesión expira */ + onSessionExpired?: (reason: SessionExpiredReason) => void; +} + +// ───────────────────────────────────────────────────────── +// SessionManager +// ───────────────────────────────────────────────────────── + +export class SessionManager { + private opts: Required; + + // Timers + private refreshTimerId: ReturnType | null = null; + private idleTimerId: ReturnType | null = null; + private warningTimerId: ReturnType | null = null; + private ssoCheckIntervalId: ReturnType | null = null; + + // State + private currentToken: string | null = null; + private lastActivityAt = Date.now(); + private warningShown = false; + private isRefreshing = false; + private destroyed = false; + + // Activity listener cleanups + private removeListeners: Array<() => void> = []; + + constructor(options: SessionManagerOptions = {}) { + this.opts = { + refreshBeforeExpirySeconds: options.refreshBeforeExpirySeconds ?? 60, + idleTimeoutMs: options.idleTimeoutMs ?? 30 * 60 * 1000, + warningBeforeIdleMs: options.warningBeforeIdleMs ?? 5 * 60 * 1000, + ssoCheckIntervalMs: options.ssoCheckIntervalMs ?? 5 * 60 * 1000, + getKeycloakInstance: options.getKeycloakInstance ?? (() => null), + onTokenRefreshed: options.onTokenRefreshed ?? (() => {}), + onSessionExpired: options.onSessionExpired ?? (() => {}) + }; + } + + // ───────────────────────────────────────────────────── + // Public API + // ───────────────────────────────────────────────────── + + /** + * Inicia el gestor de sesión con el token actual. + * Debe llamarse una vez tras la autenticación exitosa. + */ + start(initialToken: string): void { + if (!browser || this.destroyed) return; + + this.currentToken = initialToken; + this.lastActivityAt = Date.now(); + this.warningShown = false; + + this.setupActivityListeners(); + this.scheduleRefresh(initialToken); + this.scheduleIdleTimers(); + this.startSsoCheckInterval(); + } + + /** + * Actualiza el token en memoria (llamar después de un refresh exitoso externo). + */ + updateToken(newToken: string): void { + if (this.destroyed) return; + this.currentToken = newToken; + this.warningShown = false; + this.cancelRefreshTimer(); + this.scheduleRefresh(newToken); + this.resetIdleTimers(); + } + + /** + * El usuario hizo clic en "Continuar sesión" en el diálogo de advertencia. + * Fuerza un refresh inmediato y reinicia los timers de inactividad. + */ + extendSession(): void { + if (this.destroyed) return; + this.recordActivity(); + if (this.currentToken) { + void this.doRefresh('extend'); + } + } + + /** Destruye el gestor y limpia todos los recursos. */ + destroy(): void { + this.destroyed = true; + this.cancelRefreshTimer(); + this.cancelIdleTimers(); + this.stopSsoCheckInterval(); + this.teardownActivityListeners(); + } + + // ───────────────────────────────────────────────────── + // Activity tracking + // ───────────────────────────────────────────────────── + + private setupActivityListeners(): void { + const events: (keyof WindowEventMap)[] = [ + 'mousedown', + 'mousemove', + 'keydown', + 'scroll', + 'touchstart', + 'click', + 'pointerdown' + ]; + + // Limitar actualizaciones de actividad a máximo una por segundo + let debounceTimer: ReturnType | null = null; + + const handler = () => { + if (debounceTimer) return; + debounceTimer = setTimeout(() => { + debounceTimer = null; + this.recordActivity(); + }, 1000); + }; + + events.forEach((event) => { + window.addEventListener(event, handler, { passive: true }); + this.removeListeners.push(() => window.removeEventListener(event, handler)); + }); + + // Al volver a la pestaña, registrar actividad y verificar si hace falta + // un refresh inmediato (el tiempo puede haber pasado con la pestaña en segundo plano) + const visibilityHandler = () => { + if (document.visibilityState === 'visible') { + this.recordActivity(); + void this.refreshIfExpiringSoon(); + } + }; + document.addEventListener('visibilitychange', visibilityHandler); + this.removeListeners.push(() => + document.removeEventListener('visibilitychange', visibilityHandler) + ); + } + + private teardownActivityListeners(): void { + this.removeListeners.forEach((fn) => fn()); + this.removeListeners = []; + } + + private recordActivity(): void { + const wasIdle = this.isIdle(); + this.lastActivityAt = Date.now(); + + if (this.warningShown) { + // El usuario volvió activo → descartar advertencia + this.warningShown = false; + this.resetIdleTimers(); + window.dispatchEvent(new CustomEvent(SESSION_EXTENDED_EVENT)); + } else if (wasIdle) { + // Volvemos de idle → reiniciar timers + this.resetIdleTimers(); + void this.refreshIfExpiringSoon(); + } + } + + private isIdle(): boolean { + return Date.now() - this.lastActivityAt > this.opts.idleTimeoutMs; + } + + // ───────────────────────────────────────────────────── + // Token refresh scheduling + // ───────────────────────────────────────────────────── + + private parseExpiry(token: string): number | null { + try { + const payload = JSON.parse(atob(token.split('.')[1].replace(/-/g, '+').replace(/_/g, '/'))); + return typeof payload.exp === 'number' ? payload.exp * 1000 : null; + } catch { + return null; + } + } + + private scheduleRefresh(token: string): void { + const expiry = this.parseExpiry(token); + if (!expiry) return; + + const msUntilRefresh = expiry - Date.now() - this.opts.refreshBeforeExpirySeconds * 1000; + + if (msUntilRefresh <= 0) { + void this.doRefresh('scheduled'); + return; + } + + this.refreshTimerId = setTimeout(() => { + if (!this.destroyed) void this.doRefresh('scheduled'); + }, msUntilRefresh); + } + + private cancelRefreshTimer(): void { + if (this.refreshTimerId !== null) { + clearTimeout(this.refreshTimerId); + this.refreshTimerId = null; + } + } + + /** Refresca el token si le quedan menos de `refreshBeforeExpirySeconds` */ + private async refreshIfExpiringSoon(): Promise { + if (!this.currentToken) return; + const expiry = this.parseExpiry(this.currentToken); + if (!expiry) return; + if (expiry - Date.now() < this.opts.refreshBeforeExpirySeconds * 1000) { + await this.doRefresh('on-demand'); + } + } + + // ───────────────────────────────────────────────────── + // Idle session timers + // ───────────────────────────────────────────────────── + + private scheduleIdleTimers(): void { + this.cancelIdleTimers(); + const now = Date.now(); + const idleAt = this.lastActivityAt + this.opts.idleTimeoutMs; + const warnAt = idleAt - this.opts.warningBeforeIdleMs; + + const msUntilWarn = warnAt - now; + const msUntilIdle = idleAt - now; + + if (msUntilWarn > 0) { + this.warningTimerId = setTimeout(() => { + if (!this.destroyed && !this.warningShown && this.isIdle() === false) { + this.showWarning(this.opts.warningBeforeIdleMs); + } + }, msUntilWarn); + } + + if (msUntilIdle > 0) { + this.idleTimerId = setTimeout(() => { + if (!this.destroyed && this.isIdle()) { + this.handleIdleExpiry(); + } + }, msUntilIdle); + } + } + + private cancelIdleTimers(): void { + if (this.warningTimerId !== null) { + clearTimeout(this.warningTimerId); + this.warningTimerId = null; + } + if (this.idleTimerId !== null) { + clearTimeout(this.idleTimerId); + this.idleTimerId = null; + } + } + + private resetIdleTimers(): void { + this.scheduleIdleTimers(); + } + + private showWarning(remainingMs: number): void { + this.warningShown = true; + const detail: SessionWarningDetail = { remainingMs }; + window.dispatchEvent(new CustomEvent(SESSION_WARNING_EVENT, { detail })); + } + + private handleIdleExpiry(): void { + const detail: SessionExpiredDetail = { reason: 'idle' }; + window.dispatchEvent(new CustomEvent(SESSION_EXPIRED_EVENT, { detail })); + this.opts.onSessionExpired('idle'); + } + + // ───────────────────────────────────────────────────── + // Periodic Keycloak SSO session check (iframe) + // ───────────────────────────────────────────────────── + + private startSsoCheckInterval(): void { + if (this.opts.ssoCheckIntervalMs <= 0) return; + + this.ssoCheckIntervalId = setInterval(() => { + if (!this.destroyed) void this.checkKeycloakSsoSession(); + }, this.opts.ssoCheckIntervalMs); + } + + private stopSsoCheckInterval(): void { + if (this.ssoCheckIntervalId !== null) { + clearInterval(this.ssoCheckIntervalId); + this.ssoCheckIntervalId = null; + } + } + + /** + * Comprueba silenciosamente si la sesión SSO de Keycloak sigue activa. + * Si el check falla (sesión terminada remotamente) → logout. + */ + private async checkKeycloakSsoSession(): Promise { + const kc = this.opts.getKeycloakInstance(); + if (!kc?.authenticated) return; // Solo aplica al flow SSO con Keycloak JS + + try { + // updateToken(0) fuerza a Keycloak JS a intentar refrescar via SSO + // Si la sesión SSO de Keycloak ha sido terminada, lanza un error + await kc.updateToken(0); + } catch { + console.warn('[SessionManager] Keycloak SSO session ended remotely'); + const detail: SessionExpiredDetail = { reason: 'keycloak_session_ended' }; + window.dispatchEvent(new CustomEvent(SESSION_EXPIRED_EVENT, { detail })); + this.opts.onSessionExpired('keycloak_session_ended'); + } + } + + // ───────────────────────────────────────────────────── + // Token refresh execution + // ───────────────────────────────────────────────────── + + private async doRefresh(reason: string): Promise { + if (this.destroyed || this.isRefreshing) return; + + // No refrescar automáticamente si el usuario está idle + // (excepto si es un refresh forzado por "extender sesión") + if (reason === 'scheduled' && this.isIdle()) { + console.info('[SessionManager] Omitiendo refresh — usuario inactivo'); + return; + } + + this.isRefreshing = true; + + try { + const kc = this.opts.getKeycloakInstance(); + let newToken: string | null = null; + + if (kc?.authenticated) { + // ── Keycloak JS flow ────────────────────────────────────────── + // updateToken intenta un silent refresh via iframe con la cookie + // de sesión SSO de Keycloak. + // Si el SSO session ha expirado, esto lanzará un error. + const minValidity = this.opts.refreshBeforeExpirySeconds + 10; + await kc.updateToken(minValidity); + newToken = kc.token ?? null; + } else { + // ── Password login flow ─────────────────────────────────────── + // Usar el endpoint server-side de SvelteKit que lee el refresh_token + // desde la cookie HttpOnly (el JS nunca ve el refresh_token). + newToken = await this.silentRefreshViaCookie(); + } + + if (newToken) { + this.currentToken = newToken; + this.cancelRefreshTimer(); + this.scheduleRefresh(newToken); + this.opts.onTokenRefreshed(newToken); + window.dispatchEvent( + new CustomEvent(SESSION_TOKEN_REFRESHED_EVENT, { detail: { token: newToken } }) + ); + } else { + this.handleRefreshFailure(); + } + } catch (err) { + console.error('[SessionManager] Error durante refresh:', err); + this.handleRefreshFailure(); + } finally { + this.isRefreshing = false; + } + } + + private handleRefreshFailure(): void { + console.warn('[SessionManager] Refresh fallido — la sesión SSO probablemente expiró'); + const detail: SessionExpiredDetail = { reason: 'refresh_failed' }; + window.dispatchEvent(new CustomEvent(SESSION_EXPIRED_EVENT, { detail })); + this.opts.onSessionExpired('refresh_failed'); + } + + /** + * Llama al endpoint server-side de SvelteKit para realizar el refresh + * usando la cookie HttpOnly del refresh_token. + * + * El servidor lee la cookie, llama a Keycloak, obtiene los nuevos tokens, + * actualiza las cookies HttpOnly y devuelve el nuevo access_token al cliente. + * El refresh_token NUNCA toca el código JavaScript del cliente. + */ + private async silentRefreshViaCookie(): Promise { + try { + const resp = await fetch('/api-sveltekit/auth/silent-refresh', { + method: 'POST', + credentials: 'include', // Envía todas las cookies HttpOnly + headers: { 'Content-Type': 'application/json' } + }); + + if (!resp.ok) return null; + + const data = await resp.json(); + return (data as { access_token?: string }).access_token ?? null; + } catch (err) { + console.error('[SessionManager] Error en silentRefreshViaCookie:', err); + return null; + } + } +} + +// ───────────────────────────────────────────────────────── +// Singleton helpers +// ───────────────────────────────────────────────────────── + +let _instance: SessionManager | null = null; + +/** Obtiene la instancia singleton del SessionManager */ +export function getSessionManager(): SessionManager | null { + return _instance; +} + +/** + * Crea (o recrea) el SessionManager singleton. + * Destruye la instancia anterior si existe. + */ +export function createSessionManager(options?: SessionManagerOptions): SessionManager { + if (_instance) { + _instance.destroy(); + } + _instance = new SessionManager(options); + return _instance; +} + +/** Destruye el SessionManager singleton y limpia todos los recursos */ +export function destroySessionManager(): void { + if (_instance) { + _instance.destroy(); + _instance = null; + } +} diff --git a/frontend/src/routes/api-sveltekit/auth/silent-refresh/+server.ts b/frontend/src/routes/api-sveltekit/auth/silent-refresh/+server.ts new file mode 100644 index 00000000..68366d84 --- /dev/null +++ b/frontend/src/routes/api-sveltekit/auth/silent-refresh/+server.ts @@ -0,0 +1,61 @@ +/** + * Endpoint server-side para el refresh silencioso del access token. + * + * Flujo de seguridad: + * 1. El cliente llama a POST /api-sveltekit/auth/silent-refresh con credentials:'include' + * (las cookies HttpOnly se envían automáticamente, sin que JS las lea). + * 2. Este servidor lee el refresh_token de la cookie HttpOnly. + * 3. Llama al backend FastAPI /v1/auth/refresh con el refresh_token. + * 4. Si es exitoso, actualiza las cookies HttpOnly con los nuevos tokens. + * 5. Retorna solo el access_token al cliente (el refresh_token permanece en HttpOnly). + * + * De este modo el refresh_token NUNCA toca el código JavaScript del cliente. + */ + +import { json } from '@sveltejs/kit'; +import type { RequestEvent } from '@sveltejs/kit'; +import { getServerApiUrl, setAuthTokens } from '$lib/server/api'; + +export const POST = async ({ cookies, fetch }: RequestEvent) => { + const refreshToken = cookies.get('refresh_token'); + + if (!refreshToken) { + return json({ error: 'No refresh token available' }, { status: 401 }); + } + + try { + const baseUrl = getServerApiUrl(); + + const response = await fetch(`${baseUrl}v1/auth/refresh`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ refresh_token: refreshToken }) + }); + + if (!response.ok) { + // El refresh token expiró o fue invalidado por Keycloak (sesión terminada). + // Limpiar las cookies para que el servidor redirigir al login en la siguiente carga. + cookies.delete('refresh_token', { path: '/' }); + cookies.delete('access_token', { path: '/' }); + cookies.delete('active_company_id', { path: '/' }); + + const status = response.status === 401 ? 401 : 400; + return json({ error: 'Refresh token expired or invalid' }, { status }); + } + + const data = (await response.json()) as { + access_token: string; + refresh_token?: string; + expires_in?: number; + }; + + // Actualizar las cookies HttpOnly con los nuevos tokens + setAuthTokens(cookies, data.access_token, data.refresh_token); + + // Devolver solo el access_token al cliente + return json({ access_token: data.access_token }); + } catch (error) { + console.error('[silent-refresh] Error inesperado:', error); + return json({ error: 'Internal server error' }, { status: 500 }); + } +}; diff --git a/frontend/src/routes/auth/callback/+page.server.ts b/frontend/src/routes/auth/callback/+page.server.ts index 6e0426d5..b172d88c 100644 --- a/frontend/src/routes/auth/callback/+page.server.ts +++ b/frontend/src/routes/auth/callback/+page.server.ts @@ -1,7 +1,7 @@ import { redirect } from '@sveltejs/kit'; import type { PageServerLoad } from './$types'; -export const load: PageServerLoad = async ({ url, cookies }) => { +export const load: PageServerLoad = async ({ url, cookies, fetch }) => { // Obtener el código y state de los query params const code = url.searchParams.get('code'); const state = url.searchParams.get('state'); @@ -56,13 +56,14 @@ export const load: PageServerLoad = async ({ url, cookies }) => { const tokens = await tokenResponse.json(); - // Establecer las cookies en el servidor (esto es lo importante) - // Las cookies deben ser HttpOnly y Secure en producción + // Establecer las cookies en el servidor + // access_token → NO HttpOnly (el cliente JS lo usa para el header Authorization) + // refresh_token → HttpOnly (el JS nunca lo lee; el servidor lo gestiona) const isProduction = process.env.NODE_ENV === 'production'; cookies.set('access_token', tokens.access_token, { path: '/', - httpOnly: true, + httpOnly: false, // El cliente necesita leerlo para Bearer secure: isProduction, sameSite: 'lax', maxAge: 60 * 60 * 24 * 7 // 7 días @@ -71,7 +72,7 @@ export const load: PageServerLoad = async ({ url, cookies }) => { if (tokens.refresh_token) { cookies.set('refresh_token', tokens.refresh_token, { path: '/', - httpOnly: true, + httpOnly: true, // *** HttpOnly: nunca expuesto a JS *** secure: isProduction, sameSite: 'lax', maxAge: 60 * 60 * 24 * 30 // 30 días diff --git a/frontend/src/routes/dashboard/+layout.svelte b/frontend/src/routes/dashboard/+layout.svelte index adc8e934..68d3011f 100644 --- a/frontend/src/routes/dashboard/+layout.svelte +++ b/frontend/src/routes/dashboard/+layout.svelte @@ -1,5 +1,5 @@ @@ -67,3 +116,6 @@ {#if !page.url.pathname.includes('/dashboard/invoices') && !page.url.pathname.includes('/dashboard/pedimentos')} {/if} + + + diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 6c55bc56..5a1901f1 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -1,209 +1,549 @@ - - -
- -
-
-

Importación Masiva de Datos (CSV)

-
- - - - Catálogos - Transportes - Importación - Exportación - - -
- -
-

Catálogos Generales

-
- - - - -
-

Logística y Transporte

-
- - - - -
-

Operaciones de Importación

-
- - - - -
-

Operaciones de Exportación

-
- - -
- - -
-
- - - {#if allSettings[activeTab]} -
- -
- {/if} -
- - { - if (currentJobId && activeModelTarget) { - try { - isUploading = true; - const res = await api.imports.commit(currentJobId, activeModelTarget); - if (res.data?.commit_job_id) { - currentJobId = res.data.commit_job_id; - pollStatus(); - } - } catch (err) { - toast.error('Error al iniciar la importación'); - isUploading = false; - } - } - }} - onCancel={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} - onClose={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} -/> + + +
+ +
+
+

Importación Masiva de Datos (CSV)

+
+ + + + Catálogos + Transportes + Importación + Exportación + + +
+ +
+

Catálogos Generales

+
+ + + + +
+

Logística y Transporte

+
+ + + + +
+

Operaciones de Importación

+
+ + + + +
+

Operaciones de Exportación

+
+ + +
+ + +
+
+ + + {#if allSettings[activeTab]} +
+ +
+ {/if} +
+ +{#if scanResults || commitResults} + { + if (!currentJobId) return; + try { + isUploading = true; + const res = useCustomsBrokerImport + ? await api.customsBrokerImports.commit(currentJobId) + : useClientProviderImport + ? await api.clientProviderImports.commit(currentJobId) + : useExchangeRateImport + ? await api.exchangeRateImports.commit(currentJobId) + : useAmericanFractionImport + ? await api.americanFractionImports.commit(currentJobId) + : usePedimentosImport + ? await api.pedimentosImports.commit(currentJobId) + : useMaterialClassesImport + ? await api.materialClassImports.commit(currentJobId) + : useVehicleImport + ? await api.vehicleImports.commit(currentJobId) + : useDriverImport + ? await api.driverImports.commit(currentJobId) + : useTrailerImport + ? await api.trailerImports.commit(currentJobId) + : useTransporterImport + ? await api.transporterImports.commit(currentJobId) + : usePartNumbersImport + ? await api.partNumberImports.commit(currentJobId) + : useBomImport + ? await api.bomImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); + if (res.data?.commit_job_id) { + currentJobId = res.data.commit_job_id; + pollStatus(); + } + } catch (err) { + toast.error('Error al iniciar la importación'); + isUploading = false; + } + }} + onCancel={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + onClose={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + /> +{/if} diff --git a/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte b/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte index 801303a2..c52512ae 100644 --- a/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte +++ b/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte @@ -199,7 +199,9 @@ } function handleStateSelect(state: any) { - formData.state = state.m3_key || state.mex_key || state.ame_key; + // Mostrar en Estado la clave del estado (mex_key/ame_key), no el país (m3_key) + formData.state = state.mex_key || state.ame_key || state.m3_key; + if (state.m3_key) formData.country = state.m3_key; } // --- 4. GUARDADO --- @@ -538,6 +540,7 @@ placeholder="32000" disabled={loading} class="h-10" + oninput={(e) => { formData.postal_code = e.currentTarget.value.replace(/[^a-zA-Z0-9]/g, ''); }} />
diff --git a/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.server.ts b/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.server.ts new file mode 100644 index 00000000..0e4e1801 --- /dev/null +++ b/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.server.ts @@ -0,0 +1,55 @@ +import type { PageServerLoad } from './$types'; +import { error, redirect } from '@sveltejs/kit'; +import { getAuthTokens, getActiveCompanyId, authenticatedFetch } from '$lib/server/api'; + +export const load: PageServerLoad = async ({ params, cookies, fetch }) => { + const { accessToken } = getAuthTokens(cookies); + + if (!accessToken) { + throw redirect(302, '/login'); + } + + const companyId = await getActiveCompanyId(cookies, fetch); + if (!companyId) { + throw error(400, 'No se encontró una compañía seleccionada'); + } + + let agentsCount = 0; + let clientsCount = 0; + + try { + const [brokersRes, clientsRes] = await Promise.all([ + authenticatedFetch( + `v1/a76/customs-brokers?company_id=${companyId}&page=1&page_size=1`, + {}, + cookies, + fetch + ), + authenticatedFetch( + `v1/a76/clients-providers?company_id=${companyId}&page=1&page_size=1`, + {}, + cookies, + fetch + ) + ]); + + if (brokersRes.ok) { + const brokersData = await brokersRes.json(); + agentsCount = brokersData.total ?? 0; + } + if (clientsRes.ok) { + const clientsData = await clientsRes.json(); + clientsCount = clientsData.total ?? 0; + } + } catch (e) { + console.error('Error fetching prerequisites count for parts:', e); + } + + const isCreate = params.id === 'new' || params.id === undefined; + + return { + agentsCount, + clientsCount, + isCreate + }; +}; diff --git a/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.svelte b/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.svelte index 72633d4c..d87c3f3d 100644 --- a/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.svelte +++ b/frontend/src/routes/dashboard/goods/parts/edit/[[id]]/+page.svelte @@ -1,9 +1,36 @@ +{#if data?.isCreate} + (showPrerequisitesModal = false)} + onCancel={() => goto('/dashboard/goods/parts')} + /> +{/if} + \ No newline at end of file diff --git a/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte b/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte index df9e4fec..2eb4bbd9 100644 --- a/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte +++ b/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte @@ -39,6 +39,7 @@ import { useShortcuts } from '$lib/hooks/use-shortcuts'; import { obtenerAtajosEdicionFactura } from '$lib/config/shortcuts/dashboard/invoices/edit'; import { api } from '$lib/api'; + import PrerequisitesModal from '$lib/components/dashboard/PrerequisitesModal.svelte'; // Cargar companyStore solo en el cliente - no usamos sidebar en esta página let companyStore: any = $state(undefined); @@ -793,6 +794,19 @@ manejarRegresar: handleBack }) ); + + // Prerrequisitos: modal cuando no hay agentes o clientes + const agentsCount = $derived(data.customsBrokers?.length ?? 0); + const clientsCount = $derived(data.clients?.length ?? 0); + let showPrerequisitesModal = $state(false); + let hasTriedToShowPrerequisitesModal = $state(false); + $effect(() => { + if (hasTriedToShowPrerequisitesModal) return; + if (agentsCount === 0 || clientsCount === 0) { + hasTriedToShowPrerequisitesModal = true; + showPrerequisitesModal = true; + } + });
@@ -1042,3 +1056,11 @@
+ + (showPrerequisitesModal = false)} + onCancel={() => goto('/dashboard/invoices')} +/> diff --git a/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte b/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte index 8f306d7a..6fa87ac2 100644 --- a/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte +++ b/frontend/src/routes/dashboard/pedimentos/edit/[id]/+page.svelte @@ -79,6 +79,7 @@ import { getExchangeRateByDate } from '$lib/api/dashboard/a76/exchange-rate'; import ExchangeRateDialog from '$lib/components/dashboard/exchange_rate/create-edit-dialog.svelte'; import { companyStore } from '$lib/stores/company.svelte'; + import PrerequisitesModal from '$lib/components/dashboard/PrerequisitesModal.svelte'; let { data }: { data: ExtendedPageData } = $props(); @@ -118,6 +119,20 @@ let showExchangeRateDialog = $state(false); let missingExchangeRateDate = $state(''); + // Prerrequisitos: modal solo en creación cuando no hay agentes o clientes + const agentsCount = $derived(data.customsBrokers?.length ?? 0); + const clientsCount = $derived(data.clients?.length ?? 0); + let showPrerequisitesModal = $state(false); + let hasTriedToShowPrerequisitesModal = $state(false); + $effect(() => { + if (hasTriedToShowPrerequisitesModal) return; + if (!data.isCreate) return; + if (agentsCount === 0 || clientsCount === 0) { + hasTriedToShowPrerequisitesModal = true; + showPrerequisitesModal = true; + } + }); + async function checkPaymentDateRate(date: string): Promise { if (!date || !companyStore.activeCompany?.id) return true; @@ -418,6 +433,31 @@ } } + // Validar tipo de cambio en create y update + if (generalFormData) { + const rate = generalFormData.exchange_rate; + if ( + rate === null || + rate === undefined || + String(rate).trim() === '' || + Number(rate) <= 0 + ) { + saving = false; + activeTab = 'general'; + const date = generalFormData.entry_date || ''; + toast.error( + Number(rate) <= 0 && rate !== null && rate !== undefined + ? 'El tipo de cambio debe ser mayor a 0. Registra el tipo de cambio para la fecha de entrada.' + : 'No hay tipo de cambio registrado para la fecha de entrada. Por favor, regístralo antes de guardar.' + ); + if (date) { + missingExchangeRateDate = date; + showExchangeRateDialog = true; + } + return; + } + } + // Construir el payload unificado const payload: any = { // Datos generales @@ -1249,4 +1289,15 @@
+ + + {#if data.isCreate} + (showPrerequisitesModal = false)} + onCancel={() => goto('/dashboard/pedimentos')} + /> + {/if} diff --git a/frontend/src/routes/login/+page.server.ts b/frontend/src/routes/login/+page.server.ts index c8492b7a..0fd34e11 100644 --- a/frontend/src/routes/login/+page.server.ts +++ b/frontend/src/routes/login/+page.server.ts @@ -19,7 +19,7 @@ export const load: PageServerLoad = async ({ cookies, url }) => { }; export const actions = { - default: async ({ request, cookies, url }) => { + default: async ({ request, cookies, url, fetch }) => { const data = await request.formData(); const username = data.get('username')?.toString(); const password = data.get('password')?.toString(); diff --git a/frontend/static/csv/EstructuraCatAgenteAduanal.xls b/frontend/static/csv/EstructuraCatAgenteAduanal.xls deleted file mode 100644 index 8978a33e..00000000 --- a/frontend/static/csv/EstructuraCatAgenteAduanal.xls +++ /dev/null @@ -1 +0,0 @@ -TIPO(MEX=Mexicano,AME=AMERICANO) CLAVE AADUANAL PATENTE NOMBRE RFC DIRECCION CODIGO POSTAL CIUDAD ESTADO PAIS TELEFONO NUMERO FAX CORREO ELECTRONICO CURP diff --git a/frontend/static/csv/EstructuraCatClasesAF.xls b/frontend/static/csv/EstructuraCatClasesAF.xls deleted file mode 100644 index 96b7891a..00000000 --- a/frontend/static/csv/EstructuraCatClasesAF.xls +++ /dev/null @@ -1 +0,0 @@ -CLAVE CLASE DESCRIPCION ESPAOL DESCRIPCION INGLES TIPO DE MATERIAL U.M. COMERCIAL FRACCION ARANCELARIA FRACCION AMERICANA TASA DE DEPRECIACION REVISION FISICA (1/0) CODIGO DE PRODUCTO/SERVICIO CP diff --git a/frontend/static/csv/EstructuraCatClienteProv.xls b/frontend/static/csv/EstructuraCatClienteProv.xls deleted file mode 100644 index 1b2096cc..00000000 --- a/frontend/static/csv/EstructuraCatClienteProv.xls +++ /dev/null @@ -1 +0,0 @@ -PROCEDENCIA CLIENTE(E=Extranjero, N=Nacional) TIPO(C=Cliente,P=Proveedor,A=Ambos) CLAVE CLIENTE NOMBRE RFC CALLES NUM. EXTERIOR CODIGO POSTAL COLONIA o PARQUE IND. CIUDAD ESTADO PAIS TELEFONO NUMERO FAX CORREO ELECTRONICO CURP TIPO DE PROGRAMA SECON NUMERO DE PROGRAMA SECON FECHA AUT. SECON ##/##/#### ES PROGRAMA PROSEC? (SI o NO) NUMERO DE PROGRAMA PROSEC VINCULACION ES EMPRESA CERTIFICADA? REGISTRO DE EMPRESA CERT. INFORMACION ADICIONAL CONTACTO CLAVE MANUFACTURERO TAX I.D. CLAVE BROKER AMERICANO EXPO CLAVE BROKER AMERICANO IMPO CLAVE TRANSFERENCIA A.A. TRANSFORMADOR/SUBMAQUILA CLAVE INTERFACE diff --git a/frontend/static/csv/EstructuraCatConductor.xls b/frontend/static/csv/EstructuraCatConductor.xls deleted file mode 100644 index 30482cb7..00000000 --- a/frontend/static/csv/EstructuraCatConductor.xls +++ /dev/null @@ -1 +0,0 @@ -TRANSPORTISTA LINEA CLAVE CONDUCTOR LICENCIA PERMISO LINEA EXPRESS IDENTIFICACION ACE FECHA NACIMIENTO SEXO PAIS NACIMIENTO TRANSPORTA MAT. PELIGROSO? PERMISO MAT. PELIGROSO NOMBRE(S) APELLIDO PATERNO FORMA IDENTIFICACION 1 NUM. IDENTIFICACION 1 ESTADO PAIS FORMA IDENTIFICACION 2 NUM. IDENTIFICACION 2 ESTADO PAIS diff --git a/frontend/static/csv/EstructuraCatFraccAme.xls b/frontend/static/csv/EstructuraCatFraccAme.xls deleted file mode 100644 index dc1f5495..00000000 --- a/frontend/static/csv/EstructuraCatFraccAme.xls +++ /dev/null @@ -1 +0,0 @@ -FRACCION ARANCELARIA PREFIJO UNIDAD DE MEDIDA DESCRIPCION TIPO DE ADVALOREM ADVALOREM % ADVALOREM DLLS diff --git a/frontend/static/csv/EstructuraCatPartesAF.xls b/frontend/static/csv/EstructuraCatPartesAF.xls deleted file mode 100644 index e989d04f..00000000 --- a/frontend/static/csv/EstructuraCatPartesAF.xls +++ /dev/null @@ -1 +0,0 @@ -NUMERO DE PARTE DESCRIPCION EN ESPAOL DESCRIPCION EN INGLES CLASE UNIDAD DE MEDIDA COMERCIAL COSTO UNITARIO TIPO MONEDA COSTO CLAVE MONEDA PESO UNITARIO TIPO PESO FRACCION PAIS PREFERENCIA SECTOR RUTA DE LA IMAGEN diff --git a/frontend/static/csv/EstructuraCatPedimentos.xls b/frontend/static/csv/EstructuraCatPedimentos.xls deleted file mode 100644 index b2e2f7f2..00000000 --- a/frontend/static/csv/EstructuraCatPedimentos.xls +++ /dev/null @@ -1 +0,0 @@ -NUMERO DE PEDIMENTO (##-####-######) TIPO MOV(I=Impotacin,E=Expotacin) CLAVE PEDIMENTO REGIMEN FECHA INICIO FECHA FINAL FECHA DE PAGO ADUANA Y SECCION DE CRUCE ACUSE ELECTRONICO INDIVIDUAL o CONSOLIDADO (IND,CON) MET TRANS ENTRADA MET TRANS ARRIVO MET TRANS SALIDA IEPS DTA CNT PREVALIDACION MONTO TIGIE PAGO IMPUESTO? (S/N) ES MIXTO (SI/NO) OBS RECTIFICA OPCION DESTINO(Interior del Pais/Regin Fronteriza/Franja Fronteriza) VALOR IVA VALOR ME VALOR ADUANAS FLETE VALOR SEGUROS SEGUROS EMBALAJES OTROS INCREMENTABLES ESTATUS (ABIERTO/CERRADO) PERSONA REV FECHA CIERRE FECHA REVISION FECHA AUTORIZACION FECHA RECIBIDO REPRESENTANTE AA CLAVE DEST ORIGEN FECHA ENTRADA RECINTO FECHA EXTRACCION RECINTO ERRORES FORMA PAGO DTA FORMA PAGO IGI FORMA PAGO PREVAL FORMA PAGO IVA RECARGOS MULTAS IVA DE PREV CUOTAS CONPENSATORIAS IDENTIFICADORES IEPS 2 FORMA DE PAGO IEPS 2 DTA 2 FORMA DE PAGO DTA 2 IVA 2 FORMA DE PAGO IVA 2 IGI 2 FORMA DE PAGO IGI 2 PREVALIDACION FORMA DE PAGO PREVALIDACION 2 CNT 2 FORMA DE PAGO CNT 2 diff --git a/frontend/static/csv/EstructuraCatTiposCambio.xls b/frontend/static/csv/EstructuraCatTiposCambio.xls deleted file mode 100644 index 916d24d3..00000000 --- a/frontend/static/csv/EstructuraCatTiposCambio.xls +++ /dev/null @@ -1 +0,0 @@ -FECHA (##/##/####) TIPO DE CAMBIO diff --git a/frontend/static/csv/EstructuraCatTrailers.xls b/frontend/static/csv/EstructuraCatTrailers.xls deleted file mode 100644 index 52f1e1d0..00000000 --- a/frontend/static/csv/EstructuraCatTrailers.xls +++ /dev/null @@ -1 +0,0 @@ -CLAVE TRAILER/CAJA NUMERO ACE TIPO DE TRAILER PRECINTO CODIGO DE ENTIDAD PLACAS ESTADO PAIS diff --git a/frontend/static/csv/EstructuraCatTransportes.xls b/frontend/static/csv/EstructuraCatTransportes.xls deleted file mode 100644 index 32093ebf..00000000 --- a/frontend/static/csv/EstructuraCatTransportes.xls +++ /dev/null @@ -1 +0,0 @@ -CLAVE CLAVE ACE CLAVE TRANSPORTE VIN TIPO TRANSPORTE CODIGO DE ENTIDAD TRANSPONDEDOR NUMERO DOT PLACAS CIUDAD ESTADO PAIS PRECINTO EMPRESA ASEGURADORA NUM. ASEGURADORA MONTO ASEGURADO FECHA DE ASEGURADORA diff --git a/frontend/static/csv/EstructuraEncFacExpoCamReg.xls b/frontend/static/csv/EstructuraEncFacExpoCamReg.xls deleted file mode 100644 index 0fe4fd4c..00000000 --- a/frontend/static/csv/EstructuraEncFacExpoCamReg.xls +++ /dev/null @@ -1 +0,0 @@ -PEDIMENTO REMESA NUMERO FACTURA FECHA FACTURA TIPO DE CAMBIO REGIMEN CLAVE PROVEEDOR CLAVE VENDIDO A: CLAVE ENVIADO A AGENTE ADUANAL CLAVE TRANSPORTISTA NOMBRE CONDUCTOR TIPO TRANSPORTE NUMERO TRANSPORTE TIPO MONEDA CLAVE MONEDA FLETES VALOR SEGUROS SEGUROS EMBALAJES OTROS INCREMENTABLES CLAVE INCOTERM PRECINTO TIPO PESO MANIFIESTO E-DOCUMENT NUM. OPERACION ENVIADO POR ADUANA DE CRUCE OBSERVACIONES E OBSERVACIONES I FACTURA ALTERNA diff --git a/frontend/static/csv/EstructuraEncFacImpoDef.xls b/frontend/static/csv/EstructuraEncFacImpoDef.xls deleted file mode 100644 index 2708c0ed..00000000 --- a/frontend/static/csv/EstructuraEncFacImpoDef.xls +++ /dev/null @@ -1 +0,0 @@ -PEDIMENTO REMESA NUMERO FACTURA FECHA FACTURA TIPO DE CAMBIO REGIMEN CLAVE PROVEEDOR CLAVE VENDIDO A: CLAVE ENVIADO A AGENTE ADUANAL CLAVE TRANSPORTISTA NOMBRE CONDUCTOR TIPO TRANSPORTE NUMERO TRANSPORTE TIPO MONEDA CLAVE MONEDA FLETES VALOR SEGUROS SEGUROS EMBALAJES OTROS INCREMENTABLES CLAVE INCOTERM PRECINTO FECHA EMISION TIPO PESO E-DOCUMENT NUM. OPERACION ADUANA DE CRUCE OBSERVACIONES E OBSERVACIONES I diff --git a/frontend/static/csv/EstructuraEncFacImpoTemp.xls b/frontend/static/csv/EstructuraEncFacImpoTemp.xls deleted file mode 100644 index 13fcbd22..00000000 --- a/frontend/static/csv/EstructuraEncFacImpoTemp.xls +++ /dev/null @@ -1 +0,0 @@ -PEDIMENTO REMESA NUMERO FACTURA FECHA FACTURA TIPO DE CAMBIO REGIMEN CLAVE PROVEEDOR CLAVE VENDIDO A: CLAVE ENVIADO A AGENTE ADUANAL CLAVE TRANSPORTISTA NOMBRE CONDUCTOR TIPO TRANSPORTE NUMERO TRANSPORTE TIPO MONEDA CLAVE MONEDA FLETES VALOR SEGUROS SEGUROS EMBALAJES OTROS INCREMENTABLES CLAVE INCOTERM PRECINTO FECHA EMISION TIPO PESO E-DOCUMENT NUM. OPERACION ADUANA DE CRUCE OBSERVACIONES E OBSERVACIONES I FACTURA ALTERNA diff --git a/frontend/static/csv/EstructuraParExpoCamReg.xls b/frontend/static/csv/EstructuraParExpoCamReg.xls deleted file mode 100644 index 76bd69af..00000000 --- a/frontend/static/csv/EstructuraParExpoCamReg.xls +++ /dev/null @@ -1 +0,0 @@ -NUMERO FACTURA EXPO. LINEA EXPO. TIPO DE IMPO. FACTURA IMPO. LINEA IMPO. GENERA DESCARGA CANTIDAD EXPORTADA/DESCARGAR COSTO UNITARIO PESO NETO PESO BRUTO SE PAGO IMPUESTO? (SI o NO) FORMA DE PAGO DESCRIPCION EXTRA INFORMACION ADICIONAL AGREGAR(A)/SUSTITUIR(S) LOTE NUMERO ENTRADA ES PARTIDA/SUBPARTIDA LINEA PRINCIPAL FRACCION AMERICANA FRACCION ARANCELARIA diff --git a/frontend/static/csv/EstructuraParFacImpoDefAF.xls b/frontend/static/csv/EstructuraParFacImpoDefAF.xls deleted file mode 100644 index 93bcae74..00000000 --- a/frontend/static/csv/EstructuraParFacImpoDefAF.xls +++ /dev/null @@ -1 +0,0 @@ -NUMERO FACTURA LINEA CLASE CANTIDAD IMPORTADA UNIDAD DE MEDIDA COSTO UNITARIO PESO NETO PESO BRUTO CANTIDAD BULTOS CLAVE BULTOS PAIS ORIGEN FRACCION ARANCELARIA PREFERENCIA ARANCELARIA SECTOR FRACCION AMERICANA ORDEN DE COMPRA DESCRIPCION ESPAOL DESCRIPCION INGLES MARCA MODELO ES PARTIDA O SUBPARTIDA LINEA PRINCIPAL NUM. PARTE SE PAGO IMPUESTO? (SI o NO) FORMA DE PAGO METODO DE VALORACION DESCRIPCION EXTRA INFORMACION ADICIONAL AGREGAR(A)/SUSTITUIR(S) VALOR TOTAL LOTE NUMERO ENTRADA ID TYPE diff --git a/frontend/static/csv/EstructuraParFacImpoTempAF.xls b/frontend/static/csv/EstructuraParFacImpoTempAF.xls deleted file mode 100644 index cc78d0b5..00000000 --- a/frontend/static/csv/EstructuraParFacImpoTempAF.xls +++ /dev/null @@ -1 +0,0 @@ -NUMERO FACTURA LINEA CLASE CANTIDAD IMPORTADA UNIDAD DE MEDIDA COSTO UNITARIO PESO NETO PESO BRUTO CANTIDAD BULTOS CLAVE BULTOS PAIS ORIGEN FRACCION ARANCELARIA PREFERENCIA ARANCELARIA SECTOR FRACCION AMERICANA ORDEN DE COMPRA DESCRIPCION ESPAOL DESCRIPCION INGLES MARCA MODELO ES PARTIDA O SUBPARTIDA LINEA PRINCIPAL NUM. PARTE SE PAGO IMPUESTO? (SI o NO) FORMA DE PAGO METODO DE VALORACION DESCRIPCION EXTRA INFORMACION ADICIONAL AGREGAR(A)/SUSTITUIR(S) TOTAL NUMERO ENTRADA LOTE ID TYPE diff --git a/frontend/static/silent-check-sso.html b/frontend/static/silent-check-sso.html new file mode 100644 index 00000000..455d67bc --- /dev/null +++ b/frontend/static/silent-check-sso.html @@ -0,0 +1,14 @@ + + + + Silent SSO Check + + + + + diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js index d3ef3f6a..72124dee 100644 --- a/frontend/svelte.config.js +++ b/frontend/svelte.config.js @@ -9,7 +9,7 @@ const config = { kit: { adapter: adapter(), csrf: { - checkOrigin: false + trustedOrigins: (process.env.TRUSTED_ORIGINS ?? process.env.CORS_ORIGINS ?? '').split(',').filter(Boolean) } } }; diff --git a/scripts/backend-entrypoint.sh b/scripts/backend-entrypoint.sh index 20c5f89f..85cb5432 100755 --- a/scripts/backend-entrypoint.sh +++ b/scripts/backend-entrypoint.sh @@ -19,7 +19,7 @@ wait_for_tcp() { echo "Esperando a que $service esté disponible en ${host}:${port}..." while [ $attempt -le $max_attempts ]; do - if python -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + if python3 -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then echo "✓ $service está listo y accesible" return 0 fi diff --git a/scripts/init_first_time.sh b/scripts/init_first_time.sh index 601d72d4..de1c32ae 100755 --- a/scripts/init_first_time.sh +++ b/scripts/init_first_time.sh @@ -12,11 +12,6 @@ # 6. Relación usuario-tenant en tabla user_tenants # 7. Actualización del tenant_id del usuario con el valor real # 8. Licencia Enterprise para el tenant (ilimitada, 1 año de vigencia) -# 9. [OPCIONAL] Datos iniciales de ejemplo si se pasa --seed-data -# -# Uso: -# ./init_first_time.sh # Solo configuración básica -# ./init_first_time.sh --seed-data # Configuración + datos de ejemplo # # Requisitos: # - Keycloak corriendo en http://localhost:8080 @@ -24,6 +19,8 @@ # - Base de datos anexo76_core creada # - jq instalado (para procesamiento JSON) # +# Puertos: Keycloak en 18080/19000, frontend en 15173, API en 18000. +# # Nota: El atributo tenant_id se crea con valor inicial "1" y luego # se actualiza con el ID real del tenant creado en PostgreSQL. ############################################################################### @@ -33,22 +30,6 @@ set -euo pipefail # Modo strict: exit on error, undefined vars, pipe failures # Trap para cleanup en caso de error trap 'echo -e "\n${RED}✗ Error en línea $LINENO. Script abortado.${NC}" >&2' ERR -# Parsear argumentos -SEED_DATA=false -while [[ $# -gt 0 ]]; do - case $1 in - --seed-data) - SEED_DATA=true - shift - ;; - *) - echo "Uso: $0 [--seed-data]" - echo " --seed-data: Carga datos de ejemplo en las tablas" - exit 1 - ;; - esac -done - # Colores para output RED='\033[0;31m' GREEN='\033[0;32m' @@ -144,172 +125,7 @@ create_tenant_mapper() { fi } -############################################################################### -# Funciones de seed data -############################################################################### - -# Insertar datos de ejemplo para customs_brokers -seed_customs_brokers() { - echo " → Insertando customs brokers..." - exec_pg_sql " - INSERT INTO a76.customs_brokers (tenant_id, company_id, type, broker_key, name, address, postal_code, city, state, phone, email, country, tax_id, license, company, contact, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB001', 'Agente Aduanal García', 'Av. Reforma 123', '01000', 'Ciudad de México', 'CDMX', '5555555555', 'garcia@aduanas.com', 'MEX', 'GAAR800101ABC', '1234', 'García y Asociados', 'Juan García', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB002', 'Agente Aduanal López', 'Blvd. Díaz Ordaz 456', '22000', 'Tijuana', 'BC', '6641234567', 'lopez@customs.com', 'MEX', 'LOPL750505XYZ', '2345', 'López Customs', 'María López', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB003', 'Agente Aduanal Martínez', 'Calle Industria 789', '45000', 'Guadalajara', 'JAL', '3339876543', 'martinez@broker.com', 'MEX', 'MARM850315DEF', '3456', 'Martínez Brokerage', 'Pedro Martínez', now(), now()) - ON CONFLICT (broker_key, tenant_id, company_id) DO NOTHING; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para clients_and_providers -seed_clients_and_providers() { - echo " → Insertando clientes y proveedores..." - exec_pg_sql " - INSERT INTO a76.clients_and_providers (tenant_id, company_id, type_nat_foreign, name, short_name, rfc, client_or_provider, web_key, is_active, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Proveedor Tecnológico SA de CV', 'PROVTECH', 'PTE901201ABC', 'BOTH', 'PROV001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Cliente Industrial del Norte SA', 'CINORTE', 'CIN850615XYZ', 'BOTH', 'CLI001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'E', 'Global Supplies Inc', 'GLOBSUP', 'GSI123456789', 'BOTH', 'BOTH001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Manufacturas del Bajío SA', 'MANBAJIO', 'MDB920310DEF', 'BOTH', 'CLI002', true, now(), now()) - RETURNING id; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para packages -seed_packages() { - echo " → Insertando tipos de paquete..." - exec_pg_sql " - INSERT INTO a76.packages (tenant_id, company_id, key, description_es, description_en, weight_unit, plurals, plural_in, code_ace, code_aamex, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'PK01', 'Caja de Cartón', 'Cardboard Box', 0.5, 'CAJS', 'BOXS', 'CB01', 'CAJA001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK02', 'Pallet de Madera', 'Wooden Pallet', 15.0, 'PLTS', 'PLTS', 'WP01', 'PALL001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK03', 'Tambor Metálico', 'Metal Drum', 10.0, 'TMBS', 'DRMS', 'MD01', 'TAMB001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK04', 'Contenedor', 'Container', 2000.0, 'CONT', 'CONT', 'CT01', 'CONT001', now(), now()) - ON CONFLICT (tenant_id, company_id, key) DO NOTHING; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para classes -seed_classes() { - echo " → Insertando clases..." - exec_pg_sql "INSERT INTO a76.classes (tenant_id, company_id, class_code, description_es, description_en, material_key, unit_of_measure, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'CLS001', 'Componentes Electrónicos', 'Electronic Components', 'MP', 'PZA', '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS002', 'Partes Automotrices', 'Automotive Parts', 'MP', 'KGS', '8708.29.99', '8708.29.9900', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS003', 'Textiles y Telas', 'Textiles and Fabrics', 'MP', 'MT', '5407.20.01', '5407.20.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS004', 'Equipo de Computación', 'Computer Equipment', 'MP', 'PZA', '8471.30.01', '8471.30.0100', now(), now()) ON CONFLICT (tenant_id, company_id, class_code) DO NOTHING;" >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para parts -seed_parts() { - echo " → Insertando partes/componentes..." - - # Obtener un client_id para asociar las partes - local client_id - client_id=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} LIMIT 1;" | xargs) - - if [ -n "$client_id" ]; then - exec_pg_sql "INSERT INTO a76.parts (tenant_id, company_id, client_id, part_number, description_spanish, description_english, part_class, currency_key, unit_of_measure, unit_cost, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-001', 'Microcontrolador ARM Cortex-M4', 'ARM Cortex-M4 Microcontroller', 'CLS001', 'USD', 'PZA', 15.50, '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-002', 'Filtro de Aceite Automotriz', 'Automotive Oil Filter', 'CLS002', 'USD', 'PZA', 8.75, '8421.23.01', '8421.23.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-003', 'Tela de Algodón para Tapicería', 'Cotton Upholstery Fabric', 'CLS003', 'USD', 'MT', 12.00, '5208.31.01', '5208.31.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-004', 'Disco Duro SSD 500GB', '500GB SSD Hard Drive', 'CLS004', 'USD', 'PZA', 65.00, '8471.70.01', '8471.70.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-005', 'Sensor de Temperatura Digital', 'Digital Temperature Sensor', 'CLS001', 'USD', 'PZA', 5.25, '9025.19.01', '9025.19.0100', now(), now()) ON CONFLICT (tenant_id, company_id, part_number) DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Insertar datos de ejemplo para pedimentos -seed_pedimentos() { - echo " → Insertando pedimentos..." - - # Primero obtener IDs de clientes - local client_ids - client_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('CLIENT', 'BOTH') LIMIT 2;") - local client_id_1=$(echo "$client_ids" | sed -n '1p' | xargs) - local client_id_2=$(echo "$client_ids" | sed -n '2p' | xargs) - - if [ -n "$client_id_1" ]; then - exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001234', ${client_id_1}, 'imp', 'normal', 'V1', 'ITE', 'draft', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001235', ${client_id_1}, 'exp', 'normal', 'V1', 'ETE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$client_id_2" ]; then - exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001236', ${client_id_2}, 'imp', 'consolidated', 'V1', 'ITE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Insertar datos de ejemplo para invoices -seed_invoices() { - echo " → Insertando facturas..." - - # Obtener IDs de clientes/proveedores - local provider_ids - provider_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('PROVIDER', 'BOTH') LIMIT 2;") - local provider_id_1=$(echo "$provider_ids" | sed -n '1p' | xargs) - local provider_id_2=$(echo "$provider_ids" | sed -n '2p' | xargs) - - # Insertar facturas en invoice_header (mínimo requerido) - exec_pg_sql "INSERT INTO a76.invoice_header (tenant_id, company_id, system, operation_type, invoice_type, invoice_number, invoice_date, is_updated, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'DEF', 'INV-2024-001', '2024-01-10', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'exp', 'EXDEF', 'INV-2024-002', '2024-02-15', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'TEM', 'INV-2024-003', '2024-03-05', false, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - - # Obtener IDs de las facturas recién creadas - local invoice_ids - invoice_ids=$(exec_pg_sql "SELECT id FROM a76.invoice_header WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND invoice_number IN ('INV-2024-001', 'INV-2024-002', 'INV-2024-003') ORDER BY invoice_number;") - local invoice_id_1=$(echo "$invoice_ids" | sed -n '1p' | xargs) - local invoice_id_2=$(echo "$invoice_ids" | sed -n '2p' | xargs) - local invoice_id_3=$(echo "$invoice_ids" | sed -n '3p' | xargs) - - # Insertar datos financieros para las facturas - if [ -n "$invoice_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'foreign', 17.50, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'foreign', 17.45, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'local', 1.00, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - # Insertar datos de compliance mexicano con proveedores - if [ -n "$invoice_id_1" ] && [ -n "$provider_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ] && [ -n "$provider_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ] && [ -n "$provider_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, ${provider_id_2}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - # Insertar datos de logística con incoterm - if [ -n "$invoice_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'none', 'kgs', 'FOB', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'none', 'kgs', 'CIF', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'none', 'kgs', 'EXW', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Ejecutar todas las funciones de seed -execute_seed_data() { - echo -e "\n${YELLOW}[SEED] Cargando datos de ejemplo...${NC}" - - seed_customs_brokers - seed_clients_and_providers - seed_packages - seed_classes - seed_parts - seed_pedimentos - seed_invoices - - echo -e "${GREEN}✓ Datos de ejemplo cargados exitosamente${NC}" - echo -e "${YELLOW} • 3 Agentes aduanales${NC}" - echo -e "${YELLOW} • 4 Clientes/Proveedores${NC}" - echo -e "${YELLOW} • 4 Tipos de paquete${NC}" - echo -e "${YELLOW} • 4 Clases${NC}" - echo -e "${YELLOW} • 5 Parts/Componentes${NC}" - echo -e "${YELLOW} • 3 Pedimentos${NC}" - echo -e "${YELLOW} • 3 Facturas${NC}" -} - -# Variables de configuración +# Variables de configuración (puertos con prefijo 1 hardcodeados) KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080/kcauth}" KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}" KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}" @@ -328,7 +144,7 @@ DEMO_EMAIL="demo@aduanasoft.com" DEMO_FIRSTNAME="Demo" DEMO_LASTNAME="User" -TENANT_NAME="Aduanasoft" +TENANT_NAME="Aduanasoft"A TENANT_SLUG="aduanasoft" COMPANY_NAME="Aduanasoft S.A. de C.V." COMPANY_RFC="ADS010101AAA" @@ -797,12 +613,14 @@ else echo -e "${YELLOW}⚠ Company ya existe para este tenant${NC}" fi -# Obtener información de la company -COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;") +# Obtener ID de la company (necesario para user_tenants) COMPANY_ID=$(exec_pg_sql "SELECT id FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;" | xargs) - -echo -e "${GREEN}✓ Company: ${COMPANY_INFO}${NC}" -echo -e "${GREEN}✓ Company ID: ${COMPANY_ID}${NC}" +if [ -z "$COMPANY_ID" ]; then + echo -e "${RED}✗ Error: No se pudo obtener el ID de la company${NC}" + exit 1 +fi +COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;") +echo -e "${GREEN}✓ Company ID: ${COMPANY_ID} | ${COMPANY_INFO}${NC}" # Agregar tenant_id al usuario demo en Keycloak echo -e "\n${YELLOW}Asignando tenant_id al usuario demo...${NC}" @@ -818,7 +636,7 @@ curl -s -X PUT "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/users/${USER_ID}" echo -e "${GREEN}✓ Atributo tenant_id asignado al usuario${NC}" -# Agregar relación usuario-tenant en la base de datos +# Agregar relación usuario-tenant en la base de datos (usar company_id real) echo -e "\n${YELLOW}Creando relación usuario-tenant en la base de datos...${NC}" exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null @@ -880,13 +698,6 @@ else fi fi -############################################################################### -# 9. Cargar datos de ejemplo (opcional) -############################################################################### -if [ "$SEED_DATA" = true ]; then - execute_seed_data -fi - ############################################################################### # Resumen final ############################################################################### @@ -925,18 +736,6 @@ echo -e " ${GREEN}✓${NC} Plan: Enterprise (ilimitado)" echo -e " ${GREEN}✓${NC} Status: Activa" echo -e " ${GREEN}✓${NC} Features: API, Reportes Avanzados, Integraciones, Soporte Dedicado" echo -e " ${GREEN}✓${NC} Vigencia: 1 año" - -if [ "$SEED_DATA" = true ]; then -echo -e "\n${YELLOW}Datos de ejemplo:${NC}" -echo -e " ${GREEN}✓${NC} Agentes aduanales: 3" -echo -e " ${GREEN}✓${NC} Clientes/Proveedores: 4" -echo -e " ${GREEN}✓${NC} Tipos de paquete: 4" -echo -e " ${GREEN}✓${NC} Clases: 4" -echo -e " ${GREEN}✓${NC} Parts/Componentes: 5" -echo -e " ${GREEN}✓${NC} Pedimentos: 3" -echo -e " ${GREEN}✓${NC} Facturas: 3" -fi - echo -e "\n${YELLOW}Puedes acceder al sistema en:${NC}" echo -e " ${GREEN}http://localhost:5173${NC}" echo -e "\n${GREEN}════════════════════════════════════════════════════════${NC}\n"