From 0683aaa8015219f42b73b1e979ccf7c68ed8e424 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 23 Feb 2026 13:05:02 -0600 Subject: [PATCH 01/16] feature/csv-for-envoices --- .gitignore | 5 +- backend/api/__init__.py | 0 backend/api/v1/__init__.py | 0 backend/api/v1/modules/__init__.py | 0 backend/api/v1/modules/a76/__init__.py | 0 backend/api/v1/modules/a76/imports/routes.py | 141 ++- backend/api/v1/modules/a76/imports/tasks.py | 810 +++++++++++++++--- .../v1/modules/a76/imports/template_config.py | 118 +++ .../modules/a76/invoices/catalog_service.py | 1 + .../api/v1/modules/a76/invoices/schemas.py | 12 +- .../api/v1/modules/a76/invoices/services.py | 15 +- backend/api/v1/modules/a76/router.py | 2 +- backend/core/celery_app.py | 7 + backend/core/error_handlers.py | 37 +- backend/main.py | 29 +- frontend/src/lib/api.ts | 33 + .../csv-upload/ProcessingResultModal.svelte | 42 +- .../csv-upload/UploadLauncherGrid.svelte | 2 +- .../dashboard/customs_brokers/columns.ts | 14 + frontend/src/lib/config/csv-upload.ts | 12 +- .../routes/dashboard/csv-upload/+page.svelte | 207 +++-- 21 files changed, 1232 insertions(+), 255 deletions(-) create mode 100644 backend/api/__init__.py create mode 100644 backend/api/v1/__init__.py create mode 100644 backend/api/v1/modules/__init__.py create mode 100644 backend/api/v1/modules/a76/__init__.py create mode 100644 backend/api/v1/modules/a76/imports/template_config.py diff --git a/.gitignore b/.gitignore index 48bbc048..ff9b622c 100644 --- a/.gitignore +++ b/.gitignore @@ -23,9 +23,11 @@ wheels/ *.egg .pnpm-store/ -# Environment +# Environment (no subir: cada quien puede usar puertos distintos vía .env) .env .env.local +backend/.env +frontend/.env backend/SCRIPTS/ # IDEs .vscode/ @@ -62,4 +64,3 @@ node_modules/ *.dockerignore postgres-data/ backend/uploads/ -docker-compose.yml diff --git a/backend/api/__init__.py b/backend/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/__init__.py b/backend/api/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/__init__.py b/backend/api/v1/modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/a76/__init__.py b/backend/api/v1/modules/a76/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/v1/modules/a76/imports/routes.py b/backend/api/v1/modules/a76/imports/routes.py index 1ea76a74..f3c3d9ee 100644 --- a/backend/api/v1/modules/a76/imports/routes.py +++ b/backend/api/v1/modules/a76/imports/routes.py @@ -1,5 +1,6 @@ from datetime import datetime from uuid import uuid4 +import base64 import os import json import logging @@ -12,24 +13,39 @@ from core.config import settings from core.database import get_core_db from core.security import get_current_user, validate_access_to_resource -from .tasks import scan_file, insert_valid_rows +from .tasks import ( + scan_file, + insert_valid_rows, + IMPORT_FILE_KEY_PREFIX, + IMPORT_META_KEY_PREFIX, + IMPORT_REDIS_TTL, +) from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest router = APIRouter() logger = logging.getLogger(__name__) + +def _get_redis(): + """Redis client (same broker as Celery so worker can read).""" + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + @router.post("/upload/{model_target}", response_model=ImportJobResponse) async def upload_import_file( model_target: Literal["invoice_header", "invoice_details"], file: UploadFile = File(...), footer_config: Optional[str] = Form(None), # JSON string with settings - company_id: int = Query(..., description="Company ID"), # Required for context + template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas + company_id: int = Query(..., description="Company ID"), # Required for context operation_type: Optional[str] = Query("imp"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): """ Step 1: Upload CSV, save to temp, trigger scan task. + Si se envía template_id, solo se leen las columnas de esa plantilla. """ # 1. Validate Access & Get Tenant try: @@ -40,40 +56,51 @@ async def upload_import_file( if not file.filename.endswith(".csv"): raise HTTPException(status_code=400, detail="Only .csv files allowed") - + job_id = str(uuid4()) - - # Ensure directory exists (Safety check) - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") - os.makedirs(upload_dir, exist_ok=True) - - file_path = os.path.join(upload_dir, f"{job_id}.csv") - meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") - + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "footer_config": footer_config, + "operation_type": operation_type, + "template_id": template_id, + } + + # Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed) try: - # Save CSV - contents = await file.read() + redis_client = _get_redis() + redis_client.set( + f"{IMPORT_FILE_KEY_PREFIX}{job_id}", + base64.b64encode(contents), + ex=IMPORT_REDIS_TTL, + ) + redis_client.set( + f"{IMPORT_META_KEY_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Redis store error: {e}") + raise HTTPException(status_code=500, detail="Failed to queue file for processing.") + + # Optional: also write to local disk (e.g. for same-machine worker or debugging) + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"{job_id}.csv") + meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") with open(file_path, "wb") as f: f.write(contents) - - # Save Metadata (Context) - meta_data = { - "tenant_id": tenant_id, - "company_id": company_id, - "user_id": current_user.get("id"), - "footer_config": footer_config, - "operation_type": operation_type, - } with open(meta_path, "w") as f: json.dump(meta_data, f) - except Exception as e: - logger.error(f"File save error: {e}") - raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}") + logger.warning(f"Local file save failed (worker will use Redis): {e}") - # Trigger Celery Task (Async) - # Use our job_id as the Celery task_id for easier tracking - scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id) + # Trigger Celery Task (Async). Worker loads file from Redis. + scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id) return ImportJobResponse( job_id=job_id, @@ -84,24 +111,56 @@ async def upload_import_file( @router.get("/{job_id}/status") async def get_import_status(job_id: str): """ - Poll this endpoint to get % progress or final report. + Poll to get progress or final report. Always returns an object with "status". """ - # In a real app, query Redis or DB. - # For MVP, we might mock or use Celery AsyncResult if backend shares Redis. task_result = celery_app.AsyncResult(job_id) - - if task_result.state == 'PENDING': + + if task_result.state == "PENDING": return {"status": "processing", "progress": 0} - elif task_result.state == 'PROGRESS': + if task_result.state == "PROGRESS": return { - "status": "processing", - "progress": task_result.info.get('current', 0), - "total": task_result.info.get('total', 0) + "status": "processing", + "progress": (task_result.info or {}).get("current", 0), + "total": (task_result.info or {}).get("total", 0), } - elif task_result.state == 'SUCCESS': - return task_result.result # Should return the report - else: - return {"status": task_result.state, "error": str(task_result.info)} + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + # FAILURE: obtener mensaje real (traceback, result o get(propagate=False)) + logger.warning("Import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb: + logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg and len(lines) > 1: + err_msg = lines[-2] + " " + (lines[-1] or "") + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg: + result = getattr(task_result, "result", None) + info = getattr(task_result, "info", None) + if result is not None and not isinstance(result, dict): + err_msg = str(result) + elif isinstance(result, dict) and (result.get("error") or result.get("message")): + err_msg = result.get("error") or result.get("message") + if not err_msg and isinstance(info, str): + err_msg = info + elif not err_msg and isinstance(info, dict) and "error" in info: + err_msg = str(info["error"]) + if not err_msg: + err_msg = "Task failed" + return {"status": "failed", "error": err_msg} @router.post("/{job_id}/commit") diff --git a/backend/api/v1/modules/a76/imports/tasks.py b/backend/api/v1/modules/a76/imports/tasks.py index dfb82e74..2acde6a5 100644 --- a/backend/api/v1/modules/a76/imports/tasks.py +++ b/backend/api/v1/modules/a76/imports/tasks.py @@ -1,4 +1,5 @@ import os +import base64 from datetime import datetime from decimal import Decimal import csv @@ -6,9 +7,12 @@ import json import logging import re import unicodedata -from celery import shared_task -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app from core.database import CoreSessionLocal + +from .template_config import row_from_template # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process # We'll need schemas for validation @@ -17,6 +21,78 @@ from core.database import CoreSessionLocal logger = logging.getLogger(__name__) +# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem) +IMPORT_FILE_KEY_PREFIX = "import_file:" +IMPORT_META_KEY_PREFIX = "import_meta:" +IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" +IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + """Redis client using same URL as Celery broker (worker and API can share data).""" + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + """Directory on the worker for temp CSV and meta (same structure as API, but local to worker).""" + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + """ + Load file content from Redis and write to worker's upload dir. + Returns local file_path if successful, None otherwise. + """ + redis_client = _get_redis() + key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}" + data = redis_client.get(key) + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Failed to decode import file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + """Load meta from Redis and write to worker's meta file. Returns True if meta was found and written.""" + redis_client = _get_redis() + key = f"{IMPORT_META_KEY_PREFIX}{job_id}" + data = redis_client.get(key) + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Failed to decode import meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + """Remove file, meta and error lines from Redis after commit (cleanup).""" + try: + r = _get_redis() + r.delete( + f"{IMPORT_FILE_KEY_PREFIX}{job_id}", + f"{IMPORT_META_KEY_PREFIX}{job_id}", + f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Failed to delete import keys from Redis: {e}") + class ForeignKeyValidator: def __init__(self, session, tenant_id, company_id): self.session = session @@ -40,26 +116,90 @@ class ForeignKeyValidator: self.cache[key] = exists return exists -@shared_task(bind=True) -def scan_file(self, job_id: str, file_path: str, model_target: str, config: str = None): + +TRANSPORT_TYPE_VALUES = { + "none", + "transport", + "box", + "licence plates", + "truck", + "vessel", + "rail_barge", + "container", + "airplane", + "gondola", + "flatbed", +} + + +def normalize_public_code(value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = str(value).strip().upper() + return text or None + + +def validate_public_code( + validator: ForeignKeyValidator, + model, + value: Optional[str], + line_num: int, + col_name: str, + field_name: str = "code", + required: bool = False, +) -> Optional[Dict[str, Any]]: + code = normalize_public_code(value) + if not code: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, code, field_name=field_name, is_public=True): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + + +def validate_tenant_fk_id( + validator: ForeignKeyValidator, + model, + value: Optional[int], + line_num: int, + col_name: str, + required: bool = False, +) -> Optional[Dict[str, Any]]: + if value is None: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, value): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, model_target: str, config: str = None): """ Pass 1: Read CSV, Validate types, Write Errors to JSONL. + File content is loaded from Redis (written by API on upload) so worker does not need shared filesystem. """ logger.info(f"Starting scan for job {job_id} target {model_target}") - - # 1. Setup Error Log + + # 1. Get file from Redis and write to worker local disk + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + # 2. Setup Error Log error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") os.makedirs(os.path.dirname(error_path), exist_ok=True) - + total_rows = 0 error_count = 0 processed_rows = 0 - - # 2. Count Total (Quick Pass) or just estimate - # For better progress, we can get file line count first + + # 3. Count Total (Quick Pass) or just estimate try: with open(file_path, 'r', encoding='utf-8-sig') as f: - total_rows = sum(1 for _ in f) - 1 # Minus header + total_rows = sum(1 for _ in f) - 1 # Minus header except Exception as e: return {"status": "failed", "error": f"Cannot read file: {e}"} @@ -70,10 +210,65 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + tenant_id = None + company_id = None + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f_meta: + meta = json.load(f_meta) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception as e: + logger.warning(f"Failed to read meta for job {job_id}: {e}") + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Missing context (tenant/company)"} + + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + + inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM") + if not inv_type_value: + inv_type_value = "TEM" try: - with open(file_path, 'r', encoding='utf-8-sig') as f_in, \ + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, + ) + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm + from api.v1.modules.a76.parts.models import Part + + models = { + "InvoiceHeader": InvoiceHeader, + "InvoiceType": InvoiceType, + "ClientProvider": ClientProvider, + "CustomsBroker": CustomsBroker, + "RegimenPedimento": RegimenPedimento, + "CodePedimentoRegimen": CodePedimentoRegimen, + "PedimentoCode": PedimentoCode, + "CurrencyType": CurrencyType, + "CustomsSection": CustomsSection, + "Incoterm": Incoterm, + "Part": Part, + } + + with CoreSessionLocal() as session, \ + open(file_path, 'r', encoding='utf-8-sig') as f_in, \ open(error_path, 'w', encoding='utf-8') as f_err: + validator = ForeignKeyValidator(session, tenant_id, company_id) + invoice_id_cache: Dict[str, Optional[int]] = {} # Detect Delimiter sample = f_in.read(2048) @@ -94,9 +289,18 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str 'errors': error_count }) - # Validation (Phase 1: Minimal) - row_norm = normalize_row(row) - errors = validate_row_phase_1(row_norm, model_target, i, date_format) + # Solo columnas de la plantilla (respetar plantilla tal cual) + row_norm = row_from_template(row, template_id, normalize_header) + errors = validate_row_strict( + row_norm, + model_target, + i, + date_format, + validator, + inv_type_value, + invoice_id_cache, + models, + ) if errors: error_count += 1 @@ -109,14 +313,45 @@ def scan_file(self, job_id: str, file_path: str, model_target: str, config: str logger.error(f"Scan failed: {e}") return {"status": "failed", "error": str(e)} - # 4. Result + # 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker) + error_lines_list = [] + errors_detail: List[Dict[str, Any]] = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + if len(errors_detail) < 500: + errors_detail.append( + { + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + } + ) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Failed to store error lines in Redis: {e}") + + # 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV) return { "status": "waiting_confirmation", "job_id": job_id, "total_rows": processed_rows, "error_count": error_count, "valid_rows": processed_rows - error_count, - "error_file": error_path + "errors": errors_detail, } def validate_row_phase_1( @@ -124,13 +359,61 @@ def validate_row_phase_1( target: str, line_num: int, date_format: Optional[str], -) -> Dict[str, Any]: +) -> Optional[Dict[str, Any]]: """ - Minimal validation: Unique IDs and Dates. + Validation: Unique IDs, Dates, and Numeric constraint checks. Target: 'invoice_header' or 'invoice_details' """ - errors = {} - + def check_decimal(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_decimal(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"} + return None + + def check_int(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_int(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"} + return None + + def check_date(col_name): + date_str = row.get(col_name) + if date_str and str(date_str).strip(): + if not is_valid_date(date_str, date_format): + expected = display_date_format(date_format) + return { + "line": line_num, + "col": col_name, + "msg": f"Formato de fecha inválido ({expected})", + } + return None + + def check_weight(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_weight_unit(val) is None: + return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"} + return None + + def check_currency(col_name): + val = row.get(col_name) + if val and str(val).strip(): + parsed_currency = parse_currency(val, None) + val_norm = normalize_header(val) + # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"} + return None + + def check_transport_type(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES: + return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"} + return None + # A. Invoice Header if target == 'invoice_header': # 1. Unique ID @@ -138,33 +421,180 @@ def validate_row_phase_1( return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} # 2. Date Format - date_str = row.get('FECHA FACTURA') - if date_str: - if not is_valid_date(date_str, date_format): - expected = display_date_format(date_format) - return { - "line": line_num, - "col": "FECHA FACTURA", - "msg": f"Formato inválido ({expected})", - } - else: + date_str = row.get('FECHA FACTURA') or row.get('FECHA') + if not date_str or not str(date_str).strip(): return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"} + + err = check_date('FECHA FACTURA') or check_date('FECHA') + if err: return err + + err = check_date('FECHA EMISION') + if err: return err + + # 3. Numeric Fields + for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']: + err = check_decimal(col) + if err: return err + + # 4. Integer FKs + for col in ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA']: + err = check_int(col) + if err: return err + + # 5. Enums + for col in ['TIPO PESO']: + err = check_weight(col) + if err: return err + + for col in ['TIPO MONEDA']: + err = check_currency(col) + if err: return err + + for col in ['TIPO TRANSPORTE']: + err = check_transport_type(col) + if err: return err # B. Invoice Details (Parts) elif target == 'invoice_details': # 1. Line Number - if not row.get('LINEA'): + if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'): return {"line": line_num, "col": "LINEA", "msg": "Requerido"} # 2. Parent Link (Invoice Number) if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')): return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} - - # 2. Parent Link (Simplified for now, we assume parent exists or is in same batch) - # In a real scenario, we'd check if the invoice exists. - pass + + # 3. Numeric Fields + for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']: + err = check_decimal(col) + if err: return err + + for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']: + err = check_int(col) + if err: return err - return errors if errors else None + return None + + +def validate_row_strict( + row: Dict[str, Any], + target: str, + line_num: int, + date_format: Optional[str], + validator: ForeignKeyValidator, + inv_type_value: str, + invoice_id_cache: Dict[str, Optional[int]], + models: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + err = validate_row_phase_1(row, target, line_num, date_format) + if err: + return err + + InvoiceHeader = models["InvoiceHeader"] + InvoiceType = models["InvoiceType"] + ClientProvider = models["ClientProvider"] + CustomsBroker = models["CustomsBroker"] + RegimenPedimento = models["RegimenPedimento"] + CurrencyType = models["CurrencyType"] + CustomsSection = models["CustomsSection"] + Incoterm = models["Incoterm"] + Part = models["Part"] + + if target == "invoice_header": + if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): + return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"} + + provider_id = parse_int(row.get("CLAVE PROVEEDOR")) + err = validate_tenant_fk_id(validator, ClientProvider, provider_id, line_num, "CLAVE PROVEEDOR", required=True) + if err: + return err + + sold_to_id = parse_int(row.get("CLAVE VENDIDO A")) + err = validate_tenant_fk_id(validator, ClientProvider, sold_to_id, line_num, "CLAVE VENDIDO A", required=True) + if err: + return err + + shipped_to_id = parse_int(row.get("CLAVE ENVIADO A")) + err = validate_tenant_fk_id(validator, ClientProvider, shipped_to_id, line_num, "CLAVE ENVIADO A", required=True) + if err: + return err + + broker_id = parse_int(row.get("AGENTE ADUANAL")) + err = validate_tenant_fk_id(validator, CustomsBroker, broker_id, line_num, "AGENTE ADUANAL") + if err: + return err + + err = validate_public_code( + validator, + RegimenPedimento, + row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"), + line_num, + "CLAVEDOCUMENTO", + ) + if err: + return err + + err = validate_public_code( + validator, + CustomsSection, + row.get("ADUANA DE CRUCE"), + line_num, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + return err + + err = validate_public_code( + validator, + CurrencyType, + row.get("CLAVE MONEDA"), + line_num, + "CLAVE MONEDA", + ) + if err: + return err + + err = validate_public_code( + validator, + Incoterm, + row.get("CLAVE INCOTERM"), + line_num, + "CLAVE INCOTERM", + ) + if err: + return err + + elif target == "invoice_details": + invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() + if not invoice_number: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} + + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] + else: + invoice_id = ( + validator.session.query(InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == validator.tenant_id, + InvoiceHeader.company_id == validator.company_id, + InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, + ) + .scalar() + ) + invoice_id_cache[cache_key] = invoice_id + if not invoice_id: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"} + + part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip() + if not part_num: + return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} + if not validator.check_exists(Part, part_num, field_name="part_number"): + return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"} + + return None def parse_footer_config(config: Optional[str]) -> Dict[str, Any]: if not config: @@ -247,6 +677,16 @@ def parse_decimal(value: Any) -> Optional[Decimal]: return None +def decimal_or_zero(value: Any) -> Decimal: + """Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default).""" + return parse_decimal(value) or Decimal("0") + + +def int_or_zero(value: Any) -> int: + """Return parsed int or 0 for CSV nulls/empty (vanilla default).""" + return parse_int(value) if parse_int(value) is not None else 0 + + def parse_currency(value: Optional[str], currency_type: Optional[str]): from api.v1.modules.a76.invoices.models import Currency if value: @@ -319,13 +759,24 @@ def resolve_public_code( cache[normalized] = normalized if exists is not None else None return cache[normalized] -@shared_task(bind=True) +@celery_app.task(bind=True) def insert_valid_rows(self, job_id: str, model_target: str): """ Pass 2: Re-read CSV, Skip Errors, Bulk Insert. + File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem. """ logger.info(f"Starting Commit for {job_id} target {model_target}") - + + # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + upload_dir = _worker_upload_dir() + file_path = os.path.join(upload_dir, f"{job_id}.csv") + if not os.path.exists(file_path): + return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + try: from api.v1.modules.a76.invoices.models import ( InvoiceHeader, @@ -334,16 +785,18 @@ def insert_valid_rows(self, job_id: str, model_target: str): InvoiceLogistics, InvoiceSalesDetails, OperationType, + TransportType, WeightUnit, ) from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.a76.items.models import Item from api.v1.modules.a76.items.line_items.models import LineItem @@ -353,30 +806,37 @@ def insert_valid_rows(self, job_id: str, model_target: str): from api.v1.modules.a76.items.line_descriptions.models import LineDescription from api.v1.modules.a76.parts.models import Part - upload_dir = os.path.join(os.getcwd(), "uploads", "temp") - file_path = os.path.join(upload_dir, f"{job_id}.csv") error_path = file_path.replace("temp", "errors").replace(".csv", ".jsonl") - - # 1. Load Error Line Numbers + + # 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file) error_lines = set() - if os.path.exists(error_path): + try: + r = _get_redis() + raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): with open(error_path, 'r', encoding='utf-8') as f: for line in f: try: err = json.loads(line) error_lines.add(err['line']) - except: pass + except Exception: + pass # Load Metadata (Context) - meta_path = file_path.replace("temp", "temp").replace(".csv", ".meta.json") + meta_path = file_path.replace(".csv", ".meta.json") tenant_id = None company_id = None footer_config = {} + meta = {} if os.path.exists(meta_path): try: with open(meta_path, 'r') as f: - meta = json.load(f) + meta = json.load(f) or {} tenant_id = meta.get('tenant_id') company_id = meta.get('company_id') operation_type_raw = meta.get('operation_type', 'imp') @@ -408,7 +868,7 @@ def insert_valid_rows(self, job_id: str, model_target: str): # Default types from config or fallback op_type_value = OperationType(meta.get('operation_type', 'imp').lower()) - inv_type_value = footer_config.get('invoice_type', 'TEM') + inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM' logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") @@ -437,13 +897,17 @@ def insert_valid_rows(self, job_id: str, model_target: str): reader = csv.DictReader(f, dialect=dialect) + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + for i, row in enumerate(reader, start=1): if i in error_lines: continue - row_norm = normalize_row(row) + row_norm = row_from_template(row, template_id, normalize_header) - # Mapping Logic + # Mapping Logic (solo campos que acepta el modelo de facturas) if model_target == 'invoice_header': invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format) @@ -462,25 +926,148 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue - - # 2. Client/Provider (Tenant) + provider_id = parse_int(row_norm.get('CLAVE PROVEEDOR')) - if provider_id and not validator.check_exists(ClientProvider, provider_id): - skipped_missing_fk += 1 - reason = f"Proveedor ID '{provider_id}' no existe" + err = validate_tenant_fk_id( + validator, + ClientProvider, + provider_id, + i, + "CLAVE PROVEEDOR", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue - # 3. Customs Broker (Tenant) - broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) - if broker_id and not validator.check_exists(CustomsBroker, broker_id): - skipped_missing_fk += 1 - reason = f"Agente Aduanal ID '{broker_id}' no existe" + sold_to_id = parse_int(row_norm.get('CLAVE VENDIDO A')) + err = validate_tenant_fk_id( + validator, + ClientProvider, + sold_to_id, + i, + "CLAVE VENDIDO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue + shipped_to_id = parse_int(row_norm.get('CLAVE ENVIADO A')) + err = validate_tenant_fk_id( + validator, + ClientProvider, + shipped_to_id, + i, + "CLAVE ENVIADO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) + err = validate_tenant_fk_id( + validator, + CustomsBroker, + broker_id, + i, + "AGENTE ADUANAL", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + RegimenPedimento, + row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'), + i, + "CLAVEDOCUMENTO", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CustomsSection, + row_norm.get('ADUANA DE CRUCE'), + i, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CurrencyType, + row_norm.get('CLAVE MONEDA'), + i, + "CLAVE MONEDA", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + Incoterm, + row_norm.get('CLAVE INCOTERM'), + i, + "CLAVE INCOTERM", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + transport_type_val = row_norm.get('TIPO TRANSPORTE') + if transport_type_val and str(transport_type_val).strip().lower() not in TRANSPORT_TYPE_VALUES: + skipped_invalid += 1 + reason = "TIPO TRANSPORTE: Tipo de transporte invalido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + currency_val = row_norm.get('TIPO MONEDA') + if currency_val and str(currency_val).strip(): + parsed_currency = parse_currency(currency_val, None) + val_norm = normalize_header(currency_val) + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + skipped_invalid += 1 + reason = "TIPO MONEDA: Moneda invalida" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + # 2. Client/Provider and broker checks are handled above + # --- 4. Check for Existing Invoice (Upsert Logic) --- existing_header = None if invoice_number: @@ -609,12 +1196,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): financials = InvoiceFinancials( currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type), currency_type=financials_currency_type, - exchange_rate=parse_decimal(row_norm.get('TIPO DE CAMBIO')), - freight=parse_decimal(row_norm.get('FLETES')), - insurance_value=parse_decimal(row_norm.get('VALOR SEGUROS')), - insurance=parse_decimal(row_norm.get('SEGUROS')), - packaging=parse_decimal(row_norm.get('EMBALAJES')), - other_increments=parse_decimal(row_norm.get('OTROS INCREMENTABLES')), + exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')), + freight=decimal_or_zero(row_norm.get('FLETES')), + insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')), + insurance=decimal_or_zero(row_norm.get('SEGUROS')), + packaging=decimal_or_zero(row_norm.get('EMBALAJES')), + other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')), tenant_id=tenant_id, company_id=company_id, ) @@ -622,10 +1209,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) logistics = None if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): + raw_transport = (row_norm.get('TIPO TRANSPORTE') or "none") + transport_str = str(raw_transport).strip().lower() or "none" + try: + transport_type = TransportType(transport_str) + except ValueError: + transport_type = TransportType.NONE logistics = InvoiceLogistics( carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), - transport_type=str(row_norm.get('TIPO TRANSPORTE') or "none").lower(), + transport_type=transport_type, transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), weight_type=weight_type or WeightUnit.KGS, seal_number=(row_norm.get('PRECINTO') or None), @@ -648,8 +1241,9 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_invalid += 1 continue - if invoice_number in invoice_id_cache: - invoice_id = invoice_id_cache[invoice_number] + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] else: invoice_id = ( session.query(InvoiceHeader.id) @@ -657,10 +1251,11 @@ def insert_valid_rows(self, job_id: str, model_target: str): InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, ) .scalar() ) - invoice_id_cache[invoice_number] = invoice_id + invoice_id_cache[cache_key] = invoice_id if not invoice_id: logger.warning( @@ -671,6 +1266,20 @@ def insert_valid_rows(self, job_id: str, model_target: str): skipped_missing_invoice += 1 continue + part_num = (row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or '').strip() + if not part_num: + skipped_invalid += 1 + reason = "NUMPARTE: Requerido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + if not validator.check_exists(Part, part_num, field_name="part_number"): + skipped_missing_fk += 1 + reason = f"NUMPARTE '{part_num}' no existe" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + # --- Prevent Duplicates: Clear existing items for this invoice (Once per job) --- if invoice_id not in cleared_invoices: logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates") @@ -689,19 +1298,17 @@ def insert_valid_rows(self, job_id: str, model_target: str): # --- NEW LOGIC: Expanded Anexo 76 Structure --- # A. Find/Cache Part - part_num = (row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or '').strip() part_id = None - if part_num: - part_id = part_cache.get(part_num) - if part_id is None: - p = session.query(Part.id).filter( - Part.part_number == part_num, - Part.tenant_id == tenant_id, - Part.company_id == company_id - ).first() - if p: - part_id = p.id - part_cache[part_num] = part_id + part_id = part_cache.get(part_num) + if part_id is None: + p = session.query(Part.id).filter( + Part.part_number == part_num, + Part.tenant_id == tenant_id, + Part.company_id == company_id + ).first() + if p: + part_id = p.id + part_cache[part_num] = part_id line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA')) line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) @@ -728,23 +1335,23 @@ def insert_valid_rows(self, job_id: str, model_target: str): session.add(line) session.flush() # Need line.id - # 3. Financial Data + # 3. Financial Data (vanilla: nulls from CSV -> 0) price = parse_decimal(row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO')) val_com = parse_decimal(row_norm.get('VALOR COMERCIAL') or row_norm.get('VALORCOMERCIAL')) qty = parse_decimal(row_norm.get('CANTIDAD')) - + commercial_total = val_com or (price * qty if price and qty else None) + session.add(LineFinancial( item_line_id=line.id, - unit_price=price, - commercial_value=val_com or (price * qty if price and qty else None), + unit_cost_capture=decimal_or_zero(price), + total_commercial_value=decimal_or_zero(commercial_total), )) - # 4. Quantities - if qty: - session.add(LineQuantity( - item_line_id=line.id, - quantity=qty, - )) + # 4. Quantities (vanilla: nulls -> 0 so we always have a quantity row) + session.add(LineQuantity( + item_line_id=line.id, + quantity=decimal_or_zero(qty), + )) # 5. Customs/Fraction origin = row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') @@ -764,12 +1371,12 @@ def insert_valid_rows(self, job_id: str, model_target: str): description_spanish=desc, )) - # 7. Legacy Sales Details (For specific audit/UI fields) + # 7. Legacy Sales Details (For specific audit/UI fields; vanilla: nulls -> 0) detail = InvoiceSalesDetails( invoice_id=invoice_id, line_number=line_num, sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), - line_bundles=parse_int(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), + line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), tenant_id=tenant_id, company_id=company_id, ) @@ -851,15 +1458,16 @@ def insert_valid_rows(self, job_id: str, model_target: str): logger.error(traceback.format_exc()) return {"status": "failed", "error": str(e)} - # 5. Cleanup + # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely try: - if os.path.exists(file_path): + if file_path and os.path.exists(file_path): os.remove(file_path) if os.path.exists(error_path): os.remove(error_path) - except: - logger.warning("Failed to cleanup temp files") - + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) + # Ensure response is defined (fallback in case of unexpected errors) if response is None: logger.error(f"Unexpected error: response not set for job {job_id}") diff --git a/backend/api/v1/modules/a76/imports/template_config.py b/backend/api/v1/modules/a76/imports/template_config.py new file mode 100644 index 00000000..1a3fc274 --- /dev/null +++ b/backend/api/v1/modules/a76/imports/template_config.py @@ -0,0 +1,118 @@ +""" +Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean. +La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora. +Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models). +""" + +from typing import Dict, List, Any, Optional + +# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV). +# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que +# espera la lógica de validación e insert (tasks.py). +# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA). + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + # --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) --- + "imp_temp_header": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, + {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, + {"canonical": "FECHA EMISION"}, + {"canonical": "CLAVE PROVEEDOR"}, + {"canonical": "CLAVE VENDIDO A"}, + {"canonical": "CLAVE ENVIADO A"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, + {"canonical": "ADUANA DE CRUCE"}, + {"canonical": "CLAVE MONEDA"}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "TIPO PESO"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "REMESA"}, + {"canonical": "AGENTE ADUANAL"}, + {"canonical": "FLETES"}, + {"canonical": "VALOR SEGUROS"}, + {"canonical": "SEGUROS"}, + {"canonical": "EMBALAJES"}, + {"canonical": "OTROS INCREMENTABLES"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "OBSERVACIONES E"}, + {"canonical": "OBSERVACIONES I"}, + {"canonical": "E DOCUMENT"}, + {"canonical": "NUM OPERACION"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "PRECINTO"}, + ], + # --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura --- + "imp_def_header": None, # se resuelve igual que imp_temp_header + # --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura --- + "exp_def_header": None, + # --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) --- + "imp_temp_details": [ + {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]}, + {"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]}, + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]}, + {"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]}, + {"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]}, + {"canonical": "CANTIDAD"}, + {"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]}, + {"canonical": "FRACCION"}, + {"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]}, + ], + # --- Partidas: Impo Def y Expo - misma estructura --- + "imp_def_details": None, + "exp_def_details": None, +} + + +def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]: + cols = TEMPLATE_COLUMNS.get(template_id) + if cols is not None: + return cols + if template_id in ("imp_def_header", "exp_def_header"): + return TEMPLATE_COLUMNS.get("imp_temp_header") + if template_id in ("imp_def_details", "exp_def_details"): + return TEMPLATE_COLUMNS.get("imp_temp_details") + return None + + +def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]: + """ + Construye un diccionario: normalized_header -> canonical_name. + normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos). + """ + cols = _resolve_template_columns(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]: + """ + A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict + solo con las columnas de la plantilla, usando nombres canónicos. + Así la plantilla se respeta: solo entran columnas definidas en la plantilla. + """ + lookup = build_normalized_lookup(template_id, normalize_header_fn) + if not lookup: + # Sin template definido: comportamiento legacy (normalizar todo) + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/invoices/catalog_service.py b/backend/api/v1/modules/a76/invoices/catalog_service.py index 9ad64b0b..d81f14d9 100644 --- a/backend/api/v1/modules/a76/invoices/catalog_service.py +++ b/backend/api/v1/modules/a76/invoices/catalog_service.py @@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.public.reference_data.transport_types.models import TransportType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection +from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.transport_modes.models import TransportMode diff --git a/backend/api/v1/modules/a76/invoices/schemas.py b/backend/api/v1/modules/a76/invoices/schemas.py index c658e9ae..46ba29ae 100644 --- a/backend/api/v1/modules/a76/invoices/schemas.py +++ b/backend/api/v1/modules/a76/invoices/schemas.py @@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel): operation_type: Optional[OperationType] = Field( ..., description="Operation type: imp/exp/sm/ctm" ) + + @field_validator("operation_type", mode="before") + @classmethod + def normalize_operation_type(cls, v): + """Accept DB string (e.g. 'IMP') and coerce to enum value ('imp').""" + if isinstance(v, str): + return v.lower() if v else v + return v invoice_type: Optional[str] = Field( None, max_length=5, description="Invoice type key" ) @@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel): class InvoiceFinancialsBase(BaseModel): """Base fields for Financials""" - currency: Currency = Field(None, max_length=7, description="Currency code") + currency: Optional[Currency] = Field(None, max_length=7, description="Currency code") currency_type: Optional[str] = Field("USD", description="Currency type") - exchange_rate: Decimal = Field(0.00, description="Exchange rate") + exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate") exchange_rate_mm: Optional[Decimal] = Field( None, description="Exchange rate currency to currency" ) diff --git a/backend/api/v1/modules/a76/invoices/services.py b/backend/api/v1/modules/a76/invoices/services.py index f3b0681b..3c081ffa 100644 --- a/backend/api/v1/modules/a76/invoices/services.py +++ b/backend/api/v1/modules/a76/invoices/services.py @@ -45,11 +45,13 @@ class InvoiceService: # Apply filters if provided if filters: - if filters.get("status"): - query = query.filter(models.InvoiceHeader.status == filters["status"]) + if filters.get("status") is not None: + query = query.filter(models.InvoiceHeader.is_updated == filters["status"]) if filters.get("operation_type"): + ot = filters["operation_type"] + ot_val = ot.value if hasattr(ot, "value") else ot query = query.filter( - models.InvoiceHeader.operation_type == filters["operation_type"] + models.InvoiceHeader.operation_type == ot_val ) if filters.get("invoice_type"): query = query.filter( @@ -67,10 +69,9 @@ class InvoiceService: f"%{filters['pedimento']}%" ) ) - if ( - not filters.get("invoice_type") - and filters.get("operation_type") == "exp" - ): + ot_exp = filters.get("operation_type") + ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp + if not filters.get("invoice_type") and ot_exp_val == "exp": query = query.filter(models.InvoiceHeader.operation_type != "REPAR") if filters.get("manifest_number"): diff --git a/backend/api/v1/modules/a76/router.py b/backend/api/v1/modules/a76/router.py index 96d93ab9..fd8e3829 100644 --- a/backend/api/v1/modules/a76/router.py +++ b/backend/api/v1/modules/a76/router.py @@ -12,7 +12,7 @@ from .general_catalogs.router import router as general_catalogs_router from .invoices.routes import router as invoices_router from .items.routes import router as items_router from .classes import router as classes_router -from .classes import router as classes_router + from .clients_and_providers import router as client_and_provider_router from .imports.routes import router as imports_router from .invoice_settings.routes import router as invoice_settings_router diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 1665bb6a..cf64817e 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -1,6 +1,13 @@ import os from celery import Celery +# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen (mapper) +from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode +from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento +from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, +) + valkey_url = os.getenv("VALKEY_URL", "redis://valkey:6379/0") diff --git a/backend/core/error_handlers.py b/backend/core/error_handlers.py index e1f7783a..af645e09 100644 --- a/backend/core/error_handlers.py +++ b/backend/core/error_handlers.py @@ -11,11 +11,23 @@ from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from .config import settings from .exceptions import BaseAPIException logger = logging.getLogger(__name__) +def _cors_headers(request: Request) -> Dict[str, str]: + """CORS headers for error responses so browser does not block on 4xx/5xx.""" + origin = request.headers.get("origin") + if not origin or origin not in settings.cors_origins_list: + return {} + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + } + + async def base_exception_handler( request: Request, exc: BaseAPIException, @@ -36,10 +48,13 @@ async def base_exception_handler( if hasattr(exc, "errors") and exc.errors: logger.warning(f"Validation errors details: {exc.errors}") - return JSONResponse( + response = JSONResponse( status_code=exc.status_code, content=jsonable_encoder(exc.to_dict()), ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def validation_exception_handler( @@ -65,7 +80,7 @@ async def validation_exception_handler( extra={"errors": errors}, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, content={ "error": "VALIDATION_ERROR", @@ -74,6 +89,9 @@ async def validation_exception_handler( "errors": errors, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def integrity_error_handler( @@ -102,7 +120,7 @@ async def integrity_error_handler( elif "not null" in orig_msg: error_message = "Falta un campo requerido." - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_409_CONFLICT, content={ "error": "DATABASE_INTEGRITY_ERROR", @@ -110,6 +128,9 @@ async def integrity_error_handler( "status_code": status.HTTP_409_CONFLICT, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def sqlalchemy_error_handler( @@ -128,7 +149,7 @@ async def sqlalchemy_error_handler( exc_info=True, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "error": "DATABASE_ERROR", @@ -136,6 +157,9 @@ async def sqlalchemy_error_handler( "status_code": status.HTTP_500_INTERNAL_SERVER_ERROR, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response async def general_exception_handler( @@ -154,7 +178,7 @@ async def general_exception_handler( exc_info=True, ) - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "error": "INTERNAL_SERVER_ERROR", @@ -162,6 +186,9 @@ async def general_exception_handler( "status_code": status.HTTP_500_INTERNAL_SERVER_ERROR, }, ) + for k, v in _cors_headers(request).items(): + response.headers[k] = v + return response def register_exception_handlers(app) -> None: diff --git a/backend/main.py b/backend/main.py index 005dd653..ca5000f9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -17,8 +17,13 @@ from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.material_types.models import MaterialType from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod +# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen para que +# SQLAlchemy resuelva los nombres en relationship() al configurar el mapper from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento +from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, +) from api.v1.modules.public.reference_data.sectors.models import Sector from api.v1.modules.public.reference_data.states.models import State from api.v1.modules.public.reference_data.transport_modes.models import TransportMode @@ -114,6 +119,20 @@ logger = logging.getLogger(__name__) register_exception_handlers(app) +def _cors_headers_for_request(request: Request): + """Return CORS headers if request Origin is allowed (so error responses don't get blocked by browser).""" + origin = request.headers.get("origin") + if not origin: + return {} + allowed = settings.cors_origins_list + if origin in allowed: + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + } + return {} + + # Add validation error handler @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): @@ -121,10 +140,13 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE f"Validation error for {request.method} {request.url.path}: {exc.errors()}" ) logger.error(f"Request body: {await request.body()}") - return JSONResponse( + response = JSONResponse( status_code=status.HTTP_400_BAD_REQUEST, content={"detail": exc.errors(), "body": exc.body}, ) + for k, v in _cors_headers_for_request(request).items(): + response.headers[k] = v + return response # Add HTTP exception handler @@ -133,10 +155,13 @@ async def http_exception_handler(request: Request, exc: HTTPException): logger.error( f"HTTP {exc.status_code} for {request.method} {request.url.path}: {exc.detail}" ) - return JSONResponse( + response = JSONResponse( status_code=exc.status_code, content={"detail": exc.detail}, ) + for k, v in _cors_headers_for_request(request).items(): + response.headers[k] = v + return response def run_migrations(): diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index aed649d2..ef924366 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -341,6 +341,39 @@ export const api = { validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`) }, + imports: { + upload: ( + file: File, + modelTarget: string, + footerConfig: any, + companyId: number, + operationType: string, + templateId?: string + ) => { + const formData = new FormData(); + formData.append('file', file); + if (footerConfig) { + formData.append('footer_config', JSON.stringify(footerConfig)); + } + if (templateId) { + formData.append('template_id', templateId); + } + + const queryParams = new URLSearchParams({ + company_id: String(companyId), + operation_type: operationType || 'imp' + }).toString(); + + return fetchApi(`/v1/a76/imports/upload/${modelTarget}?${queryParams}`, { + method: 'POST', + body: formData + }); + }, + status: (jobId: string) => api.get(`/v1/a76/imports/${jobId}/status`), + commit: (jobId: string, modelTarget: string) => + api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget }) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte index 7375b3ab..bd7bac46 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte @@ -143,17 +143,53 @@ {#if scanResults.error_count > 0}

Se detectaron problemas en el archivo

- Las filas con errores serán omitidas automáticamente. Solo se importarán los - registros válidos. + Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para + importar solo las filas válidas (las erróneas se omitirán).

+ {#if scanResults.errors && scanResults.errors.length > 0} +
+
+
+ Detalle de errores (para corregir en el CSV) +
+ + {scanResults.errors.length} error(es) + +
+
+ + + + + + + + + + {#each scanResults.errors as err} + + + + + + {/each} + +
LíneaColumnaMensaje
{err.line}{err.col || '-'}{err.msg || '-'}
+
+
+ {/if} {:else}
diff --git a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte index 8084f135..62830c66 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte @@ -123,7 +123,7 @@ ondragover={(e) => handleDragOver(e, item.disabled)} ondrop={(e) => handleDrop(e, item)} oncontextmenu={(e) => handleContextMenu(e, item)} - roles="button" + role="button" tabindex={item.disabled ? -1 : 0} onclick={() => handleClick(item.id, item.disabled)} onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)} diff --git a/frontend/src/lib/components/dashboard/customs_brokers/columns.ts b/frontend/src/lib/components/dashboard/customs_brokers/columns.ts index 24400b55..1e770110 100644 --- a/frontend/src/lib/components/dashboard/customs_brokers/columns.ts +++ b/frontend/src/lib/components/dashboard/customs_brokers/columns.ts @@ -8,6 +8,20 @@ export type { CustomsBroker }; export function createColumns(onSuccess?: () => void): ColumnDef[] { return [ + { + accessorKey: "id", + header: "ID", + cell: ({ row }) => { + const idSnippet = createRawSnippet<[{ id: number }]>((getId) => { + const { id } = getId(); + return { + render: () => + `${id}` + }; + }); + return renderSnippet(idSnippet, { id: row.original.id }); + } + }, { accessorKey: "broker_key", header: "Clave", diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index 656b0459..2deb6775 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -249,7 +249,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Encabezado', icon: FileText, group: 'Impo. Def.', - modelTarget: 'InvoiceHeader', + modelTarget: 'invoice_header', templateUrl: '/csv/EstructuraEncFacImpoDef.xls' }, { @@ -257,7 +257,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Partidas', icon: Package, group: 'Impo. Def.', - modelTarget: 'InvoiceSalesDetails', + modelTarget: 'invoice_details', templateUrl: '/csv/EstructuraParFacImpoDefAF.xls' }, { @@ -274,7 +274,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Encabezado', icon: FileText, group: 'Compras Mex.', - modelTarget: 'InvoiceHeader', + modelTarget: 'invoice_header', disabled: true, }, { @@ -282,7 +282,7 @@ export const importacionConfig: CsvUploadItem[] = [ title: 'Partidas', icon: Package, group: 'Compras Mex.', - modelTarget: 'InvoiceSalesDetails', + modelTarget: 'invoice_details', disabled: true, }, { @@ -302,7 +302,7 @@ export const exportacionConfig: CsvUploadItem[] = [ title: 'Encabezado', icon: FileText, group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceHeader', + modelTarget: 'invoice_header', templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls' }, { @@ -310,7 +310,7 @@ export const exportacionConfig: CsvUploadItem[] = [ title: 'Partidas', icon: Package, group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceSalesDetails', + modelTarget: 'invoice_details', templateUrl: '/csv/EstructuraParExpoCamReg.xls' }, { diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 6c55bc56..b1efc2ed 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -38,25 +38,45 @@ }); async function handleUpload(file: File, config: CsvUploadItem) { + console.log('handleUpload started', { file, config }); isUploading = true; activeModelTarget = config.modelTarget || null; scanResults = null; const currentSettings = allSettings[activeTab] || {}; + const footerConfig = { ...currentSettings }; + if (activeTab === 'importacion') { + footerConfig.invoice_type = config.id?.startsWith('imp_def_') ? 'DEF' : 'TEM'; + } const companyId = companyStore.activeCompany?.id || 1; const opType = activeTab === 'exportacion' ? 'exp' : 'imp'; - const res = await api.imports.upload( - file, - config.modelTarget || '', - currentSettings, + console.log('Calling api.imports.upload', { + activeModelTarget, + footerConfig, companyId, opType - ); - if (res.data?.job_id) { - currentJobId = res.data.job_id; - pollStatus(); - } else { - toast.error('Error al subir el archivo'); + }); + try { + const res = await api.imports.upload( + file, + config.modelTarget || '', + footerConfig, + companyId, + opType, + config.id + ); + console.log('Upload response', res); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + console.error('Upload failed with response', res); + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); isUploading = false; } } @@ -64,54 +84,71 @@ async function pollStatus() { if (!currentJobId) return; - const res = await api.imports.status(currentJobId); - if (res.data?.status === 'waiting_confirmation') { - scanResults = res.data; - showResultModal = true; - toast.success('Escaneo completado. Revisa los resultados.'); - isUploading = false; - } else if (res.data?.status === 'failed') { - toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido')); - isUploading = false; - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - } else if (res.data?.status === 'warning') { - // Caso cuando no se insertaron registros pero hay información de rechazo - commitResults = res.data; - showResultModal = true; - const inserted = res.data?.inserted || 0; - const skippedInvalid = res.data?.skipped_invalid || 0; - const skippedFk = res.data?.skipped_missing_fk || 0; - const totalSkipped = skippedInvalid + skippedFk; - - if (inserted === 0) { - toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`); - } else { - toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`); + console.log('Polling status for job', currentJobId); + try { + const res = await api.imports.status(currentJobId); + console.log('Poll response', res); + if (res.error && !res.data) { + toast.error(res.error || 'Error al consultar el estado'); + isUploading = false; + currentJobId = null; + return; } - isUploading = false; - } else if (res.data?.status === 'finished') { - commitResults = res.data; - showResultModal = true; - const inserted = res.data?.inserted || 0; - const skippedInvalid = res.data?.skipped_invalid || 0; - const skippedFk = res.data?.skipped_missing_fk || 0; - const skippedDetails = res.data?.skipped_details || []; + if (res.data?.status === 'waiting_confirmation') { + scanResults = res.data; + showResultModal = true; + toast.success('Escaneo completado. Revisa los resultados.'); + isUploading = false; + } else if (res.data?.status === 'failed' || res.data?.status === 'FAILURE') { + toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido')); + isUploading = false; + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + } else if (res.data?.status === 'warning') { + // Caso cuando no se insertaron registros pero hay información de rechazo + commitResults = res.data; + showResultModal = true; + const inserted = res.data?.inserted || 0; + const skippedInvalid = res.data?.skipped_invalid || 0; + const skippedFk = res.data?.skipped_missing_fk || 0; + const totalSkipped = skippedInvalid + skippedFk; - if (inserted > 0) { - toast.success(`Importación completada: ${inserted} registros insertados`); - if (skippedInvalid > 0 || skippedFk > 0) { - const totalSkipped = skippedInvalid + skippedFk; - toast.warning(`${totalSkipped} registros fueron rechazados`); + if (inserted === 0) { + toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`); + } else { + toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`); } + isUploading = false; + } else if (res.data?.status === 'finished') { + commitResults = res.data; + showResultModal = true; + const inserted = res.data?.inserted || 0; + const skippedInvalid = res.data?.skipped_invalid || 0; + const skippedFk = res.data?.skipped_missing_fk || 0; + const skippedDetails = res.data?.skipped_details || []; + + if (inserted > 0) { + toast.success(`Importación completada: ${inserted} registros insertados`); + if (skippedInvalid > 0 || skippedFk > 0) { + const totalSkipped = skippedInvalid + skippedFk; + toast.warning(`${totalSkipped} registros fueron rechazados`); + } + } else { + toast.error('No se insertaron registros. Revisa los errores a continuación.'); + } + isUploading = false; } else { - toast.error('No se insertaron registros. Revisa los errores a continuación.'); + // Continue polling + console.log('Status not final, polling again in 2s...', res.data?.status); + setTimeout(pollStatus, 2000); } - isUploading = false; - } else { - // Continue polling + } catch (e) { + console.error('Poll exception', e); + // Retry on network error? Or fail? + // For now, let's keep retrying a few times or hard fail. + // Let's just log and retry. setTimeout(pollStatus, 2000); } } @@ -174,36 +211,38 @@ {/if}
- { - if (currentJobId && activeModelTarget) { - try { - isUploading = true; - const res = await api.imports.commit(currentJobId, activeModelTarget); - if (res.data?.commit_job_id) { - currentJobId = res.data.commit_job_id; - pollStatus(); +{#if scanResults || commitResults} + { + if (currentJobId && activeModelTarget) { + try { + isUploading = true; + const res = await api.imports.commit(currentJobId, activeModelTarget); + if (res.data?.commit_job_id) { + currentJobId = res.data.commit_job_id; + pollStatus(); + } + } catch (err) { + toast.error('Error al iniciar la importación'); + isUploading = false; } - } catch (err) { - toast.error('Error al iniciar la importación'); - isUploading = false; } - } - }} - onCancel={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} - onClose={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} -/> + }} + onCancel={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + onClose={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + /> +{/if} From a65883922a777a0c26e118e6a784d0b0c44c6152 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 23 Feb 2026 13:42:06 -0600 Subject: [PATCH 02/16] fix/conflict --- scripts/init_first_time.sh | 286 ++++++------------------------------- 1 file changed, 42 insertions(+), 244 deletions(-) diff --git a/scripts/init_first_time.sh b/scripts/init_first_time.sh index 1137af42..b9dacab2 100755 --- a/scripts/init_first_time.sh +++ b/scripts/init_first_time.sh @@ -12,18 +12,14 @@ # 6. Relación usuario-tenant en tabla user_tenants # 7. Actualización del tenant_id del usuario con el valor real # 8. Licencia Enterprise para el tenant (ilimitada, 1 año de vigencia) -# 9. [OPCIONAL] Datos iniciales de ejemplo si se pasa --seed-data -# -# Uso: -# ./init_first_time.sh # Solo configuración básica -# ./init_first_time.sh --seed-data # Configuración + datos de ejemplo # # Requisitos: -# - Keycloak corriendo en http://localhost:8080 -# - PostgreSQL corriendo en localhost:5432 +# - Keycloak y PostgreSQL corriendo (p. ej. docker compose up -d) # - Base de datos anexo76_core creada # - jq instalado (para procesamiento JSON) # +# Puertos: Keycloak en 18080/19000, frontend en 15173, API en 18000. +# # Nota: El atributo tenant_id se crea con valor inicial "1" y luego # se actualiza con el ID real del tenant creado en PostgreSQL. ############################################################################### @@ -33,22 +29,6 @@ set -euo pipefail # Modo strict: exit on error, undefined vars, pipe failures # Trap para cleanup en caso de error trap 'echo -e "\n${RED}✗ Error en línea $LINENO. Script abortado.${NC}" >&2' ERR -# Parsear argumentos -SEED_DATA=false -while [[ $# -gt 0 ]]; do - case $1 in - --seed-data) - SEED_DATA=true - shift - ;; - *) - echo "Uso: $0 [--seed-data]" - echo " --seed-data: Carga datos de ejemplo en las tablas" - exit 1 - ;; - esac -done - # Colores para output RED='\033[0;31m' GREEN='\033[0;32m' @@ -133,177 +113,12 @@ create_tenant_mapper() { fi } -############################################################################### -# Funciones de seed data -############################################################################### - -# Insertar datos de ejemplo para customs_brokers -seed_customs_brokers() { - echo " → Insertando customs brokers..." - exec_pg_sql " - INSERT INTO a76.customs_brokers (tenant_id, company_id, type, broker_key, name, address, postal_code, city, state, phone, email, country, tax_id, license, company, contact, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB001', 'Agente Aduanal García', 'Av. Reforma 123', '01000', 'Ciudad de México', 'CDMX', '5555555555', 'garcia@aduanas.com', 'MEX', 'GAAR800101ABC', '1234', 'García y Asociados', 'Juan García', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB002', 'Agente Aduanal López', 'Blvd. Díaz Ordaz 456', '22000', 'Tijuana', 'BC', '6641234567', 'lopez@customs.com', 'MEX', 'LOPL750505XYZ', '2345', 'López Customs', 'María López', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB003', 'Agente Aduanal Martínez', 'Calle Industria 789', '45000', 'Guadalajara', 'JAL', '3339876543', 'martinez@broker.com', 'MEX', 'MARM850315DEF', '3456', 'Martínez Brokerage', 'Pedro Martínez', now(), now()) - ON CONFLICT (broker_key, tenant_id, company_id) DO NOTHING; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para clients_and_providers -seed_clients_and_providers() { - echo " → Insertando clientes y proveedores..." - exec_pg_sql " - INSERT INTO a76.clients_and_providers (tenant_id, company_id, type_nat_foreign, name, short_name, rfc, client_or_provider, web_key, is_active, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Proveedor Tecnológico SA de CV', 'PROVTECH', 'PTE901201ABC', 'BOTH', 'PROV001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Cliente Industrial del Norte SA', 'CINORTE', 'CIN850615XYZ', 'BOTH', 'CLI001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'E', 'Global Supplies Inc', 'GLOBSUP', 'GSI123456789', 'BOTH', 'BOTH001', true, now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'N', 'Manufacturas del Bajío SA', 'MANBAJIO', 'MDB920310DEF', 'BOTH', 'CLI002', true, now(), now()) - RETURNING id; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para packages -seed_packages() { - echo " → Insertando tipos de paquete..." - exec_pg_sql " - INSERT INTO a76.packages (tenant_id, company_id, key, description_es, description_en, weight_unit, plurals, plural_in, code_ace, code_aamex, created_at, updated_at) - VALUES - (${TENANT_ID}, ${COMPANY_ID}, 'PK01', 'Caja de Cartón', 'Cardboard Box', 0.5, 'CAJS', 'BOXS', 'CB01', 'CAJA001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK02', 'Pallet de Madera', 'Wooden Pallet', 15.0, 'PLTS', 'PLTS', 'WP01', 'PALL001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK03', 'Tambor Metálico', 'Metal Drum', 10.0, 'TMBS', 'DRMS', 'MD01', 'TAMB001', now(), now()), - (${TENANT_ID}, ${COMPANY_ID}, 'PK04', 'Contenedor', 'Container', 2000.0, 'CONT', 'CONT', 'CT01', 'CONT001', now(), now()) - ON CONFLICT (tenant_id, company_id, key) DO NOTHING; - " >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para classes -seed_classes() { - echo " → Insertando clases..." - exec_pg_sql "INSERT INTO a76.classes (tenant_id, company_id, class_code, description_es, description_en, material_key, unit_of_measure, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'CLS001', 'Componentes Electrónicos', 'Electronic Components', 'MP', 'PZA', '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS002', 'Partes Automotrices', 'Automotive Parts', 'MP', 'KGS', '8708.29.99', '8708.29.9900', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS003', 'Textiles y Telas', 'Textiles and Fabrics', 'MP', 'MT', '5407.20.01', '5407.20.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS004', 'Equipo de Computación', 'Computer Equipment', 'MP', 'PZA', '8471.30.01', '8471.30.0100', now(), now()) ON CONFLICT (tenant_id, company_id, class_code) DO NOTHING;" >/dev/null 2>&1 -} - -# Insertar datos de ejemplo para parts -seed_parts() { - echo " → Insertando partes/componentes..." - - # Obtener un client_id para asociar las partes - local client_id - client_id=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} LIMIT 1;" | xargs) - - if [ -n "$client_id" ]; then - exec_pg_sql "INSERT INTO a76.parts (tenant_id, company_id, client_id, part_number, description_spanish, description_english, part_class, currency_key, unit_of_measure, unit_cost, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-001', 'Microcontrolador ARM Cortex-M4', 'ARM Cortex-M4 Microcontroller', 'CLS001', 'USD', 'PZA', 15.50, '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-002', 'Filtro de Aceite Automotriz', 'Automotive Oil Filter', 'CLS002', 'USD', 'PZA', 8.75, '8421.23.01', '8421.23.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-003', 'Tela de Algodón para Tapicería', 'Cotton Upholstery Fabric', 'CLS003', 'USD', 'MT', 12.00, '5208.31.01', '5208.31.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-004', 'Disco Duro SSD 500GB', '500GB SSD Hard Drive', 'CLS004', 'USD', 'PZA', 65.00, '8471.70.01', '8471.70.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-005', 'Sensor de Temperatura Digital', 'Digital Temperature Sensor', 'CLS001', 'USD', 'PZA', 5.25, '9025.19.01', '9025.19.0100', now(), now()) ON CONFLICT (tenant_id, company_id, part_number) DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Insertar datos de ejemplo para pedimentos -seed_pedimentos() { - echo " → Insertando pedimentos..." - - # Primero obtener IDs de clientes - local client_ids - client_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('CLIENT', 'BOTH') LIMIT 2;") - local client_id_1=$(echo "$client_ids" | sed -n '1p' | xargs) - local client_id_2=$(echo "$client_ids" | sed -n '2p' | xargs) - - if [ -n "$client_id_1" ]; then - exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001234', ${client_id_1}, 'imp', 'normal', 'V1', 'ITE', 'draft', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001235', ${client_id_1}, 'exp', 'normal', 'V1', 'ETE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$client_id_2" ]; then - exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001236', ${client_id_2}, 'imp', 'consolidated', 'V1', 'ITE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Insertar datos de ejemplo para invoices -seed_invoices() { - echo " → Insertando facturas..." - - # Obtener IDs de clientes/proveedores - local provider_ids - provider_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('PROVIDER', 'BOTH') LIMIT 2;") - local provider_id_1=$(echo "$provider_ids" | sed -n '1p' | xargs) - local provider_id_2=$(echo "$provider_ids" | sed -n '2p' | xargs) - - # Insertar facturas en invoice_header (mínimo requerido) - exec_pg_sql "INSERT INTO a76.invoice_header (tenant_id, company_id, system, operation_type, invoice_type, invoice_number, invoice_date, is_updated, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'DEF', 'INV-2024-001', '2024-01-10', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'exp', 'EXDEF', 'INV-2024-002', '2024-02-15', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'TEM', 'INV-2024-003', '2024-03-05', false, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - - # Obtener IDs de las facturas recién creadas - local invoice_ids - invoice_ids=$(exec_pg_sql "SELECT id FROM a76.invoice_header WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND invoice_number IN ('INV-2024-001', 'INV-2024-002', 'INV-2024-003') ORDER BY invoice_number;") - local invoice_id_1=$(echo "$invoice_ids" | sed -n '1p' | xargs) - local invoice_id_2=$(echo "$invoice_ids" | sed -n '2p' | xargs) - local invoice_id_3=$(echo "$invoice_ids" | sed -n '3p' | xargs) - - # Insertar datos financieros para las facturas - if [ -n "$invoice_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'foreign', 17.50, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'foreign', 17.45, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ]; then - exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'local', 1.00, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - # Insertar datos de compliance mexicano con proveedores - if [ -n "$invoice_id_1" ] && [ -n "$provider_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ] && [ -n "$provider_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ] && [ -n "$provider_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, ${provider_id_2}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - # Insertar datos de logística con incoterm - if [ -n "$invoice_id_1" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'none', 'kgs', 'FOB', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_2" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'none', 'kgs', 'CIF', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi - - if [ -n "$invoice_id_3" ]; then - exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'none', 'kgs', 'EXW', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1 - fi -} - -# Ejecutar todas las funciones de seed -execute_seed_data() { - echo -e "\n${YELLOW}[SEED] Cargando datos de ejemplo...${NC}" - - seed_customs_brokers - seed_clients_and_providers - seed_packages - seed_classes - seed_parts - seed_pedimentos - seed_invoices - - echo -e "${GREEN}✓ Datos de ejemplo cargados exitosamente${NC}" - echo -e "${YELLOW} • 3 Agentes aduanales${NC}" - echo -e "${YELLOW} • 4 Clientes/Proveedores${NC}" - echo -e "${YELLOW} • 4 Tipos de paquete${NC}" - echo -e "${YELLOW} • 4 Clases${NC}" - echo -e "${YELLOW} • 5 Parts/Componentes${NC}" - echo -e "${YELLOW} • 3 Pedimentos${NC}" - echo -e "${YELLOW} • 3 Facturas${NC}" -} - -# Variables de configuración -KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080/kcauth}" +# Variables de configuración (puertos con prefijo 1 hardcodeados) +KEYCLOAK_URL="http://localhost:18080/kcauth" KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}" KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}" KEYCLOAK_REALM="${KEYCLOAK_REALM:-master}" -KEYCLOACK_ADMIN_URL="${KEYCLOACK_ADMIN_URL:-http://localhost:9000/kcauth}" +KEYCLOACK_ADMIN_URL="http://localhost:19000/kcauth" POSTGRES_HOST="${POSTGRES_HOST:-localhost}" POSTGRES_PORT="${POSTGRES_PORT:-5432}" @@ -448,13 +263,13 @@ else "directAccessGrantsEnabled": true, "standardFlowEnabled": true, "implicitFlowEnabled": false, - "rootUrl": "http://localhost:8000/api", - "baseUrl": "http://localhost:8000/api", + "rootUrl": "http://localhost:18000/api", + "baseUrl": "http://localhost:18000/api", "redirectUris": [ - "http://localhost:8000/*", - "http://localhost:5173/*" + "http://localhost:18000/*", + "http://localhost:15173/*" ], - "webOrigins": ["http://localhost:8000/api"], + "webOrigins": ["http://localhost:18000/api"], "attributes": { "access.token.lifespan": "3600", "client.secret.creation.time": "0" @@ -529,30 +344,30 @@ else CREATE_FRONTEND=$(curl -s -w "\n%{http_code}" -X POST "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/clients" \ -H "Authorization: Bearer ${ACCESS_TOKEN}" \ -H "Content-Type: application/json" \ - -d '{ - "clientId": "anexo76-frontend", - "name": "Anexo76 Frontend", - "description": "Aplicación web frontend para el sistema Anexo76", - "enabled": true, - "protocol": "openid-connect", - "publicClient": true, - "directAccessGrantsEnabled": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "rootUrl": "http://localhost:5173", - "baseUrl": "http://localhost:5173", - "redirectUris": [ - "http://localhost:5173/*", - "http://localhost:3000/*" + -d "{ + \"clientId\": \"anexo76-frontend\", + \"name\": \"Anexo76 Frontend\", + \"description\": \"Aplicación web frontend para el sistema Anexo76\", + \"enabled\": true, + \"protocol\": \"openid-connect\", + \"publicClient\": true, + \"directAccessGrantsEnabled\": false, + \"standardFlowEnabled\": true, + \"implicitFlowEnabled\": false, + \"rootUrl\": \"http://localhost:15173\", + \"baseUrl\": \"http://localhost:15173\", + \"redirectUris\": [ + \"http://localhost:15173/*\", + \"http://localhost:13000/*\" ], - "webOrigins": [ - "http://localhost:5173", - "http://localhost:3000" + \"webOrigins\": [ + \"http://localhost:15173\", + \"http://localhost:13000\" ], - "attributes": { - "pkce.code.challenge.method": "S256" + \"attributes\": { + \"pkce.code.challenge.method\": \"S256\" } - }') + }") HTTP_CODE=$(echo "$CREATE_FRONTEND" | tail -n1) RESPONSE_BODY=$(echo "$CREATE_FRONTEND" | head -n -1) @@ -743,12 +558,14 @@ else echo -e "${YELLOW}⚠ Company ya existe para este tenant${NC}" fi -# Obtener información de la company -COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;") +# Obtener ID de la company (necesario para user_tenants) COMPANY_ID=$(exec_pg_sql "SELECT id FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;" | xargs) - -echo -e "${GREEN}✓ Company: ${COMPANY_INFO}${NC}" -echo -e "${GREEN}✓ Company ID: ${COMPANY_ID}${NC}" +if [ -z "$COMPANY_ID" ]; then + echo -e "${RED}✗ Error: No se pudo obtener el ID de la company${NC}" + exit 1 +fi +COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;") +echo -e "${GREEN}✓ Company ID: ${COMPANY_ID} | ${COMPANY_INFO}${NC}" # Agregar tenant_id al usuario demo en Keycloak echo -e "\n${YELLOW}Asignando tenant_id al usuario demo...${NC}" @@ -764,10 +581,10 @@ curl -s -X PUT "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/users/${USER_ID}" echo -e "${GREEN}✓ Atributo tenant_id asignado al usuario${NC}" -# Agregar relación usuario-tenant en la base de datos +# Agregar relación usuario-tenant en la base de datos (usar company_id real) echo -e "\n${YELLOW}Creando relación usuario-tenant en la base de datos...${NC}" -exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null +exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, ${COMPANY_ID}, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null echo -e "${GREEN}✓ Relación usuario-tenant creada en la base de datos${NC}" @@ -823,13 +640,6 @@ else fi fi -############################################################################### -# 9. Cargar datos de ejemplo (opcional) -############################################################################### -if [ "$SEED_DATA" = true ]; then - execute_seed_data -fi - ############################################################################### # Resumen final ############################################################################### @@ -868,18 +678,6 @@ echo -e " ${GREEN}✓${NC} Plan: Enterprise (ilimitado)" echo -e " ${GREEN}✓${NC} Status: Activa" echo -e " ${GREEN}✓${NC} Features: API, Reportes Avanzados, Integraciones, Soporte Dedicado" echo -e " ${GREEN}✓${NC} Vigencia: 1 año" - -if [ "$SEED_DATA" = true ]; then -echo -e "\n${YELLOW}Datos de ejemplo:${NC}" -echo -e " ${GREEN}✓${NC} Agentes aduanales: 3" -echo -e " ${GREEN}✓${NC} Clientes/Proveedores: 4" -echo -e " ${GREEN}✓${NC} Tipos de paquete: 4" -echo -e " ${GREEN}✓${NC} Clases: 4" -echo -e " ${GREEN}✓${NC} Parts/Componentes: 5" -echo -e " ${GREEN}✓${NC} Pedimentos: 3" -echo -e " ${GREEN}✓${NC} Facturas: 3" -fi - echo -e "\n${YELLOW}Puedes acceder al sistema en:${NC}" -echo -e " ${GREEN}http://localhost:5173${NC}" +echo -e " ${GREEN}http://localhost:15173${NC}" echo -e "\n${GREEN}════════════════════════════════════════════════════════${NC}\n" From e8e3774758d21f25dd6140bfcd6345703773e185 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 23 Feb 2026 14:22:46 -0600 Subject: [PATCH 03/16] feature/carga-agententes-aduanales-via-csv --- .../a76/customs_brokers/imports/__init__.py | 2 + .../a76/customs_brokers/imports/routes.py | 157 ++++++ .../a76/customs_brokers/imports/schemas.py | 22 + .../a76/customs_brokers/imports/tasks.py | 447 ++++++++++++++++++ .../imports/template_config.py | 55 +++ .../v1/modules/a76/customs_brokers/routes.py | 4 + backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 61 ++- 9 files changed, 742 insertions(+), 22 deletions(-) create mode 100644 backend/api/v1/modules/a76/customs_brokers/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/customs_brokers/imports/routes.py create mode 100644 backend/api/v1/modules/a76/customs_brokers/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/customs_brokers/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/customs_brokers/imports/template_config.py diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py new file mode 100644 index 00000000..c1c90d12 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/__init__.py @@ -0,0 +1,2 @@ +# CSV import flow for Agentes Aduanales (Customs Brokers). +# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py new file mode 100644 index 00000000..336f1b73 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py @@ -0,0 +1,157 @@ +""" +Rutas de importación CSV para Agentes Aduanales. +Mismo flujo que a76.imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CB_IMPORT_FILE_PREFIX, + CB_IMPORT_META_PREFIX, + CB_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"CB import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "customs_brokers", + } + + try: + r = _get_redis() + r.set( + f"{CB_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CB_IMPORT_REDIS_TTL, + ) + r.set( + f"{CB_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CB_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"CB import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"CB import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + logger.warning("CB import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg: + result = getattr(task_result, "result", None) + if result is not None and not isinstance(result, dict): + err_msg = str(result) + elif isinstance(result, dict) and (result.get("error") or result.get("message")): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py b/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py new file mode 100644 index 00000000..7d9b548c --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + +class CommitRequest(BaseModel): + pass # no body needed for single model + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py new file mode 100644 index 00000000..ecaed98f --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/tasks.py @@ -0,0 +1,447 @@ +""" +Tareas Celery para importación CSV de Agentes Aduanales. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con a76.imports) +CB_IMPORT_FILE_PREFIX = "cb_import_file:" +CB_IMPORT_META_PREFIX = "cb_import_meta:" +CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:" +CB_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"CB import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cb_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"CB import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CB_IMPORT_FILE_PREFIX}{job_id}", + f"{CB_IMPORT_META_PREFIX}{job_id}", + f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"CB import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Agente Aduanal. Retorna error dict o None.""" + clave = (row.get("CLAVE") or "").strip() + if not clave: + return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} + if len(clave) > 5: + return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"} + if not re.match(r"^[a-zA-Z0-9]+$", clave): + return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"} + + licencia = (row.get("LICENCIA") or "").strip() + if licencia and (len(licencia) > 4 or not licencia.isdigit()): + return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"CB import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"CB import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_customs_broker(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"CB import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + # Guardar números de línea con error en Redis para insert_valid_rows + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CB_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"CB import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker. + """ + logger.info(f"CB import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"CB import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_key: Dict[str, CustomsBroker] = {} + for b in ( + session.query(CustomsBroker) + .filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ) + .all() + ): + existing_by_key[b.broker_key] = b + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_customs_broker(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + clave = (row_norm.get("CLAVE") or "").strip()[:5] + if not clave: + skipped_invalid += 1 + continue + + existing = existing_by_key.get(clave) + if existing: + existing.type = _str_or_none(row_norm.get("TIPO"), 9) + existing.name = _str_or_none(row_norm.get("NOMBRE"), 80) + existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500) + existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15) + existing.city = _str_or_none(row_norm.get("CIUDAD"), 30) + existing.state = _str_or_none(row_norm.get("ESTADO"), 30) + existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30) + existing.fax = _str_or_none(row_norm.get("FAX"), 30) + existing.email = _str_or_none(row_norm.get("EMAIL"), 100) + existing.country = _str_or_none(row_norm.get("PAIS"), 3) + existing.tax_id = _str_or_none(row_norm.get("RFC"), 30) + existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20) + existing.position = _str_or_none(row_norm.get("POSICION"), 30) + lic = (row_norm.get("LICENCIA") or "").strip() + existing.license = lic[:4] if lic and lic.isdigit() else None + existing.company = _str_or_none(row_norm.get("EMPRESA"), 200) + existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80) + session.add(existing) + inserted_count += 1 + else: + lic = (row_norm.get("LICENCIA") or "").strip() + license_val = lic[:4] if lic and lic.isdigit() else None + new_broker = CustomsBroker( + tenant_id=tenant_id, + company_id=company_id, + broker_key=clave, + type=_str_or_none(row_norm.get("TIPO"), 9), + name=_str_or_none(row_norm.get("NOMBRE"), 80), + address=_str_or_none(row_norm.get("DIRECCION"), 1500), + postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), + city=_str_or_none(row_norm.get("CIUDAD"), 30), + state=_str_or_none(row_norm.get("ESTADO"), 30), + phone=_str_or_none(row_norm.get("TELEFONO"), 30), + fax=_str_or_none(row_norm.get("FAX"), 30), + email=_str_or_none(row_norm.get("EMAIL"), 100), + country=_str_or_none(row_norm.get("PAIS"), 3), + tax_id=_str_or_none(row_norm.get("RFC"), 30), + personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20), + position=_str_or_none(row_norm.get("POSICION"), 30), + license=license_val, + company=_str_or_none(row_norm.get("EMPRESA"), 200), + contact=_str_or_none(row_norm.get("CONTACTO"), 80), + ) + session.add(new_broker) + existing_by_key[clave] = new_broker + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"CB import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"CB import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + # Limpieza + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"CB import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py b/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py new file mode 100644 index 00000000..f4801729 --- /dev/null +++ b/backend/api/v1/modules/a76/customs_brokers/imports/template_config.py @@ -0,0 +1,55 @@ +""" +Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "customs_brokers": [ + {"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]}, + {"canonical": "TIPO"}, + {"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]}, + {"canonical": "DIRECCION", "aliases": ["DOMICILIO"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + {"canonical": "CIUDAD"}, + {"canonical": "ESTADO"}, + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]}, + {"canonical": "FAX"}, + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]}, + {"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]}, + {"canonical": "EMPRESA", "aliases": ["COMPANY"]}, + {"canonical": "CONTACTO", "aliases": ["CONTACT"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla customs_brokers.""" + cols = TEMPLATE_COLUMNS.get("customs_brokers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/customs_brokers/routes.py b/backend/api/v1/modules/a76/customs_brokers/routes.py index e968ac78..02f301a1 100644 --- a/backend/api/v1/modules/a76/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/customs_brokers/routes.py @@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from . import dto, services +from .imports.routes import router as imports_router router = APIRouter() +# CSV import (mismo flujo que a76.imports: upload → scan → commit) +router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"]) + customs_broker_crud = TenantCRUDRoutes( service=services.CustomsBrokerService, create_schema=dto.CustomsBrokerCreateDTO, diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index db0e14a5..6fc74f9f 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -26,6 +26,7 @@ celery_app = Celery( "api.v1.modules.a76.reports.exportacion.aviso_consolidado.task", "api.v1.modules.a76.reports.exportacion.descargo.task", "api.v1.modules.a76.imports.tasks", + "api.v1.modules.a76.customs_brokers.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task" diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ef924366..fd26e485 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -374,6 +374,21 @@ export const api = { api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget }) }, + // CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports) + customsBrokerImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/customs-brokers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/customs-brokers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index b1efc2ed..3aa127a0 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -24,6 +24,8 @@ let scanResults = $state(null); let commitResults = $state(null); let showResultModal = $state(false); + // Cuando es true, usamos API de importación de Agentes Aduanales (customs_brokers/imports) + let useCustomsBrokerImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -42,20 +44,35 @@ isUploading = true; activeModelTarget = config.modelTarget || null; scanResults = null; + useCustomsBrokerImport = config.id === 'customs_brokers'; + + const companyId = companyStore.activeCompany?.id || 1; + + if (useCustomsBrokerImport) { + try { + const res = await api.customsBrokerImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { footerConfig.invoice_type = config.id?.startsWith('imp_def_') ? 'DEF' : 'TEM'; } - const companyId = companyStore.activeCompany?.id || 1; const opType = activeTab === 'exportacion' ? 'exp' : 'imp'; - console.log('Calling api.imports.upload', { - activeModelTarget, - footerConfig, - companyId, - opType - }); try { const res = await api.imports.upload( file, @@ -65,12 +82,10 @@ opType, config.id ); - console.log('Upload response', res); if (res.data?.job_id) { currentJobId = res.data.job_id; pollStatus(); } else { - console.error('Upload failed with response', res); toast.error(res.error || 'Error al subir el archivo'); isUploading = false; } @@ -84,9 +99,10 @@ async function pollStatus() { if (!currentJobId) return; - console.log('Polling status for job', currentJobId); try { - const res = await api.imports.status(currentJobId); + const res = useCustomsBrokerImport + ? await api.customsBrokerImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -218,18 +234,19 @@ {commitResults} {isUploading} onConfirm={async () => { - if (currentJobId && activeModelTarget) { - try { - isUploading = true; - const res = await api.imports.commit(currentJobId, activeModelTarget); - if (res.data?.commit_job_id) { - currentJobId = res.data.commit_job_id; - pollStatus(); - } - } catch (err) { - toast.error('Error al iniciar la importación'); - isUploading = false; + if (!currentJobId) return; + try { + isUploading = true; + const res = useCustomsBrokerImport + ? await api.customsBrokerImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); + if (res.data?.commit_job_id) { + currentJobId = res.data.commit_job_id; + pollStatus(); } + } catch (err) { + toast.error('Error al iniciar la importación'); + isUploading = false; } }} onCancel={() => { From 4a8bea4966d9daac3ca835a54925bc49d7f39d96 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 23 Feb 2026 17:20:22 -0600 Subject: [PATCH 04/16] feature/csv-client-provider --- .../clients_and_providers/imports/__init__.py | 2 + .../clients_and_providers/imports/routes.py | 161 ++++++ .../clients_and_providers/imports/schemas.py | 25 + .../clients_and_providers/imports/tasks.py | 500 ++++++++++++++++++ .../imports/template_config.py | 56 ++ .../a76/clients_and_providers/routes.py | 4 + .../a76/customs_brokers/imports/routes.py | 12 +- backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 38 +- 10 files changed, 807 insertions(+), 7 deletions(-) create mode 100644 backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/clients_and_providers/imports/routes.py create mode 100644 backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py b/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py new file mode 100644 index 00000000..94b7cb55 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/__init__.py @@ -0,0 +1,2 @@ +# CSV import flow for Clientes y Proveedores (Client Providers). +# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit. diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py b/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py new file mode 100644 index 00000000..68029417 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/routes.py @@ -0,0 +1,161 @@ +""" +Rutas de importación CSV para Clientes y Proveedores. +Mismo flujo que a76.imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CP_IMPORT_FILE_PREFIX, + CP_IMPORT_META_PREFIX, + CP_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"CP import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "client_providers", + } + + try: + r = _get_redis() + r.set( + f"{CP_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CP_IMPORT_REDIS_TTL, + ) + r.set( + f"{CP_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"CP import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"CP import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("CP import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py b/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py new file mode 100644 index 00000000..34fea87c --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/tasks.py @@ -0,0 +1,500 @@ +""" +Tareas Celery para importación CSV de Clientes y Proveedores. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template +from api.v1.modules.a76.clients_and_providers.models import ( + ClientProvider, + ClientProviderAddress, + ClientOrProviderEnum, +) + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports) +CP_IMPORT_FILE_PREFIX = "cp_import_file:" +CP_IMPORT_META_PREFIX = "cp_import_meta:" +CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:" +CP_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"CP import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cp_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"CP import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CP_IMPORT_FILE_PREFIX}{job_id}", + f"{CP_IMPORT_META_PREFIX}{job_id}", + f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"CP import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]: + """Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido.""" + if not val or not str(val).strip(): + return None + v = str(val).strip().lower() + if v in ("client", "cliente", "c"): + return ClientOrProviderEnum.CLIENT + if v in ("provider", "proveedor", "p"): + return ClientOrProviderEnum.PROVIDER + if v in ("both", "ambos", "b", "cliente y proveedor"): + return ClientOrProviderEnum.BOTH + return None + + +def _parse_active(val: Optional[str]) -> bool: + """Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True.""" + if not val or not str(val).strip(): + return True + v = str(val).strip().lower() + if v in ("1", "true", "si", "sí", "yes", "s", "x"): + return True + if v in ("0", "false", "no", "n"): + return False + return True + + +def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Cliente/Proveedor. Retorna error dict o None.""" + rfc = (row.get("RFC") or "").strip() + if not rfc: + return {"line": line_num, "col": "RFC", "msg": "Requerido"} + if len(rfc) > 30: + return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"} + + tipo_raw = (row.get("TIPO") or "").strip() + if tipo_raw and _parse_client_or_provider(tipo_raw) is None: + return { + "line": line_num, + "col": "TIPO", + "msg": "Valor no válido. Use Cliente, Proveedor o Ambos.", + } + + name = (row.get("NOMBRE") or "").strip() + if len(name) > 256: + return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"} + + short_name = (row.get("SHORT_NAME") or "").strip() + if short_name and len(short_name) > 10: + return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"} + + curp = (row.get("CURP") or "").strip() + if curp and len(curp) > 19: + return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"CP import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"CP import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_client_provider(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"CP import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CP_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"CP import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider. + Upsert por (tenant_id, company_id, rfc). + """ + logger.info(f"CP import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"CP import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + # Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key + existing_by_rfc: Dict[str, ClientProvider] = {} + for cp in ( + session.query(ClientProvider) + .filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ) + .all() + ): + key = (cp.rfc or "").strip() + existing_by_rfc[key] = cp + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_client_provider(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + rfc = _str_or_none(row_norm.get("RFC"), 30) + if not rfc: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "RFC requerido"}) + continue + + client_or_provider = _parse_client_or_provider(row_norm.get("TIPO")) + if client_or_provider is None: + client_or_provider = ClientOrProviderEnum.BOTH + + is_active = _parse_active(row_norm.get("ACTIVO")) + + existing = existing_by_rfc.get(rfc) + if existing: + existing.name = _str_or_none(row_norm.get("NOMBRE"), 256) + existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10) + existing.curp = _str_or_none(row_norm.get("CURP"), 19) + existing.client_or_provider = client_or_provider + existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80) + existing.position = _str_or_none(row_norm.get("POSICION"), 30) + existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19) + existing.is_active = is_active + session.add(existing) + inserted_count += 1 + else: + new_cp = ClientProvider( + tenant_id=tenant_id, + company_id=company_id, + rfc=rfc, + name=_str_or_none(row_norm.get("NOMBRE"), 256), + short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10), + curp=_str_or_none(row_norm.get("CURP"), 19), + client_or_provider=client_or_provider, + responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80), + position=_str_or_none(row_norm.get("POSICION"), 30), + incoterm=_str_or_none(row_norm.get("INCOTERM"), 19), + is_active=is_active, + ) + session.add(new_cp) + session.flush() + existing_by_rfc[rfc] = new_cp + inserted_count += 1 + + # Opcional: crear dirección si hay email/teléfono/dirección + email = _str_or_none(row_norm.get("EMAIL"), 100) + phone = _str_or_none(row_norm.get("TELEFONO"), 30) + address_str = _str_or_none(row_norm.get("DIRECCION"), 100) + if email or phone or address_str: + addr = ClientProviderAddress( + client_id=new_cp.id, + tenant_id=tenant_id, + company_id=company_id, + streets=address_str, + postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15), + city=_str_or_none(row_norm.get("CIUDAD"), 30), + state=_str_or_none(row_norm.get("ESTADO"), 30), + country=_str_or_none(row_norm.get("PAIS"), 3), + phone=phone, + email=email, + contact=_str_or_none(row_norm.get("CONTACTO"), 50), + ) + session.add(addr) + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"CP import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"CP import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"CP import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py new file mode 100644 index 00000000..f0968d9d --- /dev/null +++ b/backend/api/v1/modules/a76/clients_and_providers/imports/template_config.py @@ -0,0 +1,56 @@ +""" +Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "client_providers": [ + {"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]}, + {"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]}, + {"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]}, + {"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]}, + {"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]}, + {"canonical": "CURP", "aliases": []}, + {"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]}, + {"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]}, + {"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]}, + {"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]}, + {"canonical": "ESTADO", "aliases": []}, + {"canonical": "PAIS", "aliases": ["COUNTRY"]}, + {"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]}, + {"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]}, + {"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]}, + {"canonical": "INCOTERM", "aliases": []}, + {"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla client_providers.""" + cols = TEMPLATE_COLUMNS.get("client_providers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/clients_and_providers/routes.py b/backend/api/v1/modules/a76/clients_and_providers/routes.py index b1857de0..09f1cd48 100644 --- a/backend/api/v1/modules/a76/clients_and_providers/routes.py +++ b/backend/api/v1/modules/a76/clients_and_providers/routes.py @@ -20,10 +20,14 @@ from .dto import ( ) from .service import ClientProviderService from .models import ClientProvider +from .imports.routes import router as imports_router # Create main router to add custom endpoints router = APIRouter(prefix="/clients-providers") +# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit) +router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"]) + @router.get("/", response_model=ClientProviderPaginatedResponseDTO) async def get_clients_and_providers( diff --git a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py index 336f1b73..99f7c609 100644 --- a/backend/api/v1/modules/a76/customs_brokers/imports/routes.py +++ b/backend/api/v1/modules/a76/customs_brokers/imports/routes.py @@ -121,6 +121,11 @@ async def get_import_status(job_id: str): return result return {"status": "finished", "result": result} + # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + logger.warning("CB import task %s failed: state=%s", job_id, task_result.state) err_msg = None tb = getattr(task_result, "traceback", None) @@ -135,11 +140,10 @@ async def get_import_status(job_id: str): err_msg = str(exc) except Exception: pass - if not err_msg: - result = getattr(task_result, "result", None) - if result is not None and not isinstance(result, dict): + if not err_msg and result is not None: + if not isinstance(result, dict): err_msg = str(result) - elif isinstance(result, dict) and (result.get("error") or result.get("message")): + elif result.get("error") or result.get("message"): err_msg = result.get("error") or result.get("message") return {"status": "failed", "error": err_msg or "Task failed"} diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 6fc74f9f..6929e5f7 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -27,6 +27,7 @@ celery_app = Celery( "api.v1.modules.a76.reports.exportacion.descargo.task", "api.v1.modules.a76.imports.tasks", "api.v1.modules.a76.customs_brokers.imports.tasks", + "api.v1.modules.a76.clients_and_providers.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task" diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index fd26e485..61ff1f21 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -389,6 +389,21 @@ export const api = { api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {}) }, + // CSV import for Clientes y Proveedores (flujo propio en clients_and_providers/imports) + clientProviderImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/clients-providers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/clients-providers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 3aa127a0..982b080e 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -26,6 +26,8 @@ let showResultModal = $state(false); // Cuando es true, usamos API de importación de Agentes Aduanales (customs_brokers/imports) let useCustomsBrokerImport = $state(false); + // Cuando es true, usamos API de importación de Clientes y Proveedores (clients_and_providers/imports) + let useClientProviderImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -45,6 +47,7 @@ activeModelTarget = config.modelTarget || null; scanResults = null; useCustomsBrokerImport = config.id === 'customs_brokers'; + useClientProviderImport = config.id === 'clients_providers'; const companyId = companyStore.activeCompany?.id || 1; @@ -66,6 +69,24 @@ return; } + if (useClientProviderImport) { + try { + const res = await api.clientProviderImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -102,7 +123,9 @@ try { const res = useCustomsBrokerImport ? await api.customsBrokerImports.status(currentJobId) - : await api.imports.status(currentJobId); + : useClientProviderImport + ? await api.clientProviderImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -116,7 +139,14 @@ toast.success('Escaneo completado. Revisa los resultados.'); isUploading = false; } else if (res.data?.status === 'failed' || res.data?.status === 'FAILURE') { - toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido')); + const errRaw = res.data.error; + const errText = + typeof errRaw === 'string' + ? errRaw.includes('finished') && errRaw.includes('inserted') + ? 'La importación pudo completarse. Revisa el listado de registros.' + : errRaw + : (errRaw?.message ?? 'Error desconocido'); + toast.error('Error en el procesamiento: ' + errText); isUploading = false; currentJobId = null; scanResults = null; @@ -239,7 +269,9 @@ isUploading = true; const res = useCustomsBrokerImport ? await api.customsBrokerImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useClientProviderImport + ? await api.clientProviderImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); From a81e3700fffdabe818de4dbc235a26c78b30f846 Mon Sep 17 00:00:00 2001 From: hreyes Date: Fri, 27 Feb 2026 12:51:46 -0700 Subject: [PATCH 05/16] fix/first-fix-ports --- .gitignore | 2 ++ scripts/init_first_time.sh | 57 +++++++++----------------------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 6beb5975..b8b65467 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,5 @@ postgres-data/ backend/uploads/ docker-compose.yml .mypy_cache/ + +backend/celerybeat-schedule \ No newline at end of file diff --git a/scripts/init_first_time.sh b/scripts/init_first_time.sh index c03a12f9..de1c32ae 100755 --- a/scripts/init_first_time.sh +++ b/scripts/init_first_time.sh @@ -14,9 +14,8 @@ # 8. Licencia Enterprise para el tenant (ilimitada, 1 año de vigencia) # # Requisitos: -# - Keycloak y PostgreSQL corriendo (por ejemplo: docker compose up -d) -# - Keycloak corriendo en http://localhost:8080 -# - PostgreSQL corriendo en localhost:5432 o 5939 +# - Keycloak corriendo en http://localhost:8080 +# - PostgreSQL corriendo en localhost:5432 o 5939 # - Base de datos anexo76_core creada # - jq instalado (para procesamiento JSON) # @@ -127,15 +126,11 @@ create_tenant_mapper() { } # Variables de configuración (puertos con prefijo 1 hardcodeados) -KEYCLOAK_URL="http://localhost:18080/kcauth" +KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080/kcauth}" KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}" KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}" KEYCLOAK_REALM="${KEYCLOAK_REALM:-master}" -<<<<<<< HEAD -KEYCLOACK_ADMIN_URL="http://localhost:19000/kcauth" -======= KEYCLOAK_ADMIN_URL="${KEYCLOAK_ADMIN_URL:-http://localhost:9000/kcauth}" ->>>>>>> origin/development POSTGRES_HOST="${POSTGRES_HOST:-localhost}" POSTGRES_PORT="${POSTGRES_PORT:-5432}" @@ -149,7 +144,7 @@ DEMO_EMAIL="demo@aduanasoft.com" DEMO_FIRSTNAME="Demo" DEMO_LASTNAME="User" -TENANT_NAME="Aduanasoft" +TENANT_NAME="Aduanasoft"A TENANT_SLUG="aduanasoft" COMPANY_NAME="Aduanasoft S.A. de C.V." COMPANY_RFC="ADS010101AAA" @@ -280,13 +275,13 @@ else "directAccessGrantsEnabled": true, "standardFlowEnabled": true, "implicitFlowEnabled": false, - "rootUrl": "http://localhost:18000/api", - "baseUrl": "http://localhost:18000/api", + "rootUrl": "http://localhost:8000/api", + "baseUrl": "http://localhost:8000/api", "redirectUris": [ - "http://localhost:18000/*", - "http://localhost:15173/*" + "http://localhost:8000/*", + "http://localhost:5173/*" ], - "webOrigins": ["http://localhost:18000/api"], + "webOrigins": ["http://localhost:8000/api"], "attributes": { "access.token.lifespan": "3600", "client.secret.creation.time": "0" @@ -400,27 +395,6 @@ else CREATE_FRONTEND=$(curl -s -w "\n%{http_code}" -X POST "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/clients" \ -H "Authorization: Bearer ${ACCESS_TOKEN}" \ -H "Content-Type: application/json" \ -<<<<<<< HEAD - -d "{ - \"clientId\": \"anexo76-frontend\", - \"name\": \"Anexo76 Frontend\", - \"description\": \"Aplicación web frontend para el sistema Anexo76\", - \"enabled\": true, - \"protocol\": \"openid-connect\", - \"publicClient\": true, - \"directAccessGrantsEnabled\": false, - \"standardFlowEnabled\": true, - \"implicitFlowEnabled\": false, - \"rootUrl\": \"http://localhost:15173\", - \"baseUrl\": \"http://localhost:15173\", - \"redirectUris\": [ - \"http://localhost:15173/*\", - \"http://localhost:13000/*\" - ], - \"webOrigins\": [ - \"http://localhost:15173\", - \"http://localhost:13000\" -======= -d '{ "clientId": "anexo76-frontend", "name": "Anexo76 Frontend", @@ -442,12 +416,11 @@ else "http://localhost:5173", "http://localhost:5174", "http://localhost:3000" ->>>>>>> origin/development ], - \"attributes\": { - \"pkce.code.challenge.method\": \"S256\" + "attributes": { + "pkce.code.challenge.method": "S256" } - }") + }') HTTP_CODE=$(echo "$CREATE_FRONTEND" | tail -n1) RESPONSE_BODY=$(echo "$CREATE_FRONTEND" | head -n -1) @@ -666,12 +639,8 @@ echo -e "${GREEN}✓ Atributo tenant_id asignado al usuario${NC}" # Agregar relación usuario-tenant en la base de datos (usar company_id real) echo -e "\n${YELLOW}Creando relación usuario-tenant en la base de datos...${NC}" -<<<<<<< HEAD -exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, ${COMPANY_ID}, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null -======= exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null exec_pg_sql_client "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null ->>>>>>> origin/development echo -e "${GREEN}✓ Relación usuario-tenant creada en la base de datos (Hub y Cliente)${NC}" @@ -768,5 +737,5 @@ echo -e " ${GREEN}✓${NC} Status: Activa" echo -e " ${GREEN}✓${NC} Features: API, Reportes Avanzados, Integraciones, Soporte Dedicado" echo -e " ${GREEN}✓${NC} Vigencia: 1 año" echo -e "\n${YELLOW}Puedes acceder al sistema en:${NC}" -echo -e " ${GREEN}http://localhost:15173${NC}" +echo -e " ${GREEN}http://localhost:5173${NC}" echo -e "\n${GREEN}════════════════════════════════════════════════════════${NC}\n" From e5e1287a6b1803435bc5003ab4d46af22fbed733 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 2 Mar 2026 07:24:09 -0700 Subject: [PATCH 06/16] feature/csv-exchage-rate --- .../exchange_rate/imports/__init__.py | 1 + .../exchange_rate/imports/routes.py | 160 ++++++ .../exchange_rate/imports/schemas.py | 25 + .../exchange_rate/imports/tasks.py | 465 ++++++++++++++++++ .../exchange_rate/imports/template_config.py | 42 ++ .../general_catalogs/exchange_rate/routes.py | 4 + backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 29 +- 9 files changed, 740 insertions(+), 2 deletions(-) create mode 100644 backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py new file mode 100644 index 00000000..4ac8a216 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/__init__.py @@ -0,0 +1 @@ +# Exchange rate CSV import module diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py new file mode 100644 index 00000000..75a9aef8 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Tipos de Cambio. +Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + ER_IMPORT_FILE_PREFIX, + ER_IMPORT_META_PREFIX, + ER_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"ER import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "exchange_rates", + } + + try: + r = _get_redis() + r.set( + f"{ER_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=ER_IMPORT_REDIS_TTL, + ) + r.set( + f"{ER_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=ER_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"ER import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"ER import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("ER import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py new file mode 100644 index 00000000..c214b80b --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/tasks.py @@ -0,0 +1,465 @@ +""" +Tareas Celery para importación CSV de Tipos de Cambio. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +ER_IMPORT_FILE_PREFIX = "er_import_file:" +ER_IMPORT_META_PREFIX = "er_import_meta:" +ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:" +ER_IMPORT_REDIS_TTL = 3600 # 1 hour + +DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"] +TEMPLATE_ID = "exchange_rates" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"ER import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"er_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"ER import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{ER_IMPORT_FILE_PREFIX}{job_id}", + f"{ER_IMPORT_META_PREFIX}{job_id}", + f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"ER import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _parse_date(val: Optional[str]) -> Optional[datetime]: + """Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc.""" + if not val or not str(val).strip(): + return None + raw = str(val).strip() + for fmt in DATE_FORMATS: + try: + parsed = datetime.strptime(raw, fmt) + return datetime.combine(parsed.date(), time.min) + except ValueError: + continue + return None + + +def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Tipo de Cambio. Retorna error dict o None.""" + fecha_raw = (row.get("FECHA") or "").strip() + if not fecha_raw: + return {"line": line_num, "col": "FECHA", "msg": "Requerido"} + if _parse_date(fecha_raw) is None: + return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"} + + valor_raw = (row.get("VALOR") or "").strip() + if not valor_raw: + return {"line": line_num, "col": "VALOR", "msg": "Requerido"} + try: + v = float(valor_raw.replace(",", ".")) + if v <= 0: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"} + except ValueError: + return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"} + + local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper() + if local_raw and len(local_raw) > 7: + return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"} + + foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper() + if foreign_raw and len(foreign_raw) > 7: + return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"ER import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"ER import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_exchange_rate(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"ER import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=ER_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"ER import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha). + """ + logger.info(f"ER import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"er_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"ER import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_date: Dict[tuple, ExchangeRate] = {} + for er in ( + session.query(ExchangeRate) + .filter( + ExchangeRate.tenant_id == tenant_id, + ExchangeRate.company_id == company_id, + ) + .all() + ): + d = er.date.date() if hasattr(er.date, "date") else er.date + existing_by_date[(tenant_id, company_id, d)] = er + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_exchange_rate(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + parsed_date = _parse_date(row_norm.get("FECHA")) + if not parsed_date: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FECHA: no parseable"}) + continue + + try: + v = float((row_norm.get("VALOR") or "").strip().replace(",", ".")) + value_decimal = Decimal(str(round(v, 6))) + except (ValueError, TypeError): + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "VALOR: no numérico"}) + continue + + local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7) + if local_currency: + local_currency = local_currency.upper() + foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7) + if foreign_currency: + foreign_currency = foreign_currency.upper() + + key_date = parsed_date.date() + existing = existing_by_date.get((tenant_id, company_id, key_date)) + + if existing: + existing.value = value_decimal + existing.local_currency = local_currency or None + existing.foreign_currency = foreign_currency or None + session.add(existing) + inserted_count += 1 + else: + new_er = ExchangeRate( + tenant_id=tenant_id, + company_id=company_id, + date=parsed_date, + value=value_decimal, + local_currency=local_currency, + foreign_currency=foreign_currency, + ) + session.add(new_er) + existing_by_date[(tenant_id, company_id, key_date)] = new_er + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"ER import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"ER import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"ER import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py new file mode 100644 index 00000000..c907f9a0 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/imports/template_config.py @@ -0,0 +1,42 @@ +""" +Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "exchange_rates": [ + {"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]}, + {"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]}, + {"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]}, + {"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla exchange_rates.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py index 22ab8ec9..09086071 100644 --- a/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/exchange_rate/routes.py @@ -34,6 +34,10 @@ crud_router = route_handler.router from fastapi import APIRouter custom_router = APIRouter(prefix="/exchange-rate", tags=[]) +# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered +from .imports.routes import router as imports_router +custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"]) + @custom_router.get("/test-ping") async def test_ping(): return {"message": "pong"} diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index c9d7c252..81bb297d 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -35,6 +35,7 @@ celery_app.conf.update( "api.v1.modules.a76.imports.tasks", "api.v1.modules.a76.customs_brokers.imports.tasks", "api.v1.modules.a76.clients_and_providers.imports.tasks", + "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 10aff063..1fe6d343 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -404,6 +404,21 @@ export const api = { api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {}) }, + // CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports) + exchangeRateImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/exchange-rate/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 982b080e..5f174089 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -28,6 +28,8 @@ let useCustomsBrokerImport = $state(false); // Cuando es true, usamos API de importación de Clientes y Proveedores (clients_and_providers/imports) let useClientProviderImport = $state(false); + // Cuando es true, usamos API de importación de Tipos de Cambio (exchange_rate/imports) + let useExchangeRateImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -48,6 +50,7 @@ scanResults = null; useCustomsBrokerImport = config.id === 'customs_brokers'; useClientProviderImport = config.id === 'clients_providers'; + useExchangeRateImport = config.id === 'exchange_rates'; const companyId = companyStore.activeCompany?.id || 1; @@ -87,6 +90,24 @@ return; } + if (useExchangeRateImport) { + try { + const res = await api.exchangeRateImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -125,7 +146,9 @@ ? await api.customsBrokerImports.status(currentJobId) : useClientProviderImport ? await api.clientProviderImports.status(currentJobId) - : await api.imports.status(currentJobId); + : useExchangeRateImport + ? await api.exchangeRateImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -271,7 +294,9 @@ ? await api.customsBrokerImports.commit(currentJobId) : useClientProviderImport ? await api.clientProviderImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useExchangeRateImport + ? await api.exchangeRateImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); From 442b188a45dd337dfa0b1991eb337f9bff40b0f2 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 2 Mar 2026 07:49:53 -0700 Subject: [PATCH 07/16] feature/csv-fracc-ame --- .../us_tariff_fractions/imports/__init__.py | 1 + .../us_tariff_fractions/imports/routes.py | 160 ++++++ .../us_tariff_fractions/imports/schemas.py | 25 + .../us_tariff_fractions/imports/tasks.py | 474 ++++++++++++++++++ .../imports/template_config.py | 47 ++ .../fractions/us_tariff_fractions/routes.py | 18 +- backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 29 +- 9 files changed, 763 insertions(+), 7 deletions(-) create mode 100644 backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py new file mode 100644 index 00000000..cff9f961 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py new file mode 100644 index 00000000..bf5083a3 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Fracción Americana (US Tariff Fractions). +Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + FA_IMPORT_FILE_PREFIX, + FA_IMPORT_META_PREFIX, + FA_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"FA import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "us_tariff_fractions", + } + + try: + r = _get_redis() + r.set( + f"{FA_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=FA_IMPORT_REDIS_TTL, + ) + r.set( + f"{FA_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=FA_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"FA import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"FA import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("FA import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py new file mode 100644 index 00000000..98c39a98 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py new file mode 100644 index 00000000..edac5df2 --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/tasks.py @@ -0,0 +1,474 @@ +""" +Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions). +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +FA_IMPORT_FILE_PREFIX = "fa_import_file:" +FA_IMPORT_META_PREFIX = "fa_import_meta:" +FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:" +FA_IMPORT_REDIS_TTL = 3600 # 1 hour + +TEMPLATE_ID = "us_tariff_fractions" + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"FA import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"fa_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"FA import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{FA_IMPORT_FILE_PREFIX}{job_id}", + f"{FA_IMPORT_META_PREFIX}{job_id}", + f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"FA import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _normalize_code(raw: Optional[str]) -> str: + """Normalize fraction code: strip and remove dots/dashes, max 16 chars.""" + if not raw: + return "" + s = str(raw).strip().replace(".", "").replace("-", "") + return s[:16] if len(s) > 16 else s + + +def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Fracción Americana. Retorna error dict o None.""" + code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip() + if not code_raw: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} + code_norm = _normalize_code(code_raw) + if not code_norm: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"} + if len(code_norm) > 16: + return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"} + + prefix_raw = (row.get("PREFIJO") or "").strip() + if prefix_raw and len(prefix_raw) > 10: + return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"} + + um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip() + if um_raw and len(um_raw) > 10: + return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"} + + tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip() + if tipo_raw and len(tipo_raw) > 10: + return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"} + + adv_pct = row.get("ADVALOREM_PCT") + if adv_pct is not None and str(adv_pct).strip(): + try: + v = float(str(adv_pct).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"} + + adv_dlls = row.get("ADVALOREM_DLLS") + if adv_dlls is not None and str(adv_dlls).strip(): + try: + v = float(str(adv_dlls).strip().replace(",", ".")) + if v < 0: + return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"} + except ValueError: + return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"FA import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"FA import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_us_tariff_fraction(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"FA import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=FA_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"FA import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _parse_float(val: Any) -> Optional[float]: + if val is None or str(val).strip() == "": + return None + try: + return float(str(val).strip().replace(",", ".")) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code). + """ + logger.info(f"FA import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"FA import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, TEMPLATE_ID) + err = _validate_row_us_tariff_fraction(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA")) + if not code: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"}) + continue + + prefix = _str_or_none(row_norm.get("PREFIJO"), 10) + unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10) + description = _str_or_none(row_norm.get("DESCRIPCION")) + type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10) + ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT")) + fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS")) + fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None + + existing = ( + session.query(USTariffFraction) + .filter( + USTariffFraction.tenant_id == tenant_id, + USTariffFraction.company_id == company_id, + USTariffFraction.code == code, + ) + .first() + ) + + if existing: + existing.prefix = prefix + existing.type_code = type_code + existing.ad_valorem = ad_valorem + existing.fixed_cost = fixed_cost + existing.unit_of_measure = unit_of_measure + existing.description = description + session.add(existing) + inserted_count += 1 + else: + new_row = USTariffFraction( + tenant_id=tenant_id, + company_id=company_id, + code=code, + prefix=prefix, + type_code=type_code, + ad_valorem=ad_valorem, + fixed_cost=fixed_cost, + unit_of_measure=unit_of_measure, + description=description, + ) + session.add(new_row) + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"FA import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"FA import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path_clean = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path_clean): + os.remove(meta_path_clean) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"FA import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py new file mode 100644 index 00000000..aa5a5e7f --- /dev/null +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/imports/template_config.py @@ -0,0 +1,47 @@ +""" +Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "us_tariff_fractions": [ + {"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]}, + {"canonical": "PREFIJO"}, + {"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]}, + {"canonical": "DESCRIPCION"}, + {"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]}, + {"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]}, + {"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla us_tariff_fractions.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template( + row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions" +) -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py index 158bf765..61907ffa 100644 --- a/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/fractions/us_tariff_fractions/routes.py @@ -17,23 +17,28 @@ from .dto import ( ) from .service import USTariffFractionService -# Create router using TenantCRUDRoutes factory for basic CRUD operations +# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router) crud_router = TenantCRUDRoutes( service=USTariffFractionService, create_schema=USTariffFractionCreateDTO, update_schema=USTariffFractionUpdateDTO, response_schema=USTariffFractionResponseDTO, - prefix="/us-tariff-fractions", + prefix="", tags=["a76 / general catalogs / us tariff fractions"], resource_name="US Tariff Fraction", id_name="id", enable_list=False, # We implement our custom list endpoint ) -router = crud_router.router +# Master router with prefix so all routes live under /us-tariff-fractions +from .imports.routes import router as imports_router +main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"]) +main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"]) +main_router.include_router(crud_router.router) -# Custom list endpoint with search filter -@router.get( + +# Custom list endpoint with search filter (under /us-tariff-fractions/) +@main_router.get( "/", response_model=Dict[str, Any], summary="List US Tariff Fractions", @@ -65,3 +70,6 @@ async def list_us_tariff_fractions( "page_size": page_size, "pages": (total + page_size - 1) // page_size, } + + +router = main_router diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 81bb297d..41a98b99 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -36,6 +36,7 @@ celery_app.conf.update( "api.v1.modules.a76.customs_brokers.imports.tasks", "api.v1.modules.a76.clients_and_providers.imports.tasks", "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", + "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1fe6d343..19d4d82c 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -419,6 +419,21 @@ export const api = { api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {}) }, + // CSV import for Fracción Americana (us_tariff_fractions/imports) + americanFractionImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/us-tariff-fractions/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/us-tariff-fractions/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 5f174089..05471225 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -30,6 +30,8 @@ let useClientProviderImport = $state(false); // Cuando es true, usamos API de importación de Tipos de Cambio (exchange_rate/imports) let useExchangeRateImport = $state(false); + // Cuando es true, usamos API de importación de Fracción Americana (us_tariff_fractions/imports) + let useAmericanFractionImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -51,6 +53,7 @@ useCustomsBrokerImport = config.id === 'customs_brokers'; useClientProviderImport = config.id === 'clients_providers'; useExchangeRateImport = config.id === 'exchange_rates'; + useAmericanFractionImport = config.id === 'american_fractions'; const companyId = companyStore.activeCompany?.id || 1; @@ -108,6 +111,24 @@ return; } + if (useAmericanFractionImport) { + try { + const res = await api.americanFractionImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -148,7 +169,9 @@ ? await api.clientProviderImports.status(currentJobId) : useExchangeRateImport ? await api.exchangeRateImports.status(currentJobId) - : await api.imports.status(currentJobId); + : useAmericanFractionImport + ? await api.americanFractionImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -296,7 +319,9 @@ ? await api.clientProviderImports.commit(currentJobId) : useExchangeRateImport ? await api.exchangeRateImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useAmericanFractionImport + ? await api.americanFractionImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); From 9473c05602f84b9cb63d97959e65c12a5ab740e7 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 2 Mar 2026 10:03:45 -0700 Subject: [PATCH 08/16] feature/csv-pedimento --- .../a76/pedmientos/imports/__init__.py | 1 + .../modules/a76/pedmientos/imports/routes.py | 160 +++++ .../modules/a76/pedmientos/imports/schemas.py | 25 + .../modules/a76/pedmientos/imports/tasks.py | 563 ++++++++++++++++++ .../a76/pedmientos/imports/template_config.py | 53 ++ .../api/v1/modules/a76/pedmientos/router.py | 6 + backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 37 +- scripts/backend-entrypoint.sh | 2 +- 10 files changed, 857 insertions(+), 6 deletions(-) create mode 100644 backend/api/v1/modules/a76/pedmientos/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/pedmientos/imports/routes.py create mode 100644 backend/api/v1/modules/a76/pedmientos/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/pedmientos/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/pedmientos/imports/template_config.py diff --git a/backend/api/v1/modules/a76/pedmientos/imports/__init__.py b/backend/api/v1/modules/a76/pedmientos/imports/__init__.py new file mode 100644 index 00000000..8378bf88 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Pedimentos (upload → scan → commit) diff --git a/backend/api/v1/modules/a76/pedmientos/imports/routes.py b/backend/api/v1/modules/a76/pedmientos/imports/routes.py new file mode 100644 index 00000000..f624e5dc --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/routes.py @@ -0,0 +1,160 @@ +""" +Rutas de importación CSV para Pedimentos. +Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + PED_IMPORT_FILE_PREFIX, + PED_IMPORT_META_PREFIX, + PED_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo. + """ + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Pedimentos import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "pedimentos", + } + + try: + r = _get_redis() + r.set( + f"{PED_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=PED_IMPORT_REDIS_TTL, + ) + r.set( + f"{PED_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=PED_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Pedimentos import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Pedimentos import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + """ + Polling: estado del escaneo o del commit. + """ + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = task_result.info or {} + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + """ + Fase 2: Usuario confirma; se encola la inserción de filas válidas. + """ + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/pedmientos/imports/schemas.py b/backend/api/v1/modules/a76/pedmientos/imports/schemas.py new file mode 100644 index 00000000..6fe69385 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommitRequest(BaseModel): + pass # no body needed for single model + + +class ImportJobStatus(BaseModel): + status: str + job_id: Optional[str] = None + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/pedmientos/imports/tasks.py b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py new file mode 100644 index 00000000..82b630d2 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/tasks.py @@ -0,0 +1,563 @@ +""" +Tareas Celery para importación CSV de Pedimentos. +Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from datetime import datetime +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +# Redis keys (prefijo propio para no colisionar con otros imports) +PED_IMPORT_FILE_PREFIX = "ped_import_file:" +PED_IMPORT_META_PREFIX = "ped_import_meta:" +PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" +PED_IMPORT_REDIS_TTL = 3600 # 1 hour + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"ped_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{PED_IMPORT_FILE_PREFIX}{job_id}", + f"{PED_IMPORT_META_PREFIX}{job_id}", + f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_pedimento( + row: Dict[str, Any], + line_num: int, + valid_client_ids: Set[int], + valid_regimes: Set[str], + valid_pedimento_codes: Set[str], +) -> Optional[Dict[str, Any]]: + """Valida una fila para Pedimento. Retorna error dict o None.""" + # Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN + year = (row.get("AÑO") or "").strip() + if not year: + return {"line": line_num, "col": "AÑO", "msg": "Requerido"} + if len(year) > 2: + return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"} + + customs_office = (row.get("ADUANA") or "").strip() + if not customs_office: + return {"line": line_num, "col": "ADUANA", "msg": "Requerido"} + if len(customs_office) > 3: + return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"} + + license_val = (row.get("PATENTE") or "").strip() + if not license_val: + return {"line": line_num, "col": "PATENTE", "msg": "Requerido"} + if len(license_val) > 4: + return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"} + + pedimento_number = (row.get("NUMERO") or "").strip() + if not pedimento_number: + return {"line": line_num, "col": "NUMERO", "msg": "Requerido"} + if len(pedimento_number) > 7: + return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"} + + client_id_str = (row.get("CLIENTE_ID") or "").strip() + if not client_id_str: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"} + try: + client_id = int(client_id_str) + except ValueError: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"} + if client_id not in valid_client_ids: + return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"} + + pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip() + if not pedimento_code: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"} + if len(pedimento_code) > 2: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"} + if pedimento_code not in valid_pedimento_codes: + return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"} + + regime = (row.get("REGIMEN") or "").strip() + if not regime: + return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"} + if len(regime) > 3: + return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"} + if regime not in valid_regimes: + return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"} + + # Optional numeric/string fields - validate format if present + status = (row.get("ESTATUS") or "").strip() + if status and len(status) > 30: + return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"} + + for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]: + val = (row.get(col) or "").strip() + if not val: + continue + try: + Decimal(val.replace(",", ".")) + except (InvalidOperation, ValueError): + return {"line": line_num, "col": col, "msg": "Valor numérico inválido"} + + operation_type = (row.get("TIPO_OPERACION") or "").strip().lower() + if operation_type and operation_type not in ("imp", "exp", ""): + return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"} + + pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower() + if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""): + return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"} + + return None + + +def _parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + return Decimal(str(val).strip().replace(",", ".")) + except (InvalidOperation, ValueError): + return None + + +def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]: + """Build dict for PedimentosCreate from normalized CSV row (canonical names).""" + year = (row.get("AÑO") or "").strip()[:2] + customs_office = (row.get("ADUANA") or "").strip()[:3] + license_val = (row.get("PATENTE") or "").strip()[:4] + pedimento_number = (row.get("NUMERO") or "").strip()[:7] + client_id_str = (row.get("CLIENTE_ID") or "").strip() + client_id = int(client_id_str) if client_id_str else None + pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2] + regime = (row.get("REGIMEN") or "").strip()[:3] + + data = { + "year": year, + "customs_office": customs_office, + "license": license_val, + "pedimento_number": pedimento_number, + "client_id": client_id, + "pedimento_code": pedimento_code, + "regime": regime, + } + + op = (row.get("TIPO_OPERACION") or "").strip().lower() + if op in ("imp", "exp"): + data["operation_type"] = op + + ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower() + if ptype in ("normal", "consolidated", "complementary", "automobile"): + data["pedimento_type"] = ptype + + status = (row.get("ESTATUS") or "").strip() + if status: + data["status"] = status[:30] + + data["usd_value"] = _parse_decimal(row.get("VALOR_USD")) + data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO")) + data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO")) + data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO")) + + obs = (row.get("OBSERVACIONES") or "").strip() + if obs: + data["observations"] = obs + + return data + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"Pedimentos import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Pedimentos import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + # Load valid FK sets for validation + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + for cp in session.query(ClientProvider).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + except Exception as e: + logger.error(f"Pedimentos import: failed to load FK sets: {e}") + return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header, "pedimentos") + err = _validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Pedimentos import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=PED_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create. + """ + logger.info(f"Pedimentos import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + # Reload FK sets for commit-time validation + valid_client_ids: Set[int] = set() + valid_regimes: Set[str] = set() + valid_pedimento_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + + for cp in session.query(ClientProvider).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_client_ids.add(cp.id) + for r in session.query(RegimenPedimento).all(): + valid_regimes.add(r.code) + for pc in session.query(PedimentoCode).all(): + valid_pedimento_codes.add(pc.code) + except Exception as e: + logger.error(f"Pedimentos import: failed to load FK sets: {e}") + return {"status": "failed", "error": str(e)} + + from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate + from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate + from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService + + inserted_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header, "pedimentos") + err = _validate_row_pedimento( + row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + try: + data = _row_to_pedimentos_create(row_norm) + # Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL + if "pedimento_dates" not in data or data.get("pedimento_dates") is None: + data["pedimento_dates"] = PedimentoDatesCreate( + entry_date=datetime.now(), + end_date=datetime.now(), + ) + create_data = PedimentosCreate(**data) + PedimentosService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except ValueError as ve: + if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): + skipped_duplicate += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + else: + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(ve)}) + except Exception as e: + logger.warning(f"Pedimentos import line {i}: {e}") + skipped_invalid += 1 + skipped_details.append({"line": i, "reason": str(e)}) + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Pedimentos import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + meta_path = file_path.replace(".csv", ".meta.json") + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/pedmientos/imports/template_config.py b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py new file mode 100644 index 00000000..06fea167 --- /dev/null +++ b/backend/api/v1/modules/a76/pedmientos/imports/template_config.py @@ -0,0 +1,53 @@ +""" +Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls). +Solo se leen columnas definidas aquí; el resto se ignora. +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "pedimentos": [ + {"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]}, + {"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]}, + {"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]}, + {"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]}, + {"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]}, + {"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]}, + {"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]}, + {"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]}, + {"canonical": "REGIMEN", "aliases": ["REGIME"]}, + {"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]}, + {"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]}, + {"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]}, + {"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]}, + {"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]}, + {"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]: + """normalized_header -> canonical_name para plantilla pedimentos.""" + cols = TEMPLATE_COLUMNS.get(template_id) + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]: + """Fila CSV con solo columnas de la plantilla, en nombres canónicos.""" + lookup = build_normalized_lookup(normalize_header_fn, template_id) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/pedmientos/router.py b/backend/api/v1/modules/a76/pedmientos/router.py index f294cba4..2b987b46 100644 --- a/backend/api/v1/modules/a76/pedmientos/router.py +++ b/backend/api/v1/modules/a76/pedmientos/router.py @@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import ( from .routes.pedimento_transport_means import router as pedimento_transport_means_router from .routes.pedimento_validation import router as pedimento_validation_router from .routes.pedimentos import router as pedimentos_router +from .imports.routes import router as pedimentos_imports_router router = APIRouter() @@ -117,3 +118,8 @@ router.include_router( router.include_router( pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"] ) +router.include_router( + pedimentos_imports_router, + prefix="/pedimentos/imports", + tags=["a76 / pedimentos / csv_import"], +) diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 41a98b99..f0b91d0a 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -35,6 +35,7 @@ celery_app.conf.update( "api.v1.modules.a76.imports.tasks", "api.v1.modules.a76.customs_brokers.imports.tasks", "api.v1.modules.a76.clients_and_providers.imports.tasks", + "api.v1.modules.a76.pedmientos.imports.tasks", "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 19d4d82c..55964f53 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -434,6 +434,21 @@ export const api = { api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {}) }, + // CSV import for Pedimentos (pedimentos/imports) + pedimentosImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/pedimentos/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/pedimentos/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 05471225..7b2f5aa0 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -32,6 +32,8 @@ let useExchangeRateImport = $state(false); // Cuando es true, usamos API de importación de Fracción Americana (us_tariff_fractions/imports) let useAmericanFractionImport = $state(false); + // Cuando es true, usamos API de importación de Pedimentos (pedimentos/imports) + let usePedimentosImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -54,6 +56,7 @@ useClientProviderImport = config.id === 'clients_providers'; useExchangeRateImport = config.id === 'exchange_rates'; useAmericanFractionImport = config.id === 'american_fractions'; + usePedimentosImport = config.id === 'pedimentos'; const companyId = companyStore.activeCompany?.id || 1; @@ -129,6 +132,24 @@ return; } + if (usePedimentosImport) { + try { + const res = await api.pedimentosImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -171,7 +192,9 @@ ? await api.exchangeRateImports.status(currentJobId) : useAmericanFractionImport ? await api.americanFractionImports.status(currentJobId) - : await api.imports.status(currentJobId); + : usePedimentosImport + ? await api.pedimentosImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -205,7 +228,8 @@ const inserted = res.data?.inserted || 0; const skippedInvalid = res.data?.skipped_invalid || 0; const skippedFk = res.data?.skipped_missing_fk || 0; - const totalSkipped = skippedInvalid + skippedFk; + const skippedDup = res.data?.skipped_duplicate || 0; + const totalSkipped = skippedInvalid + skippedFk + skippedDup; if (inserted === 0) { toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`); @@ -219,12 +243,13 @@ const inserted = res.data?.inserted || 0; const skippedInvalid = res.data?.skipped_invalid || 0; const skippedFk = res.data?.skipped_missing_fk || 0; + const skippedDup = res.data?.skipped_duplicate || 0; const skippedDetails = res.data?.skipped_details || []; if (inserted > 0) { toast.success(`Importación completada: ${inserted} registros insertados`); - if (skippedInvalid > 0 || skippedFk > 0) { - const totalSkipped = skippedInvalid + skippedFk; + if (skippedInvalid > 0 || skippedFk > 0 || skippedDup > 0) { + const totalSkipped = skippedInvalid + skippedFk + skippedDup; toast.warning(`${totalSkipped} registros fueron rechazados`); } } else { @@ -321,7 +346,9 @@ ? await api.exchangeRateImports.commit(currentJobId) : useAmericanFractionImport ? await api.americanFractionImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : usePedimentosImport + ? await api.pedimentosImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); diff --git a/scripts/backend-entrypoint.sh b/scripts/backend-entrypoint.sh index 20c5f89f..85cb5432 100755 --- a/scripts/backend-entrypoint.sh +++ b/scripts/backend-entrypoint.sh @@ -19,7 +19,7 @@ wait_for_tcp() { echo "Esperando a que $service esté disponible en ${host}:${port}..." while [ $attempt -le $max_attempts ]; do - if python -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then + if python3 -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then echo "✓ $service está listo y accesible" return 0 fi From 2d05ea23cfae2ecefe8e099fe7443f2e600599a3 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 2 Mar 2026 11:33:51 -0700 Subject: [PATCH 09/16] feature/csv-clases-materiales --- .../api/v1/modules/a76/classes/__init__.py | 16 +- .../modules/a76/classes/imports/__init__.py | 1 + .../v1/modules/a76/classes/imports/routes.py | 151 +++++ .../v1/modules/a76/classes/imports/schemas.py | 21 + .../v1/modules/a76/classes/imports/tasks.py | 528 ++++++++++++++++++ .../a76/classes/imports/template_config.py | 45 ++ backend/api/v1/modules/a76/classes/routes.py | 4 + backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 29 +- 10 files changed, 808 insertions(+), 3 deletions(-) create mode 100644 backend/api/v1/modules/a76/classes/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/classes/imports/routes.py create mode 100644 backend/api/v1/modules/a76/classes/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/classes/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/classes/imports/template_config.py diff --git a/backend/api/v1/modules/a76/classes/__init__.py b/backend/api/v1/modules/a76/classes/__init__.py index 1a3d8bed..fa36b353 100644 --- a/backend/api/v1/modules/a76/classes/__init__.py +++ b/backend/api/v1/modules/a76/classes/__init__.py @@ -2,6 +2,20 @@ Módulo de Class """ -from .routes import router +from typing import Any __all__ = ["router"] + + +def __getattr__(name: str) -> Any: + """ + Lazy export to avoid circular imports during Celery init. + + This package is imported when models are loaded (e.g. a76.items.models), + so importing FastAPI routes at import-time can break Celery startup. + """ + if name == "router": + from .routes import router + + return router + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/backend/api/v1/modules/a76/classes/imports/__init__.py b/backend/api/v1/modules/a76/classes/imports/__init__.py new file mode 100644 index 00000000..e9eb448f --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Clases de Materiales (upload → scan → status → commit). diff --git a/backend/api/v1/modules/a76/classes/imports/routes.py b/backend/api/v1/modules/a76/classes/imports/routes.py new file mode 100644 index 00000000..5c9c869f --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para Clases de Materiales. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + CLS_IMPORT_FILE_PREFIX, + CLS_IMPORT_META_PREFIX, + CLS_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Classes import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "material_classes", + } + + try: + r = _get_redis() + r.set( + f"{CLS_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=CLS_IMPORT_REDIS_TTL, + ) + r.set( + f"{CLS_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=CLS_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Classes import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Classes import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/classes/imports/schemas.py b/backend/api/v1/modules/a76/classes/imports/schemas.py new file mode 100644 index 00000000..2d362f01 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/schemas.py @@ -0,0 +1,21 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/classes/imports/tasks.py b/backend/api/v1/modules/a76/classes/imports/tasks.py new file mode 100644 index 00000000..ca754d06 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/tasks.py @@ -0,0 +1,528 @@ +""" +Tareas Celery para importación CSV de Clases de Materiales. +Flujo: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +CLS_IMPORT_FILE_PREFIX = "cls_import_file:" +CLS_IMPORT_META_PREFIX = "cls_import_meta:" +CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" +CLS_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Classes import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"cls_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Classes import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{CLS_IMPORT_FILE_PREFIX}{job_id}", + f"{CLS_IMPORT_META_PREFIX}{job_id}", + f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Classes import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_class( + row: Dict[str, Any], + line_num: int, + valid_material_keys: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + class_code = (row.get("CLASE") or "").strip() + if not class_code: + return {"line": line_num, "col": "CLASE", "msg": "Requerido"} + if len(class_code) > 8: + return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} + + desc_es = (row.get("DESCRIPCIONE") or "").strip() + if desc_es and len(desc_es) > 500: + return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} + desc_en = (row.get("DESCRIPCIONI") or "").strip() + if desc_en and len(desc_en) > 500: + return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} + + material_key = (row.get("CLAVEMAT") or "").strip() + if material_key: + if len(material_key) > 10: + return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"} + if valid_material_keys is not None and material_key not in valid_material_keys: + return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"} + + uom = (row.get("UNIMED") or "").strip() + if uom: + if len(uom) > 5: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} + if valid_uom_codes is not None and uom not in valid_uom_codes: + return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"} + + fraction = (row.get("FRACCION") or "").strip() + if fraction and len(fraction) > 20: + return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"} + us_fraction = (row.get("FRACCIONAME") or "").strip() + if us_fraction and len(us_fraction) > 16: + return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} + sub_key = (row.get("CLAVESUB") or "").strip() + if sub_key and len(sub_key) > 5: + return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"} + iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip() + if iva_exempt and len(iva_exempt) > 4: + return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"} + + rev_fisica = row.get("REVFISICA") + if rev_fisica is not None and rev_fisica != "": + try: + v = int(rev_fisica) + if v < -32768 or v > 32767: + return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"} + except (ValueError, TypeError): + return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Classes import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Classes import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + logger.warning(f"Classes import: could not load FK sets: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_class( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Classes import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=CLS_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Classes import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Classes import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Classes import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.classes.models import Class + + valid_material_keys: Set[str] = set() + valid_uom_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.public.reference_data.material_types.models import MaterialType + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + for m in session.query(MaterialType.key).all(): + valid_material_keys.add(m[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + except Exception as e: + logger.warning(f"Classes import: could not load FK sets: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_code: Dict[str, Class] = {} + for c in ( + session.query(Class) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + existing_by_code[c.class_code] = c + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_class( + row_norm, i, + valid_material_keys=valid_material_keys, + valid_uom_codes=valid_uom_codes, + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + class_code = _str_or_none(row_norm.get("CLASE"), 8) + if not class_code: + skipped_invalid += 1 + continue + + existing = existing_by_code.get(class_code) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10) + if material_key and material_key not in valid_material_keys: + material_key = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + fraction = _str_or_none(row_norm.get("FRACCION"), 20) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5) + physical_review = _int_or_none(row_norm.get("REVFISICA")) + iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4) + + if existing: + existing.description_es = desc_es + existing.description_en = desc_en + existing.material_key = material_key + existing.unit_of_measure = unit_of_measure + existing.fraction = fraction + existing.us_fraction = us_fraction + existing.sub_key = sub_key + existing.physical_review = physical_review + existing.iva_exempt_fraction = iva_exempt_fraction + session.add(existing) + inserted_count += 1 + else: + new_class = Class( + tenant_id=tenant_id, + company_id=company_id, + class_code=class_code, + description_es=desc_es, + description_en=desc_en, + material_key=material_key, + unit_of_measure=unit_of_measure, + fraction=fraction, + us_fraction=us_fraction, + sub_key=sub_key, + physical_review=physical_review, + iva_exempt_fraction=iva_exempt_fraction, + ) + session.add(new_class) + existing_by_code[class_code] = new_class + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Classes import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Classes import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Classes import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/classes/imports/template_config.py b/backend/api/v1/modules/a76/classes/imports/template_config.py new file mode 100644 index 00000000..4a389bd8 --- /dev/null +++ b/backend/api/v1/modules/a76/classes/imports/template_config.py @@ -0,0 +1,45 @@ +""" +Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls). +""" + +from typing import Dict, List, Any, Optional + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "material_classes": [ + {"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]}, + {"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, + {"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]}, + {"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]}, + {"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("material_classes") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/classes/routes.py b/backend/api/v1/modules/a76/classes/routes.py index 329294b0..2166610f 100644 --- a/backend/api/v1/modules/a76/classes/routes.py +++ b/backend/api/v1/modules/a76/classes/routes.py @@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse from .service import ClassService +from .imports.routes import router as imports_router # Create a new router for custom endpoints router = APIRouter() +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"]) + # Add consolidated catalog endpoints FIRST (before generic CRUD routes) # This ensures they have priority over the generic /{id} route @router.get( diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index f0b91d0a..9fb07c5c 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -38,6 +38,7 @@ celery_app.conf.update( "api.v1.modules.a76.pedmientos.imports.tasks", "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", + "api.v1.modules.a76.classes.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 55964f53..bb5f6a9a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -449,6 +449,21 @@ export const api = { api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {}) }, + // CSV import for Clases de Materiales (classes/imports) + materialClassImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/classes/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/classes/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/classes/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 7b2f5aa0..eb8598a9 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -34,6 +34,8 @@ let useAmericanFractionImport = $state(false); // Cuando es true, usamos API de importación de Pedimentos (pedimentos/imports) let usePedimentosImport = $state(false); + // Cuando es true, usamos API de importación de Clases de Materiales (classes/imports) + let useMaterialClassesImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -57,6 +59,7 @@ useExchangeRateImport = config.id === 'exchange_rates'; useAmericanFractionImport = config.id === 'american_fractions'; usePedimentosImport = config.id === 'pedimentos'; + useMaterialClassesImport = config.id === 'material_classes'; const companyId = companyStore.activeCompany?.id || 1; @@ -150,6 +153,24 @@ return; } + if (useMaterialClassesImport) { + try { + const res = await api.materialClassImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -194,7 +215,9 @@ ? await api.americanFractionImports.status(currentJobId) : usePedimentosImport ? await api.pedimentosImports.status(currentJobId) - : await api.imports.status(currentJobId); + : useMaterialClassesImport + ? await api.materialClassImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -348,7 +371,9 @@ ? await api.americanFractionImports.commit(currentJobId) : usePedimentosImport ? await api.pedimentosImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useMaterialClassesImport + ? await api.materialClassImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); From 629754cc8fcdd5030de676050a948c898df31db7 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 2 Mar 2026 13:01:15 -0700 Subject: [PATCH 10/16] feature/csv-trasportes --- .../vehicles/imports/__init__.py | 1 + .../transportation/vehicles/imports/routes.py | 192 ++++++ .../vehicles/imports/schemas.py | 22 + .../transportation/vehicles/imports/tasks.py | 607 ++++++++++++++++++ .../vehicles/imports/template_config.py | 53 ++ .../a76/transportation/vehicles/routes.py | 23 +- backend/core/celery_app.py | 1 + frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 29 +- 9 files changed, 934 insertions(+), 9 deletions(-) create mode 100644 backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py create mode 100644 backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py new file mode 100644 index 00000000..87b6b8b2 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Vehicles / Transportes (upload → scan → status → commit). diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py new file mode 100644 index 00000000..20960753 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/routes.py @@ -0,0 +1,192 @@ +""" +Rutas de importación CSV para Vehículos (Transportes). +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + VEHL_IMPORT_FILE_PREFIX, + VEHL_IMPORT_META_PREFIX, + VEHL_IMPORT_STATUS_PREFIX, + VEHL_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Vehicles import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "vehicles", + } + + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=VEHL_IMPORT_REDIS_TTL, + ) + r.set( + f"{VEHL_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Vehicles import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Vehicles import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + # Fallback sin worker: ejecutar scan en un hilo y guardar resultado en Redis + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception(f"Vehicles import: background scan failed: {e}") + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + # Si el resultado del scan está en Redis (worker o hilo de fallback), usarlo + try: + r = _get_redis() + raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug(f"Vehicles import: could not read status from Redis: {e}") + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + # Escribir "processing" en Redis para que el polling muestre estado hasta que termine el commit + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug(f"Vehicles import: could not write processing status: {e}") + + # Ejecutar commit en hilo (mismo fallback que el scan: no depende del worker de Celery) + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception(f"Vehicles import: background commit failed: {e}") + + threading.Thread(target=run_commit_background, daemon=True).start() + + # Mismo job_id para que el front siga haciendo polling y reciba el resultado de Redis + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py new file mode 100644 index 00000000..7827dae6 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py new file mode 100644 index 00000000..7e9df06c --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/tasks.py @@ -0,0 +1,607 @@ +""" +Tareas Celery para importación CSV de Vehículos (Transportes). +Flujo: scan_file (validación) → insert_valid_rows (commit). +Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key. +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +VEHL_IMPORT_FILE_PREFIX = "veh_import_file:" +VEHL_IMPORT_META_PREFIX = "veh_import_meta:" +VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:" +VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:" +VEHL_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Vehicles import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"veh_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", + f"{VEHL_IMPORT_META_PREFIX}{job_id}", + f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +# Max lengths from Vehicle model (a76.vehicle) +_MAX = { + "vehicle_key": 14, + "ace_vehicle_key": 10, + "transporter_key": 23, + "transport_identifier": 30, + "transport_type": 2, + "entity_code": 1, + "transponder_number": 16, + "dot_number": 8, + "plate_number": 17, + "city": 30, + "state": 30, + "country": 3, + "seal": 49, + "insurance_company_name": 30, + "insurance_number": 20, + "series": 30, +} + + +def _parse_decimal(val: Any) -> Optional[Decimal]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + try: + s = str(val).strip().replace(",", "") + return Decimal(s) + except Exception: + return None + + +def _parse_insurance_date(val: Any) -> Optional[int]: + """Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD.""" + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if not s: + return None + # Already integer-like + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + pass + # Try DD/MM/YYYY or similar + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + pass + break + return None + + +def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Vehículo. Retorna error dict o None.""" + clave = (row.get("CLAVE") or "").strip() + if not clave: + return {"line": line_num, "col": "CLAVE", "msg": "Requerido"} + if len(clave) > _MAX["vehicle_key"]: + return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"} + + # Optional fields: max lengths only + for col, max_len in [ + ("CLAVE ACE", _MAX["ace_vehicle_key"]), + ("CLAVE TRANSPORTE", _MAX["transporter_key"]), + ("VIN", _MAX["series"]), + ("TIPO TRANSPORTE", _MAX["transport_type"]), + ("CODIGO DE ENTIDAD", _MAX["entity_code"]), + ("TRANSPONDEDOR", _MAX["transponder_number"]), + ("NUMERO DOT", _MAX["dot_number"]), + ("PLACAS", _MAX["plate_number"]), + ("CIUDAD", _MAX["city"]), + ("ESTADO", _MAX["state"]), + ("PAIS", _MAX["country"]), + ("PRECINTO", _MAX["seal"]), + ("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]), + ("NUM. ASEGURADORA", _MAX["insurance_number"]), + ]: + val = (row.get(col) or "").strip() + if val and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + + # MONTO ASEGURADO: must be numeric if present + monto = row.get("MONTO ASEGURADO") or row.get("MONTO") + if monto is not None and str(monto).strip(): + if _parse_decimal(monto) is None: + return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"} + + # FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None) + # Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either + # reject or set None. We reject invalid date to keep data quality. + fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA") + if fecha is not None and str(fecha).strip(): + if _parse_insurance_date(fecha) is None: + return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"} + + return None + + +def _do_scan( + job_id: str, + progress_callback: Optional[Any] = None, +) -> Dict[str, Any]: + """ + Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync. + progress_callback(current, total, error_count) opcional. + """ + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Vehicles import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_vehicle(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Vehicles import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}") + + result = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + return result + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + """ + Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis. + Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery. + """ + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + """ + Fase 1: Leer CSV, validar filas, escribir errores en JSONL. + Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors. + """ + logger.info(f"Vehicles import: starting scan for job {job_id}") + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state( + state="PROGRESS", + meta={"current": current, "total": total, "errors": errors}, + ) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}") + return result + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]: + """Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row.""" + vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"]) + if not vehicle_key: + return {} + data = { + "vehicle_key": vehicle_key, + "ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]), + "transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]), + "series": _str_or_none(row.get("VIN"), _MAX["series"]), + "transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]), + "entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]), + "transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]), + "dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]), + "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), + "city": _str_or_none(row.get("CIUDAD"), _MAX["city"]), + "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), + "country": _str_or_none(row.get("PAIS"), _MAX["country"]), + "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), + "insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]), + "insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]), + "insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")), + "insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")), + } + return data + + +def _do_commit(job_id: str) -> Dict[str, Any]: + """ + Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync. + """ + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Vehicles import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.transportation.vehicles.services import VehicleService + from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_vehicle(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-", + "invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-", + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + data = _row_to_vehicle_dto(row_norm) + if not data or not data.get("vehicle_key"): + skipped_invalid += 1 + continue + + vk = data["vehicle_key"] + if vk in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append( + {"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"} + ) + continue + seen_keys_in_file[vk] = i + + existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) + try: + if existing: + update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) + VehicleService.update(session, vk, tenant_id, update_data, company_id) + updated_count += 1 + else: + create_data = VehicleCreateDTO(**data) + VehicleService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append( + {"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)} + ) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Vehicles import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.error(f"Vehicles import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Vehicles import cleanup failed: {cleanup_err}") + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + """ + Ejecuta el commit en el proceso actual y guarda el resultado en Redis. + Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery. + """ + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + """ + Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService. + """ + logger.info(f"Vehicles import: starting commit for job {job_id}") + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=VEHL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}") + return result diff --git a/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py b/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py new file mode 100644 index 00000000..3474ae50 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/vehicles/imports/template_config.py @@ -0,0 +1,53 @@ +""" +Configuración de plantilla CSV para Vehículos / Transportes (EstructuraCatTransportes.xls). +Mapeo: CLAVE → vehicle_key, CLAVE ACE → ace_vehicle_key, etc. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "vehicles": [ + {"canonical": "CLAVE", "aliases": ["CLAVE VEHICULO", "VEHICLE KEY", "KEY"]}, + {"canonical": "CLAVE ACE", "aliases": ["ACE", "CLAVEACE"]}, + {"canonical": "CLAVE TRANSPORTE", "aliases": ["TRANSPORTE", "TRANSPORTISTA"]}, + {"canonical": "VIN", "aliases": ["SERIE", "SERIES", "NUMERO SERIE"]}, + {"canonical": "TIPO TRANSPORTE", "aliases": ["TIPO", "TRANSPORT TYPE"]}, + {"canonical": "CODIGO DE ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE"]}, + {"canonical": "TRANSPONDEDOR", "aliases": ["TRANSPONDER", "TRANSPONDER NUMBER"]}, + {"canonical": "NUMERO DOT", "aliases": ["DOT", "NUM DOT"]}, + {"canonical": "PLACAS", "aliases": ["PLACA", "PLATE", "PLATE NUMBER"]}, + {"canonical": "CIUDAD", "aliases": ["CITY"]}, + {"canonical": "ESTADO", "aliases": ["STATE"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]}, + {"canonical": "PRECINTO", "aliases": ["SEAL"]}, + {"canonical": "EMPRESA ASEGURADORA", "aliases": ["ASEGURADORA", "INSURANCE COMPANY"]}, + {"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]}, + {"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]}, + {"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("vehicles") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/transportation/vehicles/routes.py b/backend/api/v1/modules/a76/transportation/vehicles/routes.py index 1afc20ed..ce823259 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/routes.py +++ b/backend/api/v1/modules/a76/transportation/vehicles/routes.py @@ -1,11 +1,19 @@ +from fastapi import APIRouter + from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO from .services import VehicleService +from .imports.routes import router as imports_router -# Create router using TenantCRUDRoutes factory -# Note: vehicle_key is a string (not int) and is used as the primary key -router = TenantCRUDRoutes( +# Main router: vehicles CRUD + CSV imports +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/vehicles/imports", tags=["a76 / vehicles / csv_import"]) + +# CRUD routes +crud_router = TenantCRUDRoutes( service=VehicleService, create_schema=VehicleCreateDTO, update_schema=VehicleUpdateDTO, @@ -13,10 +21,11 @@ router = TenantCRUDRoutes( prefix="/vehicles", tags=[], resource_name="Vehicle", - id_name="vehicle_key", # Using vehicle_key instead of numeric ID - id_type=str, # Specify that the ID is a string - enable_list=True, # Enable GET /vehicles with pagination - enable_filters=True, # Enable filtering by plate_number and transport_type + id_name="vehicle_key", + id_type=str, + enable_list=True, + enable_filters=True, default_page_size=50, max_page_size=100, ).router +router.include_router(crud_router) diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 9fb07c5c..279b0a01 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -39,6 +39,7 @@ celery_app.conf.update( "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", "api.v1.modules.a76.classes.imports.tasks", + "api.v1.modules.a76.transportation.vehicles.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index bb5f6a9a..e7669c73 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -464,6 +464,21 @@ export const api = { api.post(`/v1/a76/classes/imports/${jobId}/commit`, {}) }, + // CSV import for Vehículos / Transportes (transportation/vehicles/imports) + vehicleImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/transportation/vehicles/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/transportation/vehicles/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/transportation/vehicles/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index eb8598a9..a936bc8a 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -36,6 +36,8 @@ let usePedimentosImport = $state(false); // Cuando es true, usamos API de importación de Clases de Materiales (classes/imports) let useMaterialClassesImport = $state(false); + // Cuando es true, usamos API de importación de Vehículos / Transportes (vehicles/imports) + let useVehicleImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -60,6 +62,7 @@ useAmericanFractionImport = config.id === 'american_fractions'; usePedimentosImport = config.id === 'pedimentos'; useMaterialClassesImport = config.id === 'material_classes'; + useVehicleImport = config.id === 'transports'; const companyId = companyStore.activeCompany?.id || 1; @@ -171,6 +174,24 @@ return; } + if (useVehicleImport) { + try { + const res = await api.vehicleImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -217,7 +238,9 @@ ? await api.pedimentosImports.status(currentJobId) : useMaterialClassesImport ? await api.materialClassImports.status(currentJobId) - : await api.imports.status(currentJobId); + : useVehicleImport + ? await api.vehicleImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -373,7 +396,9 @@ ? await api.pedimentosImports.commit(currentJobId) : useMaterialClassesImport ? await api.materialClassImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useVehicleImport + ? await api.vehicleImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); From b94fd34bb2fa54e616f021d978720391e485179d Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 2 Mar 2026 15:28:50 -0700 Subject: [PATCH 11/16] feature/csv-drivers --- .../modules/a76/transportation/drivers/dto.py | 44 +- .../drivers/imports/__init__.py | 1 + .../transportation/drivers/imports/routes.py | 188 +++++ .../transportation/drivers/imports/schemas.py | 23 + .../transportation/drivers/imports/tasks.py | 640 ++++++++++++++++++ .../drivers/imports/template_config.py | 56 ++ .../a76/transportation/drivers/routes.py | 4 + frontend/package.json | 5 +- frontend/src/lib/api.ts | 14 + .../routes/dashboard/csv-upload/+page.svelte | 25 + 10 files changed, 976 insertions(+), 24 deletions(-) create mode 100644 backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/transportation/drivers/imports/routes.py create mode 100644 backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py diff --git a/backend/api/v1/modules/a76/transportation/drivers/dto.py b/backend/api/v1/modules/a76/transportation/drivers/dto.py index f07e4b4c..bfb3d8fc 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/dto.py +++ b/backend/api/v1/modules/a76/transportation/drivers/dto.py @@ -6,28 +6,28 @@ from pydantic import BaseModel class DriverBaseDTO(BaseModel): transporter_key: str line: int - driver_name: Optional[str] - license_number: Optional[str] - express_line_id: Optional[str] - ace_id: Optional[str] - birth_date: Optional[int] - gender: Optional[str] - birth_country: Optional[str] - hazardous_material_auth: Optional[str] - hazardous_material_state: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - id_key1: Optional[str] - id_number1: Optional[str] - id_state1: Optional[str] - id_country1: Optional[str] - id_key2: Optional[str] - id_number2: Optional[str] - id_state2: Optional[str] - id_country2: Optional[str] - badge_number: Optional[str] - class_type: Optional[str] - unique_badge_number: Optional[str] + driver_name: Optional[str] = None + license_number: Optional[str] = None + express_line_id: Optional[str] = None + ace_id: Optional[str] = None + birth_date: Optional[int] = None + gender: Optional[str] = None + birth_country: Optional[str] = None + hazardous_material_auth: Optional[str] = None + hazardous_material_state: Optional[str] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + id_key1: Optional[str] = None + id_number1: Optional[str] = None + id_state1: Optional[str] = None + id_country1: Optional[str] = None + id_key2: Optional[str] = None + id_number2: Optional[str] = None + id_state2: Optional[str] = None + id_country2: Optional[str] = None + badge_number: Optional[str] = None + class_type: Optional[str] = None + unique_badge_number: Optional[str] = None company_id: int tenant_id: int diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py b/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py new file mode 100644 index 00000000..98bef45c --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Drivers / Conductores (upload -> scan -> status -> commit). diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py b/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py new file mode 100644 index 00000000..ea5d2eb4 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/routes.py @@ -0,0 +1,188 @@ +""" +Rutas de importacion CSV para Conductores. +Flujo: upload -> scan -> status (polling) -> commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + DRV_IMPORT_FILE_PREFIX, + DRV_IMPORT_META_PREFIX, + DRV_IMPORT_STATUS_PREFIX, + DRV_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Drivers import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "drivers", + } + + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=DRV_IMPORT_REDIS_TTL, + ) + r.set( + f"{DRV_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Drivers import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Drivers import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception(f"Drivers import: background scan failed: {e}") + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + try: + r = _get_redis() + raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug(f"Drivers import: could not read status from Redis: {e}") + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug(f"Drivers import: could not write processing status: {e}") + + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception(f"Drivers import: background commit failed: {e}") + + threading.Thread(target=run_commit_background, daemon=True).start() + + return { + "status": "committing", + "message": "Insercion iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py b/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py new file mode 100644 index 00000000..c5d2c2cc --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/schemas.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + updated: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py b/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py new file mode 100644 index 00000000..9234776e --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/tasks.py @@ -0,0 +1,640 @@ +""" +Tareas Celery para importacion CSV de Conductores. +Flujo: scan_file (validacion) -> insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +DRV_IMPORT_FILE_PREFIX = "drv_import_file:" +DRV_IMPORT_META_PREFIX = "drv_import_meta:" +DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:" +DRV_IMPORT_STATUS_PREFIX = "drv_import_status:" +DRV_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Drivers import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"drv_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Drivers import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{DRV_IMPORT_FILE_PREFIX}{job_id}", + f"{DRV_IMPORT_META_PREFIX}{job_id}", + f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Drivers import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +_MAX = { + "transporter_key": 5, + "driver_name": 80, + "license_number": 29, + "express_line_id": 17, + "ace_id": 20, + "gender": 1, + "birth_country": 3, + "hazardous_material_auth": 2, + "hazardous_material_state": 30, + "first_name": 20, + "last_name": 20, + "id_key1": 40, + "id_number1": 20, + "id_state1": 30, + "id_country1": 3, + "id_key2": 40, + "id_number2": 20, + "id_state2": 30, + "id_country2": 3, + "badge_number": 20, + "class_type": 1, + "unique_badge_number": 100, +} + + +def _parse_int(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d+$", s): + return int(s) + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + return None + + +def _parse_birth_date(val: Any) -> Optional[int]: + if val is None or (isinstance(val, str) and not val.strip()): + return None + s = str(val).strip() + if re.match(r"^\d{8}$", s): + try: + return int(s) + except ValueError: + return None + if re.match(r"^\d+\.0+$", s): + try: + return int(float(s)) + except ValueError: + return None + for sep in ["/", "-", "."]: + if sep in s: + parts = s.split(sep) + if len(parts) == 3: + try: + a, b, c = [p.strip() for p in parts] + if len(c) == 4 and len(a) <= 2 and len(b) <= 2: + return int(c) * 10000 + int(b) * 100 + int(a) + if len(a) == 4 and len(b) <= 2 and len(c) <= 2: + return int(a) * 10000 + int(b) * 100 + int(c) + except (ValueError, TypeError): + return None + break + return None + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _dedupe_headers(headers: List[str]) -> List[str]: + counts: Dict[str, int] = {} + unique: List[str] = [] + for header in headers: + name = str(header or "").strip() or "COL" + count = counts.get(name, 0) + 1 + counts[name] = count + if count == 1: + unique.append(name) + else: + unique.append(f"{name} {count}") + return unique + + +def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + transporter_key = (row.get("TRANSPORTISTA") or "").strip() + if not transporter_key: + return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"} + if len(transporter_key) > _MAX["transporter_key"]: + return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"} + + line_val = _parse_int(row.get("LINEA")) + if line_val is None: + return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"} + if line_val <= 0: + return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"} + + driver_name = (row.get("CLAVE CONDUCTOR") or "").strip() + if driver_name and len(driver_name) > _MAX["driver_name"]: + return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"} + + for col, max_len in [ + ("LICENCIA", _MAX["license_number"]), + ("PERMISO LINEA EXPRESS", _MAX["express_line_id"]), + ("IDENTIFICACION ACE", _MAX["ace_id"]), + ("SEXO", _MAX["gender"]), + ("PAIS NACIMIENTO", _MAX["birth_country"]), + ("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]), + ("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]), + ("NOMBRE(S)", _MAX["first_name"]), + ("APELLIDO PATERNO", _MAX["last_name"]), + ("FORMA IDENTIFICACION 1", _MAX["id_key1"]), + ("NUM. IDENTIFICACION 1", _MAX["id_number1"]), + ("ESTADO", _MAX["id_state1"]), + ("PAIS", _MAX["id_country1"]), + ("FORMA IDENTIFICACION 2", _MAX["id_key2"]), + ("NUM. IDENTIFICACION 2", _MAX["id_number2"]), + ("ESTADO 2", _MAX["id_state2"]), + ("PAIS 2", _MAX["id_country2"]), + ]: + val = (row.get(col) or "").strip() + if val and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"} + + fecha = row.get("FECHA NACIMIENTO") + if fecha is not None and str(fecha).strip(): + if _parse_birth_date(fecha) is None: + return { + "line": line_num, + "col": "FECHA NACIMIENTO", + "msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)", + } + + return None + + +def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: + transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"]) + line = _parse_int(row.get("LINEA")) + if not transporter_key or line is None: + return {} + data = { + "transporter_key": transporter_key, + "line": line, + "driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]), + "license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]), + "express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]), + "ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]), + "birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")), + "gender": _str_or_none(row.get("SEXO"), _MAX["gender"]), + "birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]), + "hazardous_material_auth": _str_or_none( + row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"] + ), + "hazardous_material_state": _str_or_none( + row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"] + ), + "first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]), + "last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]), + "id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]), + "id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]), + "id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]), + "id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]), + "id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]), + "id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]), + "id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]), + "id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]), + "company_id": company_id, + "tenant_id": tenant_id, + } + return data + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Drivers import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f_in, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_driver(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Drivers import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store error lines: {e}") + + result = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + return result + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Drivers import: starting scan for job {job_id}") + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state( + state="PROGRESS", + meta={"current": current, "total": total, "errors": errors}, + ) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store scan status in Redis: {e}") + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Drivers import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.transportation.drivers.services import DriverService + from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_driver(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", + "invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-", + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + data = _row_to_driver_dto(row_norm, tenant_id, company_id) + if not data or not data.get("transporter_key") or data.get("line") is None: + skipped_invalid += 1 + continue + + key = f"{data['transporter_key']}:{data['line']}" + if key in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append( + { + "line": i, + "driver_key": key, + "invoice": key, + "reason": "Clave duplicada en el archivo (se usa la primera)", + } + ) + continue + seen_keys_in_file[key] = i + + existing = DriverService.get_driver_by_key_and_line( + session, data["transporter_key"], data["line"], str(company_id), tenant_id + ) + try: + if existing: + update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")} + for field, value in update_fields.items(): + setattr(existing, field, value) + session.add(existing) + updated_count += 1 + else: + create_data = DriverCreateDTO(**data) + DriverService.create_driver(session, create_data) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append( + {"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)} + ) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Drivers import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.error(f"Drivers import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Drivers import cleanup failed: {cleanup_err}") + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros validos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Drivers import: starting commit for job {job_id}") + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{DRV_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=DRV_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Drivers import: failed to store commit status in Redis: {e}") + return result diff --git a/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py b/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py new file mode 100644 index 00000000..cd9a3194 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/drivers/imports/template_config.py @@ -0,0 +1,56 @@ +""" +Configuracion de plantilla CSV para Conductores (EstructuraCatConductor.xls). +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "drivers": [ + {"canonical": "TRANSPORTISTA", "aliases": ["TRANSPORTISTA CLAVE", "CLAVE TRANSPORTISTA", "TRANSPORTER"]}, + {"canonical": "LINEA", "aliases": ["LINE", "LINEA CONDUCTOR"]}, + {"canonical": "CLAVE CONDUCTOR", "aliases": ["CONDUCTOR", "DRIVER", "NOMBRE CONDUCTOR"]}, + {"canonical": "LICENCIA", "aliases": ["LICENSE", "LICENCE"]}, + {"canonical": "PERMISO LINEA EXPRESS", "aliases": ["LINEA EXPRESS", "LINEA EXPRESS ID", "EXPRESS LINE"]}, + {"canonical": "IDENTIFICACION ACE", "aliases": ["ACE", "ACE ID", "IDENTIFICACION ACE ID"]}, + {"canonical": "FECHA NACIMIENTO", "aliases": ["FECHA NAC", "BIRTH DATE", "BIRTHDATE"]}, + {"canonical": "SEXO", "aliases": ["GENERO", "GENDER"]}, + {"canonical": "PAIS NACIMIENTO", "aliases": ["PAIS Nacimiento", "BIRTH COUNTRY"]}, + {"canonical": "TRANSPORTA MAT. PELIGROSO?", "aliases": ["MATERIAL PELIGROSO", "HAZMAT", "HAZARDOUS MATERIAL"]}, + {"canonical": "PERMISO MAT. PELIGROSO", "aliases": ["PERMISO MAT PELIGROSO", "HAZMAT PERMIT"]}, + {"canonical": "NOMBRE(S)", "aliases": ["NOMBRE", "NOMBRES", "FIRST NAME"]}, + {"canonical": "APELLIDO PATERNO", "aliases": ["APELLIDO", "LAST NAME", "APELLIDO P"]}, + {"canonical": "FORMA IDENTIFICACION 1", "aliases": ["FORMA IDENTIFICACION", "ID TIPO 1", "ID KEY 1"]}, + {"canonical": "NUM. IDENTIFICACION 1", "aliases": ["NUM IDENTIFICACION 1", "ID NUMERO 1", "ID NUMBER 1"]}, + {"canonical": "ESTADO", "aliases": ["ESTADO 1", "STATE 1"]}, + {"canonical": "PAIS", "aliases": ["PAIS 1", "COUNTRY 1"]}, + {"canonical": "FORMA IDENTIFICACION 2", "aliases": ["ID TIPO 2", "ID KEY 2"]}, + {"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]}, + {"canonical": "ESTADO 2", "aliases": ["STATE 2"]}, + {"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("drivers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/transportation/drivers/routes.py b/backend/api/v1/modules/a76/transportation/drivers/routes.py index 981ab353..a07b5c55 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/routes.py +++ b/backend/api/v1/modules/a76/transportation/drivers/routes.py @@ -8,9 +8,13 @@ from sqlalchemy.orm import Session from .dto import DriverCreateDTO, DriverResponseDTO from .models import Driver from .services import DriverService +from .imports.routes import router as imports_router router = APIRouter(prefix="/drivers") +# CSV import (upload -> scan -> status -> commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / drivers / csv_import"]) + @router.get("/", response_model=Dict[str, Any]) async def list_drivers( diff --git a/frontend/package.json b/frontend/package.json index b21c873b..e7cd1887 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -64,5 +64,6 @@ "lucide-svelte": "^0.553.0", "marked": "^12.0.0", "svelte-sonner": "^1.0.7" - } -} \ No newline at end of file + }, + "packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017" +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index e7669c73..fde71cfe 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -479,6 +479,20 @@ export const api = { api.post(`/v1/a76/transportation/vehicles/imports/${jobId}/commit`, {}) }, + // CSV import for Conductores (drivers/imports) + driverImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/drivers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/drivers/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/drivers/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index a936bc8a..514fe09d 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -38,6 +38,8 @@ let useMaterialClassesImport = $state(false); // Cuando es true, usamos API de importación de Vehículos / Transportes (vehicles/imports) let useVehicleImport = $state(false); + // Cuando es true, usamos API de importación de Conductores (drivers/imports) + let useDriverImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -63,6 +65,7 @@ usePedimentosImport = config.id === 'pedimentos'; useMaterialClassesImport = config.id === 'material_classes'; useVehicleImport = config.id === 'transports'; + useDriverImport = config.id === 'drivers'; const companyId = companyStore.activeCompany?.id || 1; @@ -192,6 +195,24 @@ return; } + if (useDriverImport) { + try { + const res = await api.driverImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -240,6 +261,8 @@ ? await api.materialClassImports.status(currentJobId) : useVehicleImport ? await api.vehicleImports.status(currentJobId) + : useDriverImport + ? await api.driverImports.status(currentJobId) : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { @@ -398,6 +421,8 @@ ? await api.materialClassImports.commit(currentJobId) : useVehicleImport ? await api.vehicleImports.commit(currentJobId) + : useDriverImport + ? await api.driverImports.commit(currentJobId) : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; From c7fb49a57338715a48cb510cf89c1c19c1d031c0 Mon Sep 17 00:00:00 2001 From: hreyes Date: Tue, 3 Mar 2026 07:51:11 -0700 Subject: [PATCH 12/16] feature/csv-trailers --- .../trailers/imports/__init__.py | 1 + .../transportation/trailers/imports/routes.py | 188 ++++ .../trailers/imports/schemas.py | 23 + .../transportation/trailers/imports/tasks.py | 535 ++++++++++ .../trailers/imports/template_config.py | 45 + .../a76/transportation/trailers/routes.py | 23 +- frontend/src/lib/api.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 923 +++++++++--------- 8 files changed, 1297 insertions(+), 456 deletions(-) create mode 100644 backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/transportation/trailers/imports/routes.py create mode 100644 backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py b/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py new file mode 100644 index 00000000..ef25c98a --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/__init__.py @@ -0,0 +1 @@ +# Trailers CSV import: upload -> scan -> status -> commit diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py b/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py new file mode 100644 index 00000000..7e51aca1 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/routes.py @@ -0,0 +1,188 @@ +""" +Rutas de importación CSV para Trailers y Cajas. +Flujo: upload -> scan -> status (polling) -> commit. +""" +import base64 +import json +import logging +import os +import threading +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + run_scan_sync, + run_commit_sync, + TRL_IMPORT_FILE_PREFIX, + TRL_IMPORT_META_PREFIX, + TRL_IMPORT_STATUS_PREFIX, + TRL_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Trailers import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "trailers", + } + + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=TRL_IMPORT_REDIS_TTL, + ) + r.set( + f"{TRL_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Trailers import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Trailers import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + def run_scan_background(): + try: + run_scan_sync(job_id) + except Exception as e: + logger.exception(f"Trailers import: background scan failed: {e}") + + threading.Thread(target=run_scan_background, daemon=True).start() + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + try: + r = _get_redis() + raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}") + if raw: + data = json.loads(raw.decode("utf-8")) + return data + except Exception as e: + logger.debug(f"Trailers import: could not read status from Redis: {e}") + + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.debug(f"Trailers import: could not write processing status: {e}") + + def run_commit_background(): + try: + run_commit_sync(job_id) + except Exception as e: + logger.exception(f"Trailers import: background commit failed: {e}") + + threading.Thread(target=run_commit_background, daemon=True).start() + + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": job_id, + } diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py b/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py new file mode 100644 index 00000000..c5d2c2cc --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/schemas.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + updated: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py b/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py new file mode 100644 index 00000000..c43c93cf --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/tasks.py @@ -0,0 +1,535 @@ +""" +Tareas Celery para importación CSV de Trailers y Cajas. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Upsert por trailer_number usando TrailerService. +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +TRL_IMPORT_FILE_PREFIX = "trl_import_file:" +TRL_IMPORT_META_PREFIX = "trl_import_meta:" +TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:" +TRL_IMPORT_STATUS_PREFIX = "trl_import_status:" +TRL_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Trailers import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"trl_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Trailers import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{TRL_IMPORT_FILE_PREFIX}{job_id}", + f"{TRL_IMPORT_META_PREFIX}{job_id}", + f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Trailers import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +# Max lengths from Trailer model (a76.trailer) +_MAX = { + "trailer_number": 20, + "ace_trailer_number": 10, + "trailer_type_key": 2, + "seal": 15, + "entity_code": 1, + "plate_number": 17, + "state": 30, + "country": 3, + "container_key": 3, +} + + +def _dedupe_headers(headers: List[str]) -> List[str]: + counts: Dict[str, int] = {} + unique: List[str] = [] + for header in headers: + name = str(header or "").strip() or "COL" + count = counts.get(name, 0) + 1 + counts[name] = count + if count == 1: + unique.append(name) + else: + unique.append(f"{name} {count}") + return unique + + +def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + """Valida una fila para Trailer. Retorna error dict o None.""" + trailer_number = (row.get("NUMERO TRAILER") or "").strip() + if not trailer_number: + return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"} + if len(trailer_number) > _MAX["trailer_number"]: + return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"} + + for col, max_len in [ + ("CLAVE ACE", _MAX["ace_trailer_number"]), + ("TIPO TRAILER", _MAX["trailer_type_key"]), + ("PRECINTO", _MAX["seal"]), + ("CODIGO ENTIDAD", _MAX["entity_code"]), + ("PLACAS", _MAX["plate_number"]), + ("ESTADO", _MAX["state"]), + ("PAIS", _MAX["country"]), + ("CLAVE CONTENEDOR", _MAX["container_key"]), + ]: + val = (row.get(col) or "").strip() + if val and len(val) > max_len: + return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"} + + return None + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]: + """Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row.""" + trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"]) + if not trailer_number: + return {} + return { + "trailer_number": trailer_number, + "ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]), + "trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]), + "seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]), + "entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]), + "plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]), + "state": _str_or_none(row.get("ESTADO"), _MAX["state"]), + "country": _str_or_none(row.get("PAIS"), _MAX["country"]), + "container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]), + } + + +def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Trailers import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f_in, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if progress_callback and i % 500 == 0: + progress_callback(i, total_rows, error_count) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_trailer(row_norm, i) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Trailers import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store error lines: {e}") + + result = { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + return result + + +def run_scan_sync(job_id: str) -> Dict[str, Any]: + result = _do_scan(job_id, progress_callback=None) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Trailers import: starting scan for job {job_id}") + + def on_progress(current: int, total: int, errors: int) -> None: + self.update_state( + state="PROGRESS", + meta={"current": current, "total": total, "errors": errors}, + ) + + result = _do_scan(job_id, progress_callback=on_progress) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store scan status in Redis: {e}") + return result + + +def _do_commit(job_id: str) -> Dict[str, Any]: + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Trailers import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.transportation.trailers.services import TrailerService + from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + seen_keys_in_file: Dict[str, int] = {} + + try: + with CoreSessionLocal() as session: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.reader(f, dialect=dialect) + try: + headers = next(reader) + except StopIteration: + headers = [] + headers = _dedupe_headers(headers) + dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect) + + for i, row in enumerate(dict_reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_trailer(row_norm, i) + if err: + skipped_invalid += 1 + skipped_details.append( + { + "line": i, + "trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", + "invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-", + "reason": f"{err.get('col', '')}: {err.get('msg', '')}", + } + ) + continue + + data = _row_to_trailer_dto(row_norm, tenant_id, company_id) + if not data or not data.get("trailer_number"): + skipped_invalid += 1 + continue + + tn = data["trailer_number"] + if tn in seen_keys_in_file: + skipped_duplicate += 1 + skipped_details.append( + { + "line": i, + "trailer_number": tn, + "invoice": tn, + "reason": "Clave duplicada en el archivo (se usa la primera)", + } + ) + continue + seen_keys_in_file[tn] = i + + existing = TrailerService.get_by_id(session, tn, tenant_id, company_id) + try: + if existing: + update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"}) + TrailerService.update(session, tn, tenant_id, update_data, company_id) + updated_count += 1 + else: + create_data = TrailerCreateDTO(**data) + TrailerService.create(session, create_data, tenant_id, company_id) + inserted_count += 1 + except Exception as db_err: + session.rollback() + skipped_invalid += 1 + skipped_details.append( + {"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)} + ) + continue + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Trailers import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + except Exception as e: + logger.error(f"Trailers import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Trailers import cleanup failed: {cleanup_err}") + + total_ok = inserted_count + updated_count + if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: + return { + "status": "warning", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", + } + if total_ok == 0: + return { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "updated": 0, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + return { + "status": "finished", + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_duplicate": skipped_duplicate, + "skipped_missing_fk": 0, + "skipped_details": skipped_details, + } + + +def run_commit_sync(job_id: str) -> Dict[str, Any]: + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") + return result + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Trailers import: starting commit for job {job_id}") + result = _do_commit(job_id) + try: + r = _get_redis() + r.set( + f"{TRL_IMPORT_STATUS_PREFIX}{job_id}", + json.dumps(result).encode("utf-8"), + ex=TRL_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Trailers import: failed to store commit status in Redis: {e}") + return result diff --git a/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py b/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py new file mode 100644 index 00000000..be0eb4a7 --- /dev/null +++ b/backend/api/v1/modules/a76/transportation/trailers/imports/template_config.py @@ -0,0 +1,45 @@ +""" +Configuración de plantilla CSV para Trailers / Cajas (EstructuraCatTrailers.xls). +Mapeo: NUMERO TRAILER → trailer_number, CLAVE ACE → ace_trailer_number, etc. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "trailers": [ + {"canonical": "NUMERO TRAILER", "aliases": ["CLAVE TRAILER", "TRAILER NUMBER", "TRAILER", "NUMERO"]}, + {"canonical": "CLAVE ACE", "aliases": ["ACE", "NUMERO ACE", "ACE TRAILER"]}, + {"canonical": "TIPO TRAILER", "aliases": ["TIPO", "TRAILER TYPE", "TIPO CAJA"]}, + {"canonical": "PRECINTO", "aliases": ["SEAL"]}, + {"canonical": "CODIGO ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE", "CODIGO DE ENTIDAD"]}, + {"canonical": "PLACAS", "aliases": ["PLACA", "PLATE NUMBER", "PLATE"]}, + {"canonical": "ESTADO", "aliases": ["STATE"]}, + {"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]}, + {"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("trailers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/transportation/trailers/routes.py b/backend/api/v1/modules/a76/transportation/trailers/routes.py index ad108e46..4dcea2e3 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/routes.py +++ b/backend/api/v1/modules/a76/transportation/trailers/routes.py @@ -1,11 +1,19 @@ +from fastapi import APIRouter + from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO from .services import TrailerService +from .imports.routes import router as imports_router -# Create router using TenantCRUDRoutes factory -# Note: trailer_number is a string (not int) and is used as the primary key -router = TenantCRUDRoutes( +# Main router: trailers CRUD + CSV imports +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/trailers/imports", tags=["a76 / trailers / csv_import"]) + +# CRUD routes +crud_router = TenantCRUDRoutes( service=TrailerService, create_schema=TrailerCreateDTO, update_schema=TrailerUpdateDTO, @@ -13,10 +21,11 @@ router = TenantCRUDRoutes( prefix="/trailers", tags=[], resource_name="Trailer", - id_name="trailer_number", # Using trailer_number instead of numeric ID - id_type=str, # Specify that the ID is a string - enable_list=True, # Enable GET /trailers with pagination - enable_filters=True, # Enable filtering by plate_number and trailer_type_key + id_name="trailer_number", + id_type=str, + enable_list=True, + enable_filters=True, default_page_size=50, max_page_size=100, ).router +router.include_router(crud_router) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index fde71cfe..d535507f 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -493,6 +493,21 @@ export const api = { commit: (jobId: string) => api.post(`/v1/a76/drivers/imports/${jobId}/commit`, {}) }, + // CSV import for Trailers y Cajas (transportation/trailers/imports) + trailerImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/transportation/trailers/imports/${jobId}/status`), + commit: (jobId: string) => + api.post(`/v1/a76/transportation/trailers/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 514fe09d..c5ec47e8 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -1,449 +1,474 @@ - - -
- -
-
-

Importación Masiva de Datos (CSV)

-
- - - - Catálogos - Transportes - Importación - Exportación - - -
- -
-

Catálogos Generales

-
- - - - -
-

Logística y Transporte

-
- - - - -
-

Operaciones de Importación

-
- - - - -
-

Operaciones de Exportación

-
- - -
- - -
-
- - - {#if allSettings[activeTab]} -
- -
- {/if} -
- -{#if scanResults || commitResults} - { - if (!currentJobId) return; - try { - isUploading = true; - const res = useCustomsBrokerImport - ? await api.customsBrokerImports.commit(currentJobId) - : useClientProviderImport - ? await api.clientProviderImports.commit(currentJobId) - : useExchangeRateImport - ? await api.exchangeRateImports.commit(currentJobId) - : useAmericanFractionImport - ? await api.americanFractionImports.commit(currentJobId) - : usePedimentosImport - ? await api.pedimentosImports.commit(currentJobId) - : useMaterialClassesImport - ? await api.materialClassImports.commit(currentJobId) - : useVehicleImport - ? await api.vehicleImports.commit(currentJobId) - : useDriverImport - ? await api.driverImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); - if (res.data?.commit_job_id) { - currentJobId = res.data.commit_job_id; - pollStatus(); - } - } catch (err) { - toast.error('Error al iniciar la importación'); - isUploading = false; - } - }} - onCancel={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} - onClose={() => { - currentJobId = null; - scanResults = null; - commitResults = null; - showResultModal = false; - }} - /> -{/if} + + +
+ +
+
+

Importación Masiva de Datos (CSV)

+
+ + + + Catálogos + Transportes + Importación + Exportación + + +
+ +
+

Catálogos Generales

+
+ + + + +
+

Logística y Transporte

+
+ + + + +
+

Operaciones de Importación

+
+ + + + +
+

Operaciones de Exportación

+
+ + +
+ + +
+
+ + + {#if allSettings[activeTab]} +
+ +
+ {/if} +
+ +{#if scanResults || commitResults} + { + if (!currentJobId) return; + try { + isUploading = true; + const res = useCustomsBrokerImport + ? await api.customsBrokerImports.commit(currentJobId) + : useClientProviderImport + ? await api.clientProviderImports.commit(currentJobId) + : useExchangeRateImport + ? await api.exchangeRateImports.commit(currentJobId) + : useAmericanFractionImport + ? await api.americanFractionImports.commit(currentJobId) + : usePedimentosImport + ? await api.pedimentosImports.commit(currentJobId) + : useMaterialClassesImport + ? await api.materialClassImports.commit(currentJobId) + : useVehicleImport + ? await api.vehicleImports.commit(currentJobId) + : useDriverImport + ? await api.driverImports.commit(currentJobId) + : useTrailerImport + ? await api.trailerImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); + if (res.data?.commit_job_id) { + currentJobId = res.data.commit_job_id; + pollStatus(); + } + } catch (err) { + toast.error('Error al iniciar la importación'); + isUploading = false; + } + }} + onCancel={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + onClose={() => { + currentJobId = null; + scanResults = null; + commitResults = null; + showResultModal = false; + }} + /> +{/if} From 48d082c69def75c371ec2f01916108f740d1fe3d Mon Sep 17 00:00:00 2001 From: hreyes Date: Tue, 3 Mar 2026 07:56:20 -0700 Subject: [PATCH 13/16] fix/xlcx-faltantes --- frontend/static/csv/EstructuraBOMS.xlsx | Bin 0 -> 10171 bytes .../static/csv/EstructuraCatNumerosParte.xlsx | Bin 0 -> 10712 bytes .../static/csv/EstructuraCatTransportistas.xlsx | Bin 0 -> 10253 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 frontend/static/csv/EstructuraBOMS.xlsx create mode 100644 frontend/static/csv/EstructuraCatNumerosParte.xlsx create mode 100644 frontend/static/csv/EstructuraCatTransportistas.xlsx diff --git a/frontend/static/csv/EstructuraBOMS.xlsx b/frontend/static/csv/EstructuraBOMS.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..4a998d0d564f9536ea168724142c267af5dca851 GIT binary patch literal 10171 zcmeHt1zTLn)^+3V?oMzgKyVEjB)GdnW5M0s0wmBt2oQq11b2tV-Jy{L5AGi`ckVkg zx%2&kdrv=os?O=N_S0RpcJ1AJm5KrkEG_^ZfCvBpr~sx%S=NS706-!v0DuEPgw~S; zI=Wdnx*4f^Ia#*)e#>a8*64Ek zL7~l+;T!jiu^IF#M|ff@`U-drzPXK$T#1Lcx+wa*Shp|Hj=9}e-`BgHAwK3@<+kZ1 zp0yFHUba{eRq*6IK*cVm7Qc+kam=qr6faM4_@s++<&JIxhqPyn#JnjxA<_H|A+-a` z`j`G}DP;jZreOcC1B2iAJCqO2NPQg(CJ-R7($`P8mssDM&N?J82Ro=}{UQ$62fhpX z9E@-4g&duxeGlMzga-hgo?rkfe{;(^4Gx;~XRawedmYNNTN=4o*t@c`{dWGJJN_5@ z;9q`vX|j@XHwS9?vFv@sz}3u3BBq3bhq!DrmAZeR+%iU8TplgSO8W~^Om*TAnAZWV z{`W&mD?)KU2B~6v;|;i)S}m0O4%ky&+z@NLwHC0cG|hH-MeSu|Zr^bx1!~86@MV3I zIiN<(gMH04_PU=g=ORG2mgjUh%d3YEsk>z0WH2~}qHy&&Lj9+ca9ovdRzEuk_OlCN zJdcd0J-Y|c#m*E6wEHb?<*OS2K^(Y#pK70sZ?b}s5(DLnX?b+>-<8Z(e2{$)^X3Hk z;{v^$#zp&aM(8;WyFpCHj?3t2;}!YZRE>j94)6Qu7^Xf*PiLe?(ST=N^y%ouG_i3$ zPZ^hPWOVO!5TozQ``c|d8pwl0qz^10HQKrS9oa&?ysM_#J8E0aAS*_5Io^*h7A3k| z8AX|#2Ke>+{6Pk4h8tuOv}++$X$slZQu#`QOsz0Ur8&fVC7E-@lFUAs}u%z-X#j@RrB)k3RNa|BIv*&l&%xS@_&+{p)Av6#-cqAv3?8!#YB8nFsxk4 za>FXyr>NygW#V8eoZKavV~KI5P^>bk5E&>S=W78}kX#y2EOm!uveOOCC%+79gQxO1 zsT_Cqi8s>>=8y@qtj^6)lKLVZaraH6q!HgI8AXNvw)gR+J{P}s4tpe?OooixXSO1+ zDoqTba_&uB@X4;a!sY7`oT?&Zs2BE+5*g>$r1ug0Hn!CDR1mVVmAj>rpsd(eP=>Zq z>wamz5!CtANH;giM*nK7i5z)EsW{98-*T@NkyVZ2{2ZeuOLXd ziQ*~9i+jFCmLWuD7`EP!rko|PWM<*U!r5a2__2EA-g3`ZnO<#b+)-eu&`O*T$KlvB zHA*QKhW(t=drvU|wPU#!ESUYBG*kk5eDHCJuTM*WgT9CNDd+^PhEO#8hyki_967|; z)XQ*FXNyv?w2klb;gSDT4!b_~fFNaY7HN8sNTY0dEHzE3m_)8$l)XGh3v_8)t;ms_ zw4eayn$)bi6aFHN%*FR!XFhuIXz}KR;+eevN$(($ln-ampENyFoEU%z^-S-7g@M14 z{NEt~>N!Juc9Q?@qf|{vp@##r73DF4!vo}whqd6wPJN)UkAXN?%eX{O#qEE-M#9u& zpgpU=4g(DLIv(wJzv4yTgu%J!WG;-xhV{m=KNf@oj-HLeAetVQkq5tq!@)lIv0rkC zf}Y`u(1e12>R8SI}~d!Fh4q)fjPaGd*@ zF4xc5JmK>bf2@eE))p3SuI#@zoWCuNnJL4Lc^tUmS2Ux7+N&?%iCx6Sq;a?ob0@7G zefmkUO9F|OqJk%N4jaDO|4B6L^%tWN^bTxVb^2UEyZt~;8g~^JZcSeGLWul_A>r~|Bv4k9$zuXjbv>2@*5kNGywm1Ads%$>3@u#X;Tc|ts%UbS!1Wp-M z!foZ!tn56o#R{4bqgUIk<8Aw~(9Rueuj091iO<)D@NQ3b8d_pKy!Vc(F)a8#ppAisL?Ix{a~Kpp5SEIdEukuPPQ=aIu7{zY3$;&ryC1XEM7CFvVV-#A zXVPA@MKf>+&Mix@dFhI6{uPff2qwDEZ0xwp(0O=ErG%!_(k!K5XY}h9{B`QX(dc#R^(oZG}?!VpGM9gC4-wt1zJ*n+UCEDvTJF+o8J&`$VaJF7!j518v&=?) zMxOc-6X~1ho74TOY-eO+DDTr;mjncUt@V`4U{O-j{s*j6zMjEjNd9bWKh@7&~TD zec-K3za_sUskUJrPoyk6r-$?1&*gY4;b;>EFXP$U8~xyLcUw6-VqgS#~n4 zzn&A88C-*@P8NzUlDC;AnqRw49H){{$~xh+h%U=sY)+Ks_mw3E3eImzDbq0A^x#@s zQ#rzed!cc}#xxlEaX3o@jGgqeVWdVia#L-l>(Bc)J7XoQ&3e5EjhJtJ%oSYB&OQup zL!X_Ek1(c99p*nW*w2NO`NJ6767bpXKb48VbkqfYw>)%h}nJeI_)K3&wfH7o3O~|(nwpTdXr|+Nb&gC|MYO-vDg0exP)Q! zI2iZA;=Ux{@o}bM?IFcm+cu2R@bRWZ%P8RDCwsv4BZbkc+AG-Xv<)0$oTxVgblxLo zo?>RmAkp+_kenj5A}bPUkYmJ;q-d7)L1J!#a%FoQ5xhhT{f@g5EBv_{*;3qhBljH9 ztm~Y+rDI`J0E`QLIuvEi!Aapp?NsLk&Nin};f|>7^$Di1b9nkm|K~c=Z=Ti0-bJ;oIy}gPwO=}9Q zt`oX!oZxKVAI9UmMHXkl=OKvILsZUivoQAk#cj%wpY1BjuUR#|Dy*H(XoV z(mZGRqpIe=-oUmvh8?MQn7XNW_E_#(tRsc}#Mtb$NqLR!mJ;k!tGsU$TGpztrr^<1 z9=6@ir(;&h_G!henEqofjaCyJ*6>4ENO{19V9C4s{R zJVR`$_lttT{pe3PEU@kOUbdewG%vp4wJWsNuvd^8mbhGQ_=nYvVpb%-gMuVv{B?1^Y3`V|;Xt2}CH-^?H5xsm+ zcTsAv5h>d=6B%2~DrkpT(WhwRZTZ6xD=+ZZ7ORIl*8dbA1xl_|8f0khhXas_QVCSZ*#(R2MM2Dw_Ru#(?11f!g^P(UcvB!G(;`|KQ9YjYTWe#~&=@w7~k!W4_ zw}Eu**MAr^6}K!txSEWtcXLN!6j8tj8^v8Oh1Ag+1vQW5-q2-Kr+UUx^$?ed`FOmc zBc*^%FWJQfch{$vMz-4tzRi9plPssEFiVr+UsD@e5DEdISt#@`J2Sh+9#927Q2ExuUgxs z%07OrAX*udtCkmZZQ8Nr@{N}eDfI(XP>$@dmD zxgYZlmKKZPp(WCC`XrYyUSGpo5u@D?ik4CIXN_#~HR_KaSunGV-nEWv2<$}kd+~N) zi-9r-H0L$Pbq*JTy=#W0Yv7vkz(ezP7saeCsq>^Yj$ul66ViGH4sWL4l{s3@q7P>; z=9`!?ep{@gt{sF6pA--r8FgWYq%)^#Ejop12D`Ve-NqUEVZMaNYhNMM_)*tnWW(oO zCtop92$v_bj&ZlKE2b>%JU-CoOW19Lskgf(`tdXiNVz#0G?zt`P_pd1rNX<)w*rE&i8ug& z_OC$i>gHu{;rd$|*`u!q+?2ufub6%^*7-qmO6CNs9@KLvzvT8sX{PN}A0}TLrO$kr z=EJeDmAw!*QYaN?Z0?Hj)(tq_we4nSXK-&hedN8b!0eMJ zqPL_$iR{O*?%IK^sc$|RD7=xrqGaTVsO z_im@S*oRIy;k5YDNMsiKetf8}XY`QSThVf_re%sL)=p&Z;EnF3j@q0%x(x@0Vo5?n zuBddfO%^3q__hU;^z&ld={}11ap~d;6x9}L@4_Y{73I~n`h#PRy#m)dS9Yof3vXZF z;TJgdPsR;0Fiqx8BSO$6ULQkD@aL45OJ!AUu1s|B*G?CtKhk}#e+2iD@a&Fu>p&BA zfZ)B~-sKO?pJhXouLEz-an6DTNyHzQ0L3G?x=lN%bMSBQK^ z(~i)N@+=#DN%zG8xhCm5Z7lBKvA$eU?#?|6R_~DE*$Jdf<8ePSxB{0R8|oO%temp} z@+#tGR7ZA_6>c3N^)Z8c%Bp$?*T~k=K2|@RC=|vi?ak>{ai|9C{ z2(MwCthj=q@S=u=0#V9t?kf#CJt-rqj8?u(?ZykWvUHbOOl6N3QV6T%Fio1r8?sor z;Z6>8mj)1w92FXXxkhNh?XJTfvZXnI#=0X-*!6{!N5=P;y|JM?=829#bDD# z9)kwv7kWZ*wEXigE8yFXG0GmD_|FacT?~jJ{oxaGA_;t_zNJ@!C{uPALMI&_;r?sz zuN6IHl?fUFwt>3xMqZi% z7`PUxXJV?F2xQDBVt={*E};somh3*Mv=D4HP@+DQ=8yKtKHAUMU*y{z)4XdVl%-gb>mPY4U<^`pwK# zs@V0g#uovl@4wP|!<%*8nhE!nD`gH#l%TW4H?0KOq3e$w&EbIJ6|GK%bgU}u<`!2$ zqKGtom!7?igqxFoWL=g!!?V1oBS!B<2C;c&c&3u!0&xW8Wz+U@*Mv28eh4< z(e4kV?nO$UnW8nTvg44#N>1h1z@;z|ZTpH~vTKHEWwoNF{~PMw8_qhJ<4rEw=8Q>^ zc!Rb4L9Q{=rpw$p4x^#q9<0xT3vLVNiK1YrfKqtbzE27B9vJfx{Br$UG?ylh?R*rN z1Ya(LtmLl&HJb)PlyE{tYd^m#$aJAP?n2Ruhaqq*&3ggP)i;4?CLkvqbr-|_>tf)RDxaPGpZ;JI!^t?&tIxA zJ9_m50u38LY-6wTZ-$_|qS&rGsG+%>h%{X?P+n>w#xR~Ds6IA7Xg;SpGCjjGi{s? zN&4+xl_aDx(DAEjotK-cGEv$KqPxdcpQm(T+_j?~CX-8h?if+G z8_4-Tci!d}e@iY_%i|cu&D)ok@gN3!6f_J+aa0Vpj;u|kB|@pkjikA^#*4F*(M<85%#B61OWuK~R28cB(}J!u zA!Dcd=JA9knFm9ZEB$tA;jB-Fh-v1=N6r{+y_j^tjz#UH+^gMbKEo3Pswt?Q$97#| zNo=7tZP;`A=C)6Xn4ZIybCF?3_zfcYl*nDh@u83=d;U?{uv07Toq65Rg1g{61lyhh ze2&{s!1+5uo+H??+9^KX#T!7V2RVD!J9+X&_WoCQ_auG+Xamzi7j!q8n8=*73%g_2 zrj)g&o_;Wp9alJZP$$sAfZpt3t@(RLdn?Mp>Ubu)PdSc{@EU0KQpn?FYb%=X)Jv-; z=NJ2;PUYE;RDh2nI!EtPyT0F57uj#!Yq8%}{W!Xy@ z5v?X`r`|zU3k`lA-n($*yUkbNv#SCVoeMdCx7B8PIsC73ZNoEclJ^F(*!pd%NM;|9 zLLO4X2A}>hB_YaL-}|lBneqI_{Uar@HgT~qf9vLA<7oAJX5yAOA^%(nlwN~-qJqeY zgMNfW=Ih8~f0lSZ?jTB8>#S$}elb5c0H|qz?|$LDGqUVsb+Thd+}W&izitnIoQw-? zrZ~>kwT;kn;xNjAAHaF;7WI;9L>cj>p6~r-6eunZGQ;{F zy-MGGU@T@L>o^W+D)MOR97@T%60_E^jw)orysmr6cJDL&JO(TRbw;ulHV3-8;S6b> z7{_8Bg`!1K{`w|Ha!ix40L+(H2DZsu0*6KNYT$1<(*8TnDyF_Ap1sJr0|8d1i$4`V zvrRvSmPxE#M?W&H0_d2Dr*_%zZyBOCfrAA9IAVF8>YIwsqhWo<3FbA-K5;#gDEYkWl~McXMpalE1WKuN&sp2hEeaPW0%e%%e%ow z;Gh4@T@QD1*jTlNnblg)6;q3MdYjvN6>fZRO-}17=Tu1PSeH5yQL8<9Yb=RA(-^}XQ zTlv3XloQ7)P0^0#$6)=my7uzB8ZRT(Dn5*c$qic%)iXBK8)^yDGFYE@t7kNtX%tb= zfENj(X)WKt^oA#4U3gKmj(op_Te*TSOd^HrlH?y#^OHws)loBY$3`S!N{%07FaMJ$ z-GbdAoZ`#@{3xECeE*66#f_Vk5RCPQyWO>%e=Zo#K2V-UJW~)J1^~ePD+NuQoc_na zXX^cNWF{*pZ*ky;Jy+%fB3r(`L6OKC;@splh00Y!sd))jGMlNRSz9X1G7{yBVqmPy zSn(qz3lOPqS_JydN=Oh$QWd<{H^;PRG0-O1w79rC_%M*D$PkwRh3pK#)w*MKk1V-|BwO;i9x-4|Vb*9unw@htl2h~)l(l+3Lh@=Zg* zPH4Hhsyz0!%1^T#@(#Dptg#A(8VYClEWIh8?}n>Jr7kToUu8@_Mp#GmFY|?8L1wtxitG<)A+{0p_CPlx{73d6vHP zv+ta*fgAhdD_=X|)=bE*fas2O0i+*c_Roj|1#UH*BS)W0X|-|atano?2tyMe!N z==eAAxAw!cNBm`b$FIO&MeaYLEzh#|FOv7K;J-_Ve?kEO5Yq49|Bs;ftDRqUr9Ul! zQU32s{D;c)tCe3xgFmhCq5q?V@T-AeixGbspv3>(z@MdwU!lJy4Szyw$o~cXHHG-q z!rz1FpLhUZn*sp%kC6H+{O=3kU*Xoz8p{8{|5y%H6yTm`7XUzheg!^9Z92N&uKpie CS{dJNstFJMS0l zR-aRql~p%Rcjk?Z=)55(1qO}=cn5$2002Y)gR?9%9S{H@790RT1wesn3fbB?8QVDN zD!JPkJ8ILpSz8h2f`d}z06^c~|KIUnJOZVOBQiY{TcetDa1kV`kMh*7X6Q{VfV`5&nan$~w60^nC*- zq#c;p@eUg5=Q(J&{EXPTx=|(>fK>a29t9TREdj|=95ZwbKEIu;wURqur9Td?K*K&`|Z0KgXw3^xc z{zL)OivT&xh+2#?CYuTG-jC>cpGGda3D=%Tw@~qVH*k%bvtwe7WUz>>X*Tl*vV~}<+bpFKC}`Xt>Rro%Qy%S|A*sk5l11%Hw|bFX=5FS0)5Ij*DO}oOs7e|Oze){l zd=QA%>$0Iq8X=or9rj>7qPppkyYoFa7G8}ix;g-=AZ&G3(7sy zgxCOS&W*1|Ez9LdmU}NdbWh3P#ZW*5LE-vag!)e>=`7^JYI<{!_irwQ_%>6{Oe@M9aSpGjFwr6SU|2L-Bp085 z?~m=p_WZc#ly|+RP{a)mnEuvSSug3eVFW>4#3_%SfTT{TtpF?4b}OBaJtp0iN*hG* zPT(XZQL0r){MwUY)}oJ@OlDDL2ho%+W~7Qr4eSz+SAj@ zl6oYCRWq5I$=@3xrRwx6_zs)7Dw_PvI1Yo5Z$lutCs>DCr{?5DQf0uM4-ha^wohp zz*?|A)3f$law%(YeI-)C8CamU&Amym42>KnB7Ex@3;yydPY z>3UuXyUk|6ZQX5P8DPS?$e(OCuyipyQ)4Hs!FRY#D1G3_YXmSmTg~yrbX=E@-18Wr z+X~sa$k)bcGH;vVKL8=}3!FjgOhl`zo@UjwP3z~KKAn;}i5zk}7Sn#NKjkXpwhM@p z(bDjaODM>pRS*?<7wwv`vUtB@%l&jg@W$c)WVt_IV#?Lqljb+3;{c#Q-dO&xXz*9A z|2r~(yd`RHzVhFFlqyI|^)eu}!M=ttxMsMZBQH786Q3v_BSH<;Qm&8@G5cI^;8Hbf ztIbQ%gV_eVpN|c=+_55TgQ4DZQ5S}zfP0`?opVFjj$Ms`K^dHu;RlF8prV}oI4(Jb zMM!relGiQGc{Y<)J7b|1Iw%6o*{!z+=( zFJ|TS3U(q485AS%rHpsn%S~%hKHqW@d>W?gSoujaa(!;HJ2Wu2@ix={Nf~jkI_S1H zx^TSZ_*idm{IM`Pni(5AInw`rVf<}{%uHIb&0s(a*&sjl<@b?o+NQZNV$z`{ z+uhMy_&PBpq#0C1Rmvr!;V|I0sxGHJMHJxyEfJJJD#!rpYnc-_z zd(tG?lIDc)>flgL`bM9nV3d+iJtZQfBw3i1XQmBeipT2R6Q4Ued`E8|hY}>SCBF)n zx5QeV?|U^rcOqD&mMQu z^h~;f!wY>xVu5=Re(Qx5=#h(rSz|`)E1c!+k&ld6i$%+GxPy~oq;lJsM9Yi6gO_Eb z^4LH^&#S(JS7fB}+K@xfo4%7)Exh-hK`-_J9u5b;)|^|l1S`!zv=lgWQ5&PvY`ByW zRqd9SacXnS=h}EIA!)2?nb8ivETObd!r>&&1Z8Kze!U}?zT@$_%Io+?%HQhM3N%6j z0La9@{g3^f@=j*P*2eU|pMPijiN;tAZUcG;y`)JxB8$vV~sAs`?L3-G4oP*qG?H>Kyh<6_1HHPM9HmU2IsNuZVYTqN%m+-* z0+Ze17+A&!)>(tiW8R-%%528+;*x48m2)!vwPgmzg01~qM|}_HAh^fjbI^MnjatqzkzI$v;Lx34!PdJ%)nPPP(bLh5Caig#hm16J$zNCFO zDcqAj693`ci`|ngO*A4DorI-Mr|=n^IiL7AxU}+uCqSbOWp|H#qt)&8CgK3-e%Xgf zpACe4KV|%lNjZ6yNG931iQx5h0Qhp_de{McT|v}+9g0dZelGESeVuFAcuDk7vk0Qp zd3`8R)%CskN$-3AN}$VCdk3DKyoHK`8YVMH>M?5Q#%~Cd@i8qtL*f(hCt7Gcf18jW zap5$ZLpaQs<+4_&eCV;pTAfcNrWgw~;-zTLqt6WCw400vr4vCj0K^+DQdn7)p=sVG zwIur(#&)|g-p;VS%_*v&Ye{V?NaQNBY^F?#RuX7=abW)hv^~$7X`+S;Zvk@`sV>De zwutwqEV8u8B6RVfNusDOdP__JyE@~Ho$rMcX~#t&yWOJQ)t#dPK7y&n{ZDTBb|(s) zU5wPY&rl_UB0sHYO^jE?sV=VL8KOlUiSwoQxLr0hkV#^JwT1F$s3f(~#9;jSmSbysvfDgISk>b92k=&#pfjaT11CAR zUXugkP3WMXh}(VUiDD>Di2V}ncU)G$o1K^sf;7D%S z>&L*EEdz-4$Wn7Zs5V0&kGuqhuB%2Z4pTN%Vp?y#qlhf^ATJm?4*!Bm1K#oMZt(?C z<)#t6L#nNYz5-9D#Nl?!C#Y@=sUltxbY;k9yP4M7Mhf>F*j~HTkZcQ^z7nuspJU$^ z;2j%KP^h2NN~ytHaEW?;`b0xvXlgMw7{^jZ0A0a)#R!+4ooN2a=E=_-Jmt|{4-~cm zZE=vWYtFVXe4IE_(3gTido+JknVwX#DX=~co9#u(LAt@5uWZ|pZ(=#CpkvpREKv=8 z#|I2Yc8R05SSi@1{-?m0tLZNA7&Id@=N3ADH$D96^WzO@tnWxC1VPrsApSG+k%R*bs z_b+s?b?g_lBHbLMu+$lWmr0ZtM&PzR?oP>1XCKk+*l(P_+ZBn(W*F4qz~GeKhFsLi z5HC1@Ne!n3kRGDUyFo>a3o&Q9v_V`)Z86CZxqb7w>$ZXpp=TzoMhCJLk)kWWS4A>K zWRxa6kQanQ^_s2TTwmGt_>+=_8SQdaI{6mu3bt+fnD2HT*QX2_2$&R~+)anpJGsD8 z@=0Ng>qgzLe6AzW^>3NTeIQM*PI8MR>cuJJ_jHvZ#UlVuD>*T%2E0j0zMvrf&p>B(dK)GGcxc(G%K+g)3Sa9QeTZUOfBzJ6Bim3+psXDO6q zi>zbJT75>~42o^b&F9sd!a45Vf=U*DTZHiu(-p5rxcC7i8Dv!&?o$)!)wF3QUiAJ3 zMLykwlussMWLHgek~K=NsWeDgy3TEr%AETl1MaMyDEt}em@11Zlj^5S0Uk9YA~g^# z=;OnSmN&(;tx1b`H8w%gmQx~{+SW3&&Sf?x^9Uo^%ir`3DI1sTh--%+f~PsTN5>rK zchjhoRG007R03SuHXfsNyph=6p?9odDgUTz)-`AMsFSRiDul=r-9&uaI^dTTv7a1l z_rmHiN76jl_?UW`WlOj{=D(1I9aFOEwIj#6&c4A9zV&KL*cr!9jR$d4iHShMGIsneq3zYuwA~g( z^Qo8x>Z$+uaQWU2T*<%pRC2{BUwW>cs~?HIozQbJNaf|6*VKxK89I=NF*0{eZ|7k= z&9VJqe}CxkKswoCW4l^1+^bYmwV=GuU1>DIi*p|62IV27T_T=3(NjCPQ`H_t{cJY?pA^=MKH)rXHWO!UJSj1a2q$+)7+{XbIb z>nUAD57$&(s!6CKiq&GNJ6Xf~h{LuQ&K`qp1CfP5ckhVQv-Ot+*Vy;CR-qEaaj(=delL&J2T#SC_a4~Uc68q!)v{&HffNk zlk7RYEDrKLFIxWaP7hN`c6>X;fD-E)I!b&)L--oD*I4oy{8^q!lNa8x03cUCZNHty zWqhJP_ak%Hp)sw;=aKm-=uEvy@AnV|4!!2Y5h__ZSA+OfIID0r^tfxx>O4vlqP_IX z;+6W@M1{~5WfjmYP!|UV;0jaRE@>jmA5`t%8X~SCkqL96>_MijiPmEOr6lC2YX6=F-Ok|wa1z9)CS$` z0W44^pPBvErC-$Vvd6tTEakZUzC;Vy@;o24KE@W08!WSEH&Byn@X@y2OReUC>jQF$CpL_NPpJ`ZfqRzp(PU4;`7%{b|bUtR_BJ@p0uU+#|t zr9!OHlgrWWN-`nCtB2ilC`z*6`EVg_lT3dHa3b<$Dsv$RJggeZkLyEsXy3Ey(cRDHd`lC)lX$!{?0z)l z?Jt+k91$!*po?x^^S4CMnmAiP&4~VFddZ`1T4A}cyq@t9>O=pnTVE5__O$o=Zj=3y zc~-bl-2}cN6qY%bnRp04RBlP}*31-@oXDG<6E~RTVvQgakDGU z?0MF^cWj%c#Cfmr2TAxHSf|4t02tRz$n*wDEvZuGbLsr>Hk8xI& z*X74_D2#>QyThFBo}*e@uc#SlggcaBtP?%oW+G`xpXQ6!-uO1eG-1$uo4dfEI~>r9 z{FQsjY3Vxl<2ZIZ{5(7IQZuM+uA&Y5IjX2 zKfg2yUGV_|X^@-axTW{v+?8^RIQpOn3<-KbEiwVa z+jJg7TEroGr3d1%we4$+(D80K%mbJ!qA+DrMy=(q`Bg}reVUwpIt>|g6I|aOhC#c- z=8jP?+f$LbwCX#8{l086Ri2~Z#lNd!(&xN1 z-yR%-)!%IW`t7DRgCZbF4$2qanCSh21dQfkXyk0@Jo<^G^l`=*NK8W|VKi%Ul*5F+ z|8;ga1Qa|*F+>Oi?VS=0gRY@`H6x8$Egv>4T5Fde?Lj5Ks_YGHIE$LW3GdKi%hOMB zNM5a0!Y(GCOEsF9MNMZfhKG%bFP`>n4^Xt&bO=DR_iyOX>u zb}~8k)9X4#o=X7q-Sci`?Ftmc%3PC6eD7y2KsY5CV@1s zbV~H{kcc$R;cvs6Z}oojjJNAR6T$s}3_^~`gzy4G?y|*DIgdamjG#&0U281>he+3% zwJp7ZCH%S-x1gxEJWm)-75&9Xk8iKUIfF=Es_Hn||2}hf;!?{f8cRR(WSDSmz)~TY z7N`T2Y@~N)kJ#RaL>gdI)Pcvm-jnP(GKDFh2-kIP*&P%|7g*DdvY=&T0gOd*8!=f3 z4LZYU;QK}h(_I`LxZ7;SF-8(}X{xrrs1aE36p#mL(OZDQ@c6}c@d=CN41A(?hMjf! z0TAelPv7k+K6aILmWG`k;aY!#KZmFmZ1{SZmtrm;UO^0L)8DxC@Hbs*@OJe$fWSX! zxf{Q_E6`WJk??U^Zy}Sw_%6{tGDj!$tSyeB)h-Whm;(LzB`IhK_>U<`?TA!x_}g~d zBLo0|_D4!$rte^Er0C>eZe#j;W)i40WwG)mMi873(D@AKl*}OXW7&sOjTEKQHpRbx zq(Z54mN*%85QTjTdl}bxGHh|tGgg@oAfU-|bdEXWRjr8_IDO}{IIk@I9>Vnd48W?* zVqV!B6T&#o>lwxG_){BOMnS#&g^{)$qc6~AUkegTvxTDZym@Vb!KuWvh4rj~y2(4~yk($&_wVJ+}r=IFrAvY5+Hxu&9p;yzIQ*7?TU_GlyOVO04_ogLG- zKQS;>Kt`#{M4$aj5U-P;VL{P0=bi$fNA_U%A?4FXj(wGQ8h(PAWNk_g8V{NMmyd>w zub@d3p$^-7ISQC4vARfod-oRs2PjOGi*~{jnMQELdNJCTKOk4qQ3t_oplZ5JQsL<+ z=?r0KPnx|g`S7M%gw=b}46P&-iu`EMG-K}@Q++AhGvja)2?W+_sWn~qyF4r$3V^;( z!pL0Lp?a4bqydo-%<$p*5kveLWeOLIxKTMB4g@0+sp1il)Y(xkZ zB+G9ggn_K{!VOHTaJ|WMJ`Qi^kgEl zJOg1mJ-7=TS+i^or`fzFovIHjg4nIzFjJ)Hio1DOj%q%a!fAW>aEt2rq5RHI8bfwN zVIJW6a=m|D5Xr4VVKw2)qMo#yD8?wYfuNZ`z6eZ`!BAp9HvV^t8g{!bdzvNZp%hb{ z6WHX*g~=<`54_jmT0d(E9H8VggHvvdaI)XmEKX9?s>DBvoU|kp3SubruOIg;a9Gb- z*D)Oz)FEW9rC|WypQK~>c+dw#K5Sp8QDLX3>DI<*IAj)KT7=r(J^!YV{pVJ(U7j}s)x&_+w*k>1G~}64kE$6KPe>3wCHt#ric?xaHwUdZoSiv#LbRP?ZrNS1-pM@%iKB#F zAz^u4(r8G@nqJR6_X=Qq#wmc*jV#r)5u)FOFc(bW0>qrpY7GQ@K<+d6-uv-*IG`J^ z+R-cCVIu1H(&zQSEChSmrGUHb&GjY=1~%ygmOMJq!7p1WeLac|7FtF1)Ci#%0jM_q z?$(r780${{vf0az7W>b{EN$idtEWfS@GX$~ns`YE zQzt&0Vl1*(5}DE6>L!Nqx^b6dmKj@C?PE(H2ZiE&QDXqhdwf9LYutZgpG|SjX7n5T zkih@|q`$IH-_GuTk09~&rZNSF zU!#peBjZkHV(O=6k6eC45ffzlcqBG)o}`!d2=fICj+r(MGc>9_R?^Py?7wc842KdW zPWJi*aS<2cK0PF;=Pw!<#3Y7^Mhk_6X5AsB~;mgDxP3&EIEvUSiM%gh7m znsOq2N?eXUZ;Mn&2`1LIq3YnuzMslh=Q+BsGxt5G&mPIXL&g;e*;BT@ORsP7{a-66 z2q^7ar2pp{x_{5vzpwvLQ70$$cLRT~_W3vPw`Re_sdx3il!T c3;d5&QBDftZFT_wn75DLTig~T|LyMo09K4SMgRZ+ literal 0 HcmV?d00001 diff --git a/frontend/static/csv/EstructuraCatTransportistas.xlsx b/frontend/static/csv/EstructuraCatTransportistas.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..5db522695c002ac38fe9e8437c003a61c7ae7317 GIT binary patch literal 10253 zcmeHtg;yQf^7X|bxVt;SoghI1!QI{6-Q67m1P>nEB?NcBxLa_y;Qrm2H}5wydGq}R z@AX})`*g3~wfdgkRlBGq0|HU&DUwNGsdI;x#+s5k)u`(~_$OZG6Rx^xZoB8Y=g2$u!E=B>Ie{1+ACvWmUb>88FswdlE|x4&Hi|+w;kTv-ku#au~Ap@58pX4&Gi_x zvIPT31E&@M@>WsxSQV_c<37FbG4rKI&bvugA1OD`33}J@jlXBd#T(1wklE1xC>+TC zAjiW+7wG$SaOfL%r`(AVfwyhZB+NFX=-n&E4eG)7%T8gGp-xf?pRki3g9p4`#}hkR zL8lif2Y#$C&;Y>eD+ECPZ*JM3!c2Yz;+izb>)=6dspn{7?Zm|R+x~y<_+RXUfBEa> zi869M%m|@p;!k0Nx3jD9D8f>1LgFo?%D(;*E65Eo`4sr89n=IU%6LH#AN|^VpN5xL z`C<+SNpCjU%cIaRc*z^xDndU$IJm&jQ8}iFIh1eoqPosr&)%ecl=Pr+ZHuET`&OJQ zHMCAHHhuA-24$R44G#{zkRTL`KP^DBUq)li@U9YUR#@qzBDA`RE9dajc!uA6O3?x0 z`*0rVvK_^4A#Y*o%Ytrj`JQXE#9*b(jY{z$G9(u;sotGanI?$iJ8DujD70KAq z?pVh^4p8P?`)Ss*UyNjV^m4)VlntH_1x67UuYnZmKb<6Qu`{s=g>nv5{_)e_o6xP2W?Pd6QMCIsOPzPj9RhmfR?9RHKAK z?i_;a&id(o`atZUrBKlg7EmBzZ(u|C*go_|m&Ga1l99Al@w*{biswm&rc+8rpp#az zub}w8nzVSUNcgoD^Nb}WtAL7zl7odCN&4qea37`cC1-P$8?0}U2Q5W6tSYmB8jQkf zWCj{B6|C`jMT$>3Dq9-?k{LKfwt7Jgf=P>(r@!sZy@71S$M9V1F}b*@h-^u@=F}Z4ZBb=S(T-TUW%8*{pomXhd}f>&ZVKdo*k1*`S_JRfS9j{=V2Y*mSz9osG05kSK+0|Q zHGtj2n{vEXU@0{u6b7O9H+cCm5HB2jWBXA!3%j;AT{RK> z<(NnlJZAis+7*c-&X+&Ye&I;Y6oSb`#BU+0)%+cWZq?^R^SP)g32skt%y-djeRx{O z%g4;PpG8rU4SMp`Qp-ZxB9AaakA;ujHDKr8mD%MP2O9c zKJfutBmQv<_+T^;|5O$#+XJZ4dw;V{D5a`!kZ;K<*AU)Bo&v>Ei#xBPoU=2?yhyp)Cp`#ri9+jQI zBc(f`Hwz>V(r06E@7~c+;)dlHk$t#oo(g5R1Twp@S~^V-4uj(2Nd30|L$K0uMPPkX z6G=L)>CS7j$np*Jmp~8V$xB=~2|h#*z~uf6>D#+5(Hq#Gtxu*h9zd(E{0D>`f-)(h zQVxFaP-l{`!H>j#w24kTd1);wXB*DKkHfSb%RlKyuFg!ihX!WXL2vp$!y*SJ*`z#( zE_)zC;ehV=BP=?Zo0vE|G5vmE`7J(XCTW4x?*f=O*IysUN7<)B(eqr;mr@B88PJ%fVG0CNuiNx}L&?WR)2K&9F zHbLXyjzk8Vl%kM7(IK=)t#?G-NZO5R!bS<^GLyG9HX^{Wcr!SfE|p?H#Va3VYTz7V zvJ_&`^Ux2oZF=d(ZA)X_6{8$#u`8!bG8RSJIVmjUhhyQ^p>$}BOl8hE`bKF3uS~TX zzYR;ps}D;r%?Htl@gQTD30pLdlAK{L84$FYyL-C_O>>65s`2h5TA3(YF6T(cvgmz% zPa)EMr*ZJ^HN)6E%)I843Ivnl%d&9ZYF&ySD*PJF9B-7$&eVn~75Niy4!_?LzN}{5 zkoC$T>B)KfdO#(;LJDA0vX1)HFyLrcjBd&(jRiKEk{{B^ugUt`FAk zxFx{ZLLCdSlpZ%LTPn$DcQRFJN>Z<#h=a8&7{Z-N_e+lqe!jyD z4O$f}I&m8pl1?*mD3~P@I?|U@sy!r`i0{t5f1TpqoZV&_h30^;I5lJ<;|-5_HM(^X z)8Q;(TQ`}02d~XpRu?<6>oVuE820X{`8XH&z^FB|*l9w1WW8T2Cwm&x?d__c&l-nh zowOSTf!}a{*eT|aO|Vc513j&CPGxMO?Ef-q=XM*O(4>q8xSY9 z*>BibF0K8kOHALaJXiX}uGtep*D0@yAFE!P#E(eV^rdS5j_Ds+c%xr0^bH08KqdR_ z!}50)I-8r=m@xgm|DAxxnqzVJO_&`pgRg?4I&-7@UIsi9U%75hJvWkdw#6lqLUn=m zf{HC~F~HvJ%}51+TTDs?ta;KzLYw?UofDyYu)0h`%<@epX$FFg3RrJxD1meQ#5V!koB}~L0cEO#yqk780@tia zWT;MemwR--J6^WdqSXh}jH2XaEahl)`FUgq{PJ>Qls0Ahq~L|ddOoPa7ee0>o6GX( zwL$=*vksTfqQ9`5&7A$&HPuKe=Xff&W%`XJXL~_MV-L?h`=i0?l$~JO%fl&KEN{L+ zPdC1NsNL}oFW-f;8TwW*le=A{c7el&?t~DoobbtQZ|qg;hUGDWwh(%5UgvJ!v7&Xn z-E(lL$?6zJ{=PtJOTF^GJ6vodfH_G|B$arDQf((&_@-{@w+|w8N}=;ak`CRJ2#*FR zg8Q@2_K$Y-vB~pMA5IZ01jK=mGAi?40L?bE?H#W5R`-|dsJ&N@i#}|o>{ocg zNs|Irm6R1y*%Xsz;+L0!*XL`u{f^g{Wn{gVp_tDmPi1~DFSAYS&qYLbU>kOr5J~@I z2#*cBQqCG(05jf1yYsQk3~Rnlyd1-2^oco=;RnlJ`FO}Q0Qp**5?)SqXo|mCJ=r0S zrQLpvzcXUz$0S|Im81?G3~G&eHfttLD+Qc__-p?-oCDvgS(2tJe-T?3r5?={uBeZ6 z7FF6r38r|+1ZhkclNGj*eS=BH*4N_kw4;)+?QXH|+RjlSU*S}f{zvyh`(s6(E|$0W zPq1Z!qCc$}OifnA>CUebnPbJA$O@%)6a4oWc%Df^dv~25S+oW?6=^(Ar#ymrn8?GP zyiqo|T^x%fl>2?@#z{Z0yxSRudcS+~X7ct2Oy;r6t&$|YTZaHDUjU?9kuqfXiwPw` zd#l{HHllg!fQYvvS0AiR=CgBv`}?qS@AyBbtBrv{|Rn)O}kEic3?YXX9zS@_F%4HbV;l07DxCuLNkR_XU zgpe9e3#8mfn{$Va0*bI@y0$@G#cZ(3lDZf8-gaBVg)y;F)?&VLmQZ3UBGyE+MrD+L zx~DFRgzYt7xxTuz>j|KwiZI@OSMBUqvMt>9!`EWF^QiIjkfD%i>GADUc%!o`JgtBf z7EmwdZaJudLNB0YJnx<|9i%4Fq`i0*f?jU2lmx_(X=QsDz@Emm^6(BTUZw2k3Xw`O zVxtr>?sdiCMZTa6L=&lj6&oA*?;~Gu5x!e>A1IzxpS7N}`&P}RyuBHat+CLw5VgxV zwSL%ey+(UmLY6vZxZk#Qe5gnr%`3t^+cn6lzf{Or@+yawY>{({ zTW!n;o<_54xei*nE}rG>EvjY@utNflm@RuhAS4c;$fByz^PZT(t)xw{@?-WlDGBK9 zeU>(jpt@{kl&n*JNu@{0(sOB>P~q7P8}Q)hL=((N$5vfXozOU04D_rU5v_x2!2}L3 zSY4Mgv?ebQ)Y*o}SWSv*>Db85xK!Ah&LNFtFBKRV(SBQMAgdpO3Z3HN9UXII+D?0$ zthQtyq8jMhw*C;K>x04xjoGn^qjK2rUC)Bcvq7?IvKT5~><9AW#-5;@sKdlyyEjgc z1&Y?*`uo(2EIX3Tv4Ht3+_MY!FIs5pJLf^Ze+-wdiCYK2T0Eqt*dM9TOYZIs6a?DZp4t9uibx`W@Q>aXMs}VO2HRf>OcJ4*huRp zw!f<8T1!C}RjMBUwv!{Wk1S$y{`4W#E*Mn=eEXJEBimp}c$I61H$gi;x`Q%Rz=u^6 zgQujvSbYyN5w0Y^q0JW`u}-nF`0GgSQW@e!-YeqbtQh=y(|Zw7W7N%-U0wgJ|> z+)BB)g2k`iba1jsW;+ijt$LJuQtdg zErzSAsDfjMJ>N5gRGj2>O%q)rSDT=uwpcZ0F2!L-~<%%@j%6!4>!Pm_FbpGAp^zG=6-B~?Icgw((GEC2MB!V}u4`@x96zfvZ1 zM7RuzG4}gvfEAMV_~|@)My#~i1)qjlmDT*xTE=@=^8Oq5zGj@wDIdaa)7_Cd4uny? zPXa?|?6d6CiBSINyprN6`+2d}7jliU+HlS99HA(V29x*0MK28zTNIek2_QwL3+nPe z-PaPIpO_OdT)xZx7^@1VRal`L7dGpMA!-2i1xx9=FfT870Irlbz@nqwhyX%qFh79!<#*z|)$$8?2H;4{pY&g~sf3Jf()o-S zkcXI*@5#niH?Oe5f!zq$dv7jDBUC6^w3l)VYfw7-w0Qh=n=%;3-xb^sgLg+T-gT0J zv)bdTI;O*Os?ZR9e3L{r%jnNrsv|%1tVw_^U!fc(q<}nnsjwfojupWOBisrv{MlOw z+np8mIXRsf_BjL(d*&#=LXwz=wbQlH(xh4ij)F@>u_~^S&S!flN0-D#1$-z$s7+w0 z^eTZm$TPrs2APEL`e8p)gBa1bt6o=NM^*O1y6*t~>yqhYWsah^;I^u1pUdK0duSL= z|96|0g6sMWn!sdvSU*G)Qo?zOH(LAQ(KF$5n8%VbM;T*aaZS}Eu^cHej^hRaSJ{zJ zu!vZtP+?FEx61U)dPWMhEcEL20=V!PtzE(ld)0z!a@X*Y?COTc{6h;Zk3Yp>__bR} zx>$WL)am0E%pBF_ca&<>i4+`p=!<`r{C3T-c_Ap_w`(;b+tvQIUkw{PoE7A7Q>bwt zU)K2Ym>n+?x9Q^ge$qJ3Cj2qLETczfB8u8kn1NGMq;{uI`KUmxEs^yDWTbDwE3w9} zq?l~WMxuY7hTe^Aeq+q5#2qxWA{!x;NC6G>$U1Z+d_e=HywY!prHc8?V;K2Il45Rn zK)2GjX#FV&L-f-5E^oeyZ=7#e9$_2=UNWRnv3N!1o7q6OlrY3E5LM$RgMDA_l<4K7 z5N)0%+C;S2=q>O{wC}(W#V1Dvqef;$dVWLgy1`sMhs5{+NsGF>-bM%knXxl#Q)U_G z!^=j(ypsOX97!Zy>=$Ewft@m!3{nNDnxm9}yUgwJ3vJ_A9D~f`VUpDWE5%TTS6$c? zWBpSHtw0OUPcaUGg!ay3@}@4@^SOrgCg2%xdmR{ z78z(2C9N# z{E?EF8#tO6D>*w_*qZ&GnK&mZ$n}t5gsjUvAqTq+>;|zEoj86x5LT3ea6TK0wEC0` z?2dl%n#3huI!U<9>S=C9sd)cL{=l4n1dYEnYZ#GG^O$>ZOxm9eB+6!SY;TnDAt;Zy z`0U`GwX!Y>^uVZ{>-%>7awQzT+*DMxYRHX?Ke)_%(||m1A-H0}?y{w6o3vfBqnl%X5q~h4x0}O-b_<3hI{++c0|cA?~ZrsB#qUXvDJT4_5G;g6-(edp!_L^2R8q#blc>O8YJn!8 z_!E@T{bz+S{Tb8hBWQljLGz0WDoPvM8Ob}^**h^A**Tj0v1H(XwP+CWG81*AdYExS z&Y)gJhI}%72XS)8CWa~rU}Xu`Lbg{`Z3M*gR-dkmPE@(BBnM?2fsgiUBEyTioRie* zYE~+_^b!h?kyZ+2HX+$+QSOmsdueFplW*vp0b!W9^l4 zvu;E6kMEEuoFwdvNo?zrN5jfi4SVi*m%)M4&ViKfRHQ7r_lry(E* zCj23H2!V+RukQTn2QLKs$!uTCpSyyyVjad81CclP*IGQ;8*MV`MTyC`$Im_Q&u6Oe z=1Y$vA@T};1Z(N*Y7MuBaOrH;eXVQ{<>^3>v9m;RgYjJM6u3g*Os``y?Ml9DqSb$7 zfaVOR*(U43qTVS?U9Z9l-?b2km?q0qaTWPVI}|m~*wg3z*~vYAfZ~-Z8&E&)bI-NU z^^e{iqTw$e3t}HS1OR~YSN0j$+y9Szpeg%f%S@E9TmG#<3~@t@u!_&KDvuCQZYa)s zRc!#a*=QzPWRx3g93Gu;JRjFMIiumW6hq9U%TH!u8~4b0#F7w~D?cJy$%??JDscSf zTHUwhA&=D%7PFp3KUY@#aL)HhpOe-#b%ET(5Vcqy(Wz-FT|3AtC!{{e`7rgSGqETu z*5!n7P1J3{NZ%)Gj|(p@Iv4a6#*1G%KeqsdI>cuV~_pSFJBM(Pkx#v_zj!X)um6` z&0GJeP+;H;AT|2ug;)Q6Xa7F_!!j&+slOZe`x23V1AjYy206xG7L5D~{IzZVC$tsR zNdKjY{ww(J{m4I|06-4h@8JJWck)*|zc#7e-H5=EvsLx{Muyr(~1VtKeilx zHSlZk;!gv@Sic+ivy|~G^w%WgPv|Vszo5US9KTxlyNv#c2LL!o0D%7x)nDO%kAr`O g*MWL2{|EmgD#}Yif!;0v@CNkr2kEvO Date: Tue, 3 Mar 2026 08:16:50 -0700 Subject: [PATCH 14/16] feature/faltantes-csv --- frontend/src/lib/api.ts | 42 ++++++++++ .../csv-upload/UploadLauncherGrid.svelte | 3 +- frontend/src/lib/config/csv-upload.ts | 21 +++++ .../routes/dashboard/csv-upload/+page.svelte | 79 ++++++++++++++++++- 4 files changed, 142 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index d535507f..0e8b57f9 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -508,6 +508,48 @@ export const api = { api.post(`/v1/a76/transportation/trailers/imports/${jobId}/commit`, {}) }, + // CSV import for Transportistas (transporters/imports) + transporterImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/transporters/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/transporters/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/transporters/imports/${jobId}/commit`, {}) + }, + + // CSV import for Números de parte (parts/imports) + partNumberImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/parts/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/parts/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/parts/imports/${jobId}/commit`, {}) + }, + + // CSV import for BOMs (boms/imports) + bomImports: { + upload: (file: File, companyId: number) => { + const formData = new FormData(); + formData.append('file', file); + return fetchApi( + `/v1/a76/boms/imports/upload?company_id=${companyId}`, + { method: 'POST', body: formData } + ); + }, + status: (jobId: string) => api.get(`/v1/a76/boms/imports/${jobId}/status`), + commit: (jobId: string) => api.post(`/v1/a76/boms/imports/${jobId}/commit`, {}) + }, + // Generic request for custom needs (like file uploads) request: (endpoint: string, options: RequestInit = {}) => fetchApi(endpoint, options) }; diff --git a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte index 62830c66..3f17fc78 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte @@ -93,10 +93,11 @@ e.preventDefault(); return; } - if (!item.templateUrl) return; e.preventDefault(); + if (!item.templateUrl) return; + const link = document.createElement('a'); link.href = item.templateUrl; link.download = item.templateUrl.split('/').pop() || 'plantilla.xls'; diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index 2deb6775..da79ab77 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -161,6 +161,20 @@ export const catalogosConfig: CsvUploadItem[] = [ modelTarget: 'MaterialClass', templateUrl: '/csv/EstructuraCatClasesAF.xls' }, + { + id: 'part_numbers', + title: 'Números de parte', + icon: Hash, + modelTarget: 'Part', + templateUrl: '/csv/EstructuraCatNumerosParte.xlsx', + }, + { + id: 'boms', + title: 'BOMs', + icon: Briefcase, + modelTarget: 'Bom', + templateUrl: '/csv/EstructuraBOMS.xlsx', + }, { id: 'items', title: 'Partidas (Permisos)', @@ -194,6 +208,13 @@ export const catalogosConfig: CsvUploadItem[] = [ ]; export const transportesConfig: CsvUploadItem[] = [ + { + id: 'transporters', + title: 'Transportistas', + icon: Ship, + modelTarget: 'Transporter', + templateUrl: '/csv/EstructuraCatTransportistas.xlsx', + }, { id: 'transports', title: 'Transportes', diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index c5ec47e8..5a1901f1 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -42,6 +42,12 @@ let useDriverImport = $state(false); // Cuando es true, usamos API de importación de Trailers y Cajas (trailers/imports) let useTrailerImport = $state(false); + // Cuando es true, usamos API de importación de Transportistas (transporters/imports) + let useTransporterImport = $state(false); + // Cuando es true, usamos API de importación de Números de parte (parts/imports) + let usePartNumbersImport = $state(false); + // Cuando es true, usamos API de importación de BOMs (boms/imports) + let useBomImport = $state(false); // Initialize settings for all tabs upfront to avoid reactivity loops let allSettings = $state>(() => { @@ -69,6 +75,9 @@ useVehicleImport = config.id === 'transports'; useDriverImport = config.id === 'drivers'; useTrailerImport = config.id === 'trailers'; + useTransporterImport = config.id === 'transporters'; + usePartNumbersImport = config.id === 'part_numbers'; + useBomImport = config.id === 'boms'; const companyId = companyStore.activeCompany?.id || 1; @@ -234,6 +243,60 @@ return; } + if (useTransporterImport) { + try { + const res = await api.transporterImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (usePartNumbersImport) { + try { + const res = await api.partNumberImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + + if (useBomImport) { + try { + const res = await api.bomImports.upload(file, companyId); + if (res.data?.job_id) { + currentJobId = res.data.job_id; + pollStatus(); + } else { + toast.error(res.error || 'Error al subir el archivo'); + isUploading = false; + } + } catch (e) { + console.error('Upload exception', e); + toast.error('Error inesperado al subir el archivo'); + isUploading = false; + } + return; + } + const currentSettings = allSettings[activeTab] || {}; const footerConfig = { ...currentSettings }; if (activeTab === 'importacion') { @@ -286,7 +349,13 @@ ? await api.driverImports.status(currentJobId) : useTrailerImport ? await api.trailerImports.status(currentJobId) - : await api.imports.status(currentJobId); + : useTransporterImport + ? await api.transporterImports.status(currentJobId) + : usePartNumbersImport + ? await api.partNumberImports.status(currentJobId) + : useBomImport + ? await api.bomImports.status(currentJobId) + : await api.imports.status(currentJobId); console.log('Poll response', res); if (res.error && !res.data) { toast.error(res.error || 'Error al consultar el estado'); @@ -448,7 +517,13 @@ ? await api.driverImports.commit(currentJobId) : useTrailerImport ? await api.trailerImports.commit(currentJobId) - : await api.imports.commit(currentJobId, activeModelTarget || ''); + : useTransporterImport + ? await api.transporterImports.commit(currentJobId) + : usePartNumbersImport + ? await api.partNumberImports.commit(currentJobId) + : useBomImport + ? await api.bomImports.commit(currentJobId) + : await api.imports.commit(currentJobId, activeModelTarget || ''); if (res.data?.commit_job_id) { currentJobId = res.data.commit_job_id; pollStatus(); From 343054d27405c76e4940c4804645525a6ef2a815 Mon Sep 17 00:00:00 2001 From: hreyes Date: Tue, 3 Mar 2026 09:09:14 -0700 Subject: [PATCH 15/16] feature/csv-partes --- .../v1/modules/a76/parts/imports/__init__.py | 1 + .../v1/modules/a76/parts/imports/routes.py | 151 ++++ .../v1/modules/a76/parts/imports/schemas.py | 22 + .../api/v1/modules/a76/parts/imports/tasks.py | 639 +++++++++++++++ .../a76/parts/imports/template_config.py | 54 ++ backend/api/v1/modules/a76/parts/routes.py | 36 +- backend/core/celery_app.py | 1 + frontend/src/lib/config/csv-upload.ts | 774 +++++++++--------- 8 files changed, 1278 insertions(+), 400 deletions(-) create mode 100644 backend/api/v1/modules/a76/parts/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/parts/imports/routes.py create mode 100644 backend/api/v1/modules/a76/parts/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/parts/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/parts/imports/template_config.py diff --git a/backend/api/v1/modules/a76/parts/imports/__init__.py b/backend/api/v1/modules/a76/parts/imports/__init__.py new file mode 100644 index 00000000..025db976 --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/__init__.py @@ -0,0 +1 @@ +# CSV import for Parts (Números de parte) diff --git a/backend/api/v1/modules/a76/parts/imports/routes.py b/backend/api/v1/modules/a76/parts/imports/routes.py new file mode 100644 index 00000000..bffb87ac --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para Números de Parte. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + PART_IMPORT_FILE_PREFIX, + PART_IMPORT_META_PREFIX, + PART_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"Parts import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "part_numbers", + } + + try: + r = _get_redis() + r.set( + f"{PART_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=PART_IMPORT_REDIS_TTL, + ) + r.set( + f"{PART_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=PART_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"Parts import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"Parts import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("Parts import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/parts/imports/schemas.py b/backend/api/v1/modules/a76/parts/imports/schemas.py new file mode 100644 index 00000000..7827dae6 --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/parts/imports/tasks.py b/backend/api/v1/modules/a76/parts/imports/tasks.py new file mode 100644 index 00000000..c87bca7c --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/tasks.py @@ -0,0 +1,639 @@ +""" +Tareas Celery para importación CSV de Números de Parte. +Flujo: scan_file (validación) → insert_valid_rows (commit). +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +PART_IMPORT_FILE_PREFIX = "part_import_file:" +PART_IMPORT_META_PREFIX = "part_import_meta:" +PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:" +PART_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"Parts import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"part_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"Parts import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{PART_IMPORT_FILE_PREFIX}{job_id}", + f"{PART_IMPORT_META_PREFIX}{job_id}", + f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"Parts import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_part( + row: Dict[str, Any], + line_num: int, + valid_class_codes: Optional[Set[str]] = None, + valid_uom_codes: Optional[Set[str]] = None, + valid_currency_codes: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + part_number = (row.get("NUMPARTE") or "").strip() + if not part_number: + return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} + if len(part_number) > 70: + return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"} + + commercial = (row.get("NUMPARTECOM") or "").strip() + if commercial and len(commercial) > 70: + return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"} + + desc_es = (row.get("DESCRIPCIONE") or "").strip() + if desc_es and len(desc_es) > 500: + return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"} + desc_en = (row.get("DESCRIPCIONI") or "").strip() + if desc_en and len(desc_en) > 500: + return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"} + + part_class = (row.get("CLASE") or "").strip() + if part_class and len(part_class) > 8: + return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"} + # Si CLASE no existe en catálogo se guardará null (no se rechaza la fila) + + uom = (row.get("UNIMED") or "").strip() + if uom and len(uom) > 5: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"} + # Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila) + + currency_key = (row.get("MONEDA") or "").strip() + if currency_key and len(currency_key) > 3: + return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"} + # Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila) + + unit_cost = row.get("COSTOUNIT") + if unit_cost is not None and unit_cost != "": + try: + Decimal(str(unit_cost)) + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"} + + unit_weight = row.get("PESOUNIT") + if unit_weight is not None and unit_weight != "": + try: + Decimal(str(unit_weight)) + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"} + + fraction = (row.get("FRACCION") or "").strip() + if fraction and len(fraction) > 10: + return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"} + us_fraction = (row.get("FRACCIONAME") or "").strip() + if us_fraction and len(us_fraction) > 16: + return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"} + fda_key = (row.get("FDAKEY") or "").strip() + if fda_key and len(fda_key) > 20: + return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"} + fcc_key = (row.get("FCCKEY") or "").strip() + if fcc_key and len(fcc_key) > 30: + return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"} + license_code = (row.get("LICENCIA") or "").strip() + if license_code and len(license_code) > 3: + return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"} + eccn = (row.get("ECCN") or "").strip() + if eccn and len(eccn) > 20: + return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"} + export_code = (row.get("EXPORTCODE") or "").strip() + if export_code and len(export_code) > 2: + return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"} + exclusion = (row.get("EXCLUSION") or "").strip() + if exclusion and len(exclusion) > 19: + return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"} + weight_type = (row.get("TIPOPESO") or "").strip() + if weight_type and len(weight_type) > 6: + return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"Parts import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"Parts import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_class_codes: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_currency_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + for c in ( + session.query(Class.class_code) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + valid_class_codes.add(c[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + for cur in session.query(CurrencyType.code).all(): + valid_currency_codes.add(cur[0]) + except Exception as e: + logger.warning(f"Parts import: could not load FK sets: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + ) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"Parts import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=PART_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"Parts import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]: + if val is None: + return None + s = str(val).strip() + if not s: + return None + if max_len and len(s) > max_len: + return s[:max_len] + return s + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _bool_from_row(val: Any) -> bool: + if val is None or val == "": + return True + s = str(val).strip().upper() + if s in ("0", "F", "FALSE", "NO", "N"): + return False + return True + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"Parts import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"part_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"Parts import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + from api.v1.modules.a76.parts.models import Part + + valid_class_codes: Set[str] = set() + valid_uom_codes: Set[str] = set() + valid_currency_codes: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + for c in ( + session.query(Class.class_code) + .filter( + Class.tenant_id == tenant_id, + Class.company_id == company_id, + ) + .all() + ): + valid_class_codes.add(c[0]) + for u in ( + session.query(UnitOfMeasure.code) + .filter( + UnitOfMeasure.tenant_id == tenant_id, + UnitOfMeasure.company_id == company_id, + ) + .all() + ): + valid_uom_codes.add(u[0]) + for cur in session.query(CurrencyType.code).all(): + valid_currency_codes.add(cur[0]) + except Exception as e: + logger.warning(f"Parts import: could not load FK sets: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_missing_fk = 0 + skipped_duplicate = 0 + skipped_details: List[Dict[str, Any]] = [] + response = None + + try: + with CoreSessionLocal() as session: + existing_by_part_number: Dict[str, Part] = {} + for p in ( + session.query(Part) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + existing_by_part_number[p.part_number] = p + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_part( + row_norm, + i, + valid_class_codes=valid_class_codes, + valid_uom_codes=valid_uom_codes, + valid_currency_codes=valid_currency_codes, + ) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + part_number = _str_or_none(row_norm.get("NUMPARTE"), 70) + if not part_number: + skipped_invalid += 1 + continue + + commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70) + desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500) + desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500) + part_class = _str_or_none(row_norm.get("CLASE"), 8) + if part_class and part_class not in valid_class_codes: + part_class = None + unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5) + if unit_of_measure and unit_of_measure not in valid_uom_codes: + unit_of_measure = None + currency_key = _str_or_none(row_norm.get("MONEDA"), 3) + if currency_key and currency_key not in valid_currency_codes: + currency_key = None + + unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT")) + currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None + unit_weight = _decimal_or_none(row_norm.get("PESOUNIT")) + weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6) + fraction = _str_or_none(row_norm.get("FRACCION"), 10) + us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16) + fda_key = _str_or_none(row_norm.get("FDAKEY"), 20) + fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30) + license_code = _str_or_none(row_norm.get("LICENCIA"), 3) + eccn = _str_or_none(row_norm.get("ECCN"), 20) + export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2) + exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19) + is_active = _bool_from_row(row_norm.get("ACTIVO")) + + existing = existing_by_part_number.get(part_number) + if existing: + existing.commercial_part_number = commercial + existing.description_spanish = desc_es + existing.description_english = desc_en + existing.part_class = part_class + existing.unit_of_measure = unit_of_measure + existing.unit_cost = unit_cost + existing.currency_type = currency_type + existing.currency_key = currency_key + existing.unit_weight = unit_weight + existing.weight_type = weight_type + existing.fraction = fraction + existing.us_fraction = us_fraction + existing.fda_key = fda_key + existing.fcc_key = fcc_key + existing.license_code = license_code + existing.eccn = eccn + existing.export_code = export_code + existing.exclusion_symbol = exclusion_symbol + existing.is_active = is_active + session.add(existing) + inserted_count += 1 + else: + new_part = Part( + tenant_id=tenant_id, + company_id=company_id, + client_id=company_id, + part_number=part_number, + commercial_part_number=commercial, + description_spanish=desc_es, + description_english=desc_en, + part_class=part_class, + unit_of_measure=unit_of_measure, + unit_cost=unit_cost, + currency_type=currency_type, + currency_key=currency_key, + unit_weight=unit_weight, + weight_type=weight_type, + fraction=fraction, + us_fraction=us_fraction, + fda_key=fda_key, + fcc_key=fcc_key, + license_code=license_code, + eccn=eccn, + export_code=export_code, + exclusion_symbol=exclusion_symbol, + is_active=is_active, + ) + session.add(new_part) + existing_by_part_number[part_number] = new_part + inserted_count += 1 + + try: + session.commit() + except Exception as db_err: + session.rollback() + logger.error(f"Parts import DB error: {db_err}") + return {"status": "failed", "error": str(db_err)} + + total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate + if inserted_count == 0 and total_skipped > 0: + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + "message": f"No se insertaron registros. {total_skipped} rechazados.", + } + elif inserted_count == 0: + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + else: + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + + except Exception as e: + logger.error(f"Parts import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"Parts import cleanup failed: {cleanup_err}") + + if response is None: + response = { + "status": "failed", + "error": "Error inesperado", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": skipped_missing_fk, + "skipped_duplicate": skipped_duplicate, + "skipped_details": skipped_details, + } + return response diff --git a/backend/api/v1/modules/a76/parts/imports/template_config.py b/backend/api/v1/modules/a76/parts/imports/template_config.py new file mode 100644 index 00000000..7cc240c4 --- /dev/null +++ b/backend/api/v1/modules/a76/parts/imports/template_config.py @@ -0,0 +1,54 @@ +""" +Configuración de plantilla CSV para Números de Parte (EstructuraCatPartesAF.xls). +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "part_numbers": [ + {"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE", "PART NUMBER", "NUM PARTE"]}, + {"canonical": "NUMPARTECOM", "aliases": ["NUMERO PARTE COMERCIAL", "COMMERCIAL PART", "PARTE COMERCIAL"]}, + {"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES", "DESC ESPANOL"]}, + {"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION", "DESC INGLES"]}, + {"canonical": "CLASE", "aliases": ["CLASS", "CLASE MATERIAL", "PART CLASS"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM", "UNIT OF MEASURE"]}, + {"canonical": "COSTOUNIT", "aliases": ["COSTO UNITARIO", "UNIT COST", "COSTO"]}, + {"canonical": "MONEDA", "aliases": ["CURRENCY", "MONEDA CLAVE", "CURRENCY KEY"]}, + {"canonical": "PESOUNIT", "aliases": ["PESO UNITARIO", "UNIT WEIGHT", "PESO"]}, + {"canonical": "TIPOPESO", "aliases": ["WEIGHT TYPE", "TIPO PESO"]}, + {"canonical": "FRACCION", "aliases": ["FRACCION MEX"]}, + {"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]}, + {"canonical": "FDAKEY", "aliases": ["FDA", "FDA KEY"]}, + {"canonical": "FCCKEY", "aliases": ["FCC", "FCC KEY"]}, + {"canonical": "LICENCIA", "aliases": ["LICENSE CODE", "LICENSE"]}, + {"canonical": "ECCN", "aliases": ["ECCN CODE"]}, + {"canonical": "EXPORTCODE", "aliases": ["EXPORT CODE", "CODIGO EXPORT"]}, + {"canonical": "EXCLUSION", "aliases": ["EXCLUSION SYMBOL", "SIMBOLO EXCLUSION"]}, + {"canonical": "ACTIVO", "aliases": ["IS ACTIVE", "ACTIVE", "ACTIVO"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("part_numbers") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/parts/routes.py b/backend/api/v1/modules/a76/parts/routes.py index 033ad472..17bf937a 100644 --- a/backend/api/v1/modules/a76/parts/routes.py +++ b/backend/api/v1/modules/a76/parts/routes.py @@ -2,21 +2,31 @@ Endpoints API para gestión de partes (SCAII) """ -from api.v1.common.tenant_crud_routes import TenantCRUDRoutes +from fastapi import APIRouter + +from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from .dto import PartCreateDTO, PartResponseDTO, PartUpdateDTO from .service import PartService +from .imports.routes import router as imports_router +router = APIRouter() -router = TenantCRUDRoutes( - service=PartService, - create_schema=PartCreateDTO, - update_schema=PartUpdateDTO, - response_schema=PartResponseDTO, - prefix="/parts", - tags=["a76 / parts"], - resource_name="Part", - id_name="part_id", - enable_list=True, - enable_filters=True, -).router \ No newline at end of file +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/parts/imports", tags=["a76 / parts / csv_import"]) + +# CRUD +router.include_router( + TenantCRUDRoutes( + service=PartService, + create_schema=PartCreateDTO, + update_schema=PartUpdateDTO, + response_schema=PartResponseDTO, + prefix="/parts", + tags=["a76 / parts"], + resource_name="Part", + id_name="part_id", + enable_list=True, + enable_filters=True, + ).router +) \ No newline at end of file diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index 279b0a01..a856f339 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -39,6 +39,7 @@ celery_app.conf.update( "api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks", "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", "api.v1.modules.a76.classes.imports.tasks", + "api.v1.modules.a76.parts.imports.tasks", "api.v1.modules.a76.transportation.vehicles.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task", diff --git a/frontend/src/lib/config/csv-upload.ts b/frontend/src/lib/config/csv-upload.ts index da79ab77..9ab11d00 100644 --- a/frontend/src/lib/config/csv-upload.ts +++ b/frontend/src/lib/config/csv-upload.ts @@ -1,387 +1,387 @@ -import { - User, - Users, - FileText, - Truck, - Container, - Ship, - Plane, - Package, - Briefcase, - Globe, - CreditCard, - DollarSign, - Calendar, - Hash, - MapPin, - ShieldCheck, - FileDigit, - Scale, -} from 'lucide-svelte'; - -// --- Interfaces --- - -export interface CsvUploadItem { - id: string; - title: string; - icon: any; - group?: string; // For grouping within a tab - modelTarget?: string; // The backend model this maps to - description?: string; - templateUrl?: string; // Path to the template file in static/ - disabled?: boolean; // New property to mark items as "Coming Soon" -} - -export interface CsvUploadField { - name: string; - label: string; - type: 'text' | 'select' | 'boolean' | 'date' | 'radio'; - options?: { label: string; value: string | boolean | number }[]; - required?: boolean; - defaultValue?: any; -} - -// Map of Tab ID -> Array of Fields -export const tabSettings: Record = { - catalogos: [ - { - name: 'mode', - label: 'Modo de Carga', - type: 'radio', - options: [ - { label: 'Actualizar', value: 'update' }, - { label: 'Reemplazar', value: 'replace' } - ], - defaultValue: 'update' - } - ], - transportes: [ - { - name: 'mode', - label: 'Modo de Carga', - type: 'radio', - options: [ - { label: 'Actualizar', value: 'update' }, - { label: 'Reemplazar', value: 'replace' } - ], - defaultValue: 'update' - } - ], - importacion: [ - { - name: 'autonumber_remesas', - label: 'Autonumerar Remesas', - type: 'boolean', - defaultValue: false - }, - { - name: 'recalculate_dates', - label: 'Recalcular Fechas', - type: 'boolean', - defaultValue: false - }, - { - name: 'dateFormat', - label: 'Formato de Fecha', - type: 'select', - options: [ - { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, - { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, - { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } - ], - defaultValue: 'dd/mm/yyyy' - } - ], - exportacion: [ - { - name: 'invoice_type', - label: 'Tipo de Factura', - type: 'select', - options: [ - { label: 'AFIJO', value: 'AFIJO' }, - { label: 'NORMAL', value: 'NORMAL' }, - ], - defaultValue: 'AFIJO', - }, - { - name: 'is_regime_change', - label: 'Es Cambio de Régimen', - type: 'boolean', - defaultValue: false, - }, - { - name: 'dateFormat', - label: 'Formato de Fecha', - type: 'select', - options: [ - { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, - { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, - { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } - ], - defaultValue: 'dd/mm/yyyy' - } - ] -}; - -// --- DATA DEFINITIONS (Items only, no config) --- - -export const catalogosConfig: CsvUploadItem[] = [ - { - id: 'customs_brokers', - title: 'Agentes Aduanales', - icon: User, - modelTarget: 'CustomsBroker', - templateUrl: '/csv/EstructuraCatAgenteAduanal.xls' - }, - { - id: 'clients_providers', - title: 'Clientes y Proveedores', - icon: Users, - modelTarget: 'ClientProvider', - templateUrl: '/csv/EstructuraCatClienteProv.xls' - }, - { - id: 'exchange_rates', - title: 'Tipo de Cambios', - icon: DollarSign, - modelTarget: 'ExchangeRate', - templateUrl: '/csv/EstructuraCatTiposCambio.xls' - }, - { - id: 'american_fractions', - title: 'Fracc. Ame.', - icon: Globe, - modelTarget: 'AmericanFraction', - templateUrl: '/csv/EstructuraCatFraccAme.xls' - }, - { - id: 'material_classes', - title: 'Clases de Materiales', - icon: Package, - modelTarget: 'MaterialClass', - templateUrl: '/csv/EstructuraCatClasesAF.xls' - }, - { - id: 'part_numbers', - title: 'Números de parte', - icon: Hash, - modelTarget: 'Part', - templateUrl: '/csv/EstructuraCatNumerosParte.xlsx', - }, - { - id: 'boms', - title: 'BOMs', - icon: Briefcase, - modelTarget: 'Bom', - templateUrl: '/csv/EstructuraBOMS.xlsx', - }, - { - id: 'items', - title: 'Partidas (Permisos)', - icon: FileText, - group: 'Permisos', - modelTarget: 'ItemPermission', - templateUrl: '/csv/EstructuraCatPartesAF.xls' - }, - { - id: 'headers', - title: 'Encabezados (Permisos)', - icon: FileText, - group: 'Permisos', - modelTarget: 'HeaderPermission', - disabled: true, - }, - { - id: 'historical_fractions', - title: 'Fracciones Históricas', - icon: Calendar, - modelTarget: 'HistoricalFraction', - disabled: true, - }, - { - id: 'pedimentos', - title: 'Pedimentos', - icon: FileDigit, - modelTarget: 'Pedimento', - templateUrl: '/csv/EstructuraCatPedimentos.xls' - }, -]; - -export const transportesConfig: CsvUploadItem[] = [ - { - id: 'transporters', - title: 'Transportistas', - icon: Ship, - modelTarget: 'Transporter', - templateUrl: '/csv/EstructuraCatTransportistas.xlsx', - }, - { - id: 'transports', - title: 'Transportes', - icon: Truck, - modelTarget: 'Transport', - templateUrl: '/csv/EstructuraCatTransportes.xls' - }, - { - id: 'drivers', - title: 'Conductores', - icon: User, - modelTarget: 'Driver', - templateUrl: '/csv/EstructuraCatConductor.xls' - }, - { - id: 'trailers', - title: 'Trailers y Cajas', - icon: Container, - modelTarget: 'Trailer', - templateUrl: '/csv/EstructuraCatTrailers.xls' - }, -]; - -export const importacionConfig: CsvUploadItem[] = [ - // Impo Temp - { - id: 'imp_temp_header', - title: 'Encabezado', - icon: FileText, - group: 'Impo. Temp.', - modelTarget: 'invoice_header', - templateUrl: '/csv/EstructuraEncFacImpoTemp.xls' - }, - { - id: 'imp_temp_details', - title: 'Partidas', - icon: Package, - group: 'Impo. Temp.', - modelTarget: 'invoice_details', - templateUrl: '/csv/EstructuraParFacImpoTempAF.xls' - }, - { - id: 'imp_temp_series', - title: 'Series', - icon: Hash, - group: 'Impo. Temp.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - // Impo Def - { - id: 'imp_def_header', - title: 'Encabezado', - icon: FileText, - group: 'Impo. Def.', - modelTarget: 'invoice_header', - templateUrl: '/csv/EstructuraEncFacImpoDef.xls' - }, - { - id: 'imp_def_details', - title: 'Partidas', - icon: Package, - group: 'Impo. Def.', - modelTarget: 'invoice_details', - templateUrl: '/csv/EstructuraParFacImpoDefAF.xls' - }, - { - id: 'imp_def_series', - title: 'Series', - icon: Hash, - group: 'Impo. Def.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - // Compras Mex - { - id: 'comp_mex_header', - title: 'Encabezado', - icon: FileText, - group: 'Compras Mex.', - modelTarget: 'invoice_header', - disabled: true, - }, - { - id: 'comp_mex_details', - title: 'Partidas', - icon: Package, - group: 'Compras Mex.', - modelTarget: 'invoice_details', - disabled: true, - }, - { - id: 'comp_mex_series', - title: 'Series', - icon: Hash, - group: 'Compras Mex.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, -]; - -export const exportacionConfig: CsvUploadItem[] = [ - // Expo Def / Cam. Reg. - { - id: 'exp_def_header', - title: 'Encabezado', - icon: FileText, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'invoice_header', - templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls' - }, - { - id: 'exp_def_details', - title: 'Partidas', - icon: Package, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'invoice_details', - templateUrl: '/csv/EstructuraParExpoCamReg.xls' - }, - { - id: 'exp_def_series', - title: 'Series', - icon: Hash, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - { - id: 'exp_def_nodes', - title: 'NODES', - icon: Briefcase, - group: 'Expo. Def./Cam. Reg.', - modelTarget: 'Nodes', - disabled: true, - }, - // Expo Rep - { - id: 'exp_rep_header', - title: 'Encabezado', - icon: FileText, - group: 'Expo. Rep.', - modelTarget: 'InvoiceHeader', - disabled: true, - }, - { - id: 'exp_rep_details', - title: 'Partidas', - icon: Package, - group: 'Expo. Rep.', - modelTarget: 'InvoiceSalesDetails', - disabled: true, - }, - { - id: 'exp_rep_series', - title: 'Series', - icon: Hash, - group: 'Expo. Rep.', - modelTarget: 'InvoiceSeries', - disabled: true, - }, - // Manifiesto - { - id: 'manifest_header', - title: 'Encabezado', - icon: FileText, - group: 'Manifiesto', - modelTarget: 'Manifest', - disabled: true, - }, -]; +import { + User, + Users, + FileText, + Truck, + Container, + Ship, + Plane, + Package, + Briefcase, + Globe, + CreditCard, + DollarSign, + Calendar, + Hash, + MapPin, + ShieldCheck, + FileDigit, + Scale, +} from 'lucide-svelte'; + +// --- Interfaces --- + +export interface CsvUploadItem { + id: string; + title: string; + icon: any; + group?: string; // For grouping within a tab + modelTarget?: string; // The backend model this maps to + description?: string; + templateUrl?: string; // Path to the template file in static/ + disabled?: boolean; // New property to mark items as "Coming Soon" +} + +export interface CsvUploadField { + name: string; + label: string; + type: 'text' | 'select' | 'boolean' | 'date' | 'radio'; + options?: { label: string; value: string | boolean | number }[]; + required?: boolean; + defaultValue?: any; +} + +// Map of Tab ID -> Array of Fields +export const tabSettings: Record = { + catalogos: [ + { + name: 'mode', + label: 'Modo de Carga', + type: 'radio', + options: [ + { label: 'Actualizar', value: 'update' }, + { label: 'Reemplazar', value: 'replace' } + ], + defaultValue: 'update' + } + ], + transportes: [ + { + name: 'mode', + label: 'Modo de Carga', + type: 'radio', + options: [ + { label: 'Actualizar', value: 'update' }, + { label: 'Reemplazar', value: 'replace' } + ], + defaultValue: 'update' + } + ], + importacion: [ + { + name: 'autonumber_remesas', + label: 'Autonumerar Remesas', + type: 'boolean', + defaultValue: false + }, + { + name: 'recalculate_dates', + label: 'Recalcular Fechas', + type: 'boolean', + defaultValue: false + }, + { + name: 'dateFormat', + label: 'Formato de Fecha', + type: 'select', + options: [ + { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, + { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, + { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } + ], + defaultValue: 'dd/mm/yyyy' + } + ], + exportacion: [ + { + name: 'invoice_type', + label: 'Tipo de Factura', + type: 'select', + options: [ + { label: 'AFIJO', value: 'AFIJO' }, + { label: 'NORMAL', value: 'NORMAL' }, + ], + defaultValue: 'AFIJO', + }, + { + name: 'is_regime_change', + label: 'Es Cambio de Régimen', + type: 'boolean', + defaultValue: false, + }, + { + name: 'dateFormat', + label: 'Formato de Fecha', + type: 'select', + options: [ + { label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' }, + { label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' }, + { label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' } + ], + defaultValue: 'dd/mm/yyyy' + } + ] +}; + +// --- DATA DEFINITIONS (Items only, no config) --- + +export const catalogosConfig: CsvUploadItem[] = [ + { + id: 'customs_brokers', + title: 'Agentes Aduanales', + icon: User, + modelTarget: 'CustomsBroker', + templateUrl: '/csv/EstructuraCatAgenteAduanal.xls' + }, + { + id: 'clients_providers', + title: 'Clientes y Proveedores', + icon: Users, + modelTarget: 'ClientProvider', + templateUrl: '/csv/EstructuraCatClienteProv.xls' + }, + { + id: 'exchange_rates', + title: 'Tipo de Cambios', + icon: DollarSign, + modelTarget: 'ExchangeRate', + templateUrl: '/csv/EstructuraCatTiposCambio.xls' + }, + { + id: 'american_fractions', + title: 'Fracc. Ame.', + icon: Globe, + modelTarget: 'AmericanFraction', + templateUrl: '/csv/EstructuraCatFraccAme.xls' + }, + { + id: 'material_classes', + title: 'Clases de Materiales', + icon: Package, + modelTarget: 'MaterialClass', + templateUrl: '/csv/EstructuraCatClasesAF.xls' + }, + { + id: 'part_numbers', + title: 'Números de parte', + icon: Hash, + modelTarget: 'Part', + templateUrl: '/csv/EstructuraCatPartesAF.xls', + }, + { + id: 'boms', + title: 'BOMs', + icon: Briefcase, + modelTarget: 'Bom', + templateUrl: '/csv/EstructuraBOMS.xlsx', + }, + { + id: 'items', + title: 'Partidas (Permisos)', + icon: FileText, + group: 'Permisos', + modelTarget: 'ItemPermission', + templateUrl: '/csv/EstructuraCatPartesAF.xls' + }, + { + id: 'headers', + title: 'Encabezados (Permisos)', + icon: FileText, + group: 'Permisos', + modelTarget: 'HeaderPermission', + disabled: true, + }, + { + id: 'historical_fractions', + title: 'Fracciones Históricas', + icon: Calendar, + modelTarget: 'HistoricalFraction', + disabled: true, + }, + { + id: 'pedimentos', + title: 'Pedimentos', + icon: FileDigit, + modelTarget: 'Pedimento', + templateUrl: '/csv/EstructuraCatPedimentos.xls' + }, +]; + +export const transportesConfig: CsvUploadItem[] = [ + { + id: 'transporters', + title: 'Transportistas', + icon: Ship, + modelTarget: 'Transporter', + templateUrl: '/csv/EstructuraCatTransportistas.xlsx', + }, + { + id: 'transports', + title: 'Transportes', + icon: Truck, + modelTarget: 'Transport', + templateUrl: '/csv/EstructuraCatTransportes.xls' + }, + { + id: 'drivers', + title: 'Conductores', + icon: User, + modelTarget: 'Driver', + templateUrl: '/csv/EstructuraCatConductor.xls' + }, + { + id: 'trailers', + title: 'Trailers y Cajas', + icon: Container, + modelTarget: 'Trailer', + templateUrl: '/csv/EstructuraCatTrailers.xls' + }, +]; + +export const importacionConfig: CsvUploadItem[] = [ + // Impo Temp + { + id: 'imp_temp_header', + title: 'Encabezado', + icon: FileText, + group: 'Impo. Temp.', + modelTarget: 'invoice_header', + templateUrl: '/csv/EstructuraEncFacImpoTemp.xls' + }, + { + id: 'imp_temp_details', + title: 'Partidas', + icon: Package, + group: 'Impo. Temp.', + modelTarget: 'invoice_details', + templateUrl: '/csv/EstructuraParFacImpoTempAF.xls' + }, + { + id: 'imp_temp_series', + title: 'Series', + icon: Hash, + group: 'Impo. Temp.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + // Impo Def + { + id: 'imp_def_header', + title: 'Encabezado', + icon: FileText, + group: 'Impo. Def.', + modelTarget: 'invoice_header', + templateUrl: '/csv/EstructuraEncFacImpoDef.xls' + }, + { + id: 'imp_def_details', + title: 'Partidas', + icon: Package, + group: 'Impo. Def.', + modelTarget: 'invoice_details', + templateUrl: '/csv/EstructuraParFacImpoDefAF.xls' + }, + { + id: 'imp_def_series', + title: 'Series', + icon: Hash, + group: 'Impo. Def.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + // Compras Mex + { + id: 'comp_mex_header', + title: 'Encabezado', + icon: FileText, + group: 'Compras Mex.', + modelTarget: 'invoice_header', + disabled: true, + }, + { + id: 'comp_mex_details', + title: 'Partidas', + icon: Package, + group: 'Compras Mex.', + modelTarget: 'invoice_details', + disabled: true, + }, + { + id: 'comp_mex_series', + title: 'Series', + icon: Hash, + group: 'Compras Mex.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, +]; + +export const exportacionConfig: CsvUploadItem[] = [ + // Expo Def / Cam. Reg. + { + id: 'exp_def_header', + title: 'Encabezado', + icon: FileText, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'invoice_header', + templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls' + }, + { + id: 'exp_def_details', + title: 'Partidas', + icon: Package, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'invoice_details', + templateUrl: '/csv/EstructuraParExpoCamReg.xls' + }, + { + id: 'exp_def_series', + title: 'Series', + icon: Hash, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + { + id: 'exp_def_nodes', + title: 'NODES', + icon: Briefcase, + group: 'Expo. Def./Cam. Reg.', + modelTarget: 'Nodes', + disabled: true, + }, + // Expo Rep + { + id: 'exp_rep_header', + title: 'Encabezado', + icon: FileText, + group: 'Expo. Rep.', + modelTarget: 'InvoiceHeader', + disabled: true, + }, + { + id: 'exp_rep_details', + title: 'Partidas', + icon: Package, + group: 'Expo. Rep.', + modelTarget: 'InvoiceSalesDetails', + disabled: true, + }, + { + id: 'exp_rep_series', + title: 'Series', + icon: Hash, + group: 'Expo. Rep.', + modelTarget: 'InvoiceSeries', + disabled: true, + }, + // Manifiesto + { + id: 'manifest_header', + title: 'Encabezado', + icon: FileText, + group: 'Manifiesto', + modelTarget: 'Manifest', + disabled: true, + }, +]; From 9da84990a8925bed4cedc40de8790c39c740739c Mon Sep 17 00:00:00 2001 From: hreyes Date: Tue, 3 Mar 2026 09:54:12 -0700 Subject: [PATCH 16/16] feature/BOMs-csv-WIP --- backend/api/v1/modules/a76/boms/__init__.py | 7 + .../v1/modules/a76/boms/imports/__init__.py | 1 + .../api/v1/modules/a76/boms/imports/routes.py | 151 ++++++ .../v1/modules/a76/boms/imports/schemas.py | 22 + .../api/v1/modules/a76/boms/imports/tasks.py | 430 ++++++++++++++++++ .../a76/boms/imports/template_config.py | 42 ++ backend/api/v1/modules/a76/boms/routes.py | 13 + backend/api/v1/modules/a76/router.py | 2 + backend/core/celery_app.py | 1 + 9 files changed, 669 insertions(+) create mode 100644 backend/api/v1/modules/a76/boms/__init__.py create mode 100644 backend/api/v1/modules/a76/boms/imports/__init__.py create mode 100644 backend/api/v1/modules/a76/boms/imports/routes.py create mode 100644 backend/api/v1/modules/a76/boms/imports/schemas.py create mode 100644 backend/api/v1/modules/a76/boms/imports/tasks.py create mode 100644 backend/api/v1/modules/a76/boms/imports/template_config.py create mode 100644 backend/api/v1/modules/a76/boms/routes.py diff --git a/backend/api/v1/modules/a76/boms/__init__.py b/backend/api/v1/modules/a76/boms/__init__.py new file mode 100644 index 00000000..ea1f7540 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/__init__.py @@ -0,0 +1,7 @@ +""" +Módulo de importación CSV para BOMs (Bills of Materials). +""" + +from .routes import router + +__all__ = ["router"] diff --git a/backend/api/v1/modules/a76/boms/imports/__init__.py b/backend/api/v1/modules/a76/boms/imports/__init__.py new file mode 100644 index 00000000..7585c5fe --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/__init__.py @@ -0,0 +1 @@ +# boms.imports diff --git a/backend/api/v1/modules/a76/boms/imports/routes.py b/backend/api/v1/modules/a76/boms/imports/routes.py new file mode 100644 index 00000000..4a87ed0e --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/routes.py @@ -0,0 +1,151 @@ +""" +Rutas de importación CSV para BOMs. +Flujo: upload → scan → status (polling) → commit. +""" +import base64 +import json +import logging +import os +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends +from sqlalchemy.orm import Session +from typing import Dict, Any + +from core.celery_app import celery_app +from core.database import get_core_db +from core.security import get_current_user, validate_access_to_resource + +from .schemas import ImportJobResponse +from .tasks import ( + scan_file, + insert_valid_rows, + BOM_IMPORT_FILE_PREFIX, + BOM_IMPORT_META_PREFIX, + BOM_IMPORT_REDIS_TTL, +) + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +@router.post("/upload", response_model=ImportJobResponse) +async def upload_import_file( + file: UploadFile = File(...), + company_id: int = Query(..., description="Company ID"), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + try: + tenant_id = validate_access_to_resource(db, company_id, current_user) + except Exception as e: + logger.error(f"BOMs import: access validation failed: {e}") + raise HTTPException(status_code=403, detail="Invalid company access") + + if not file.filename or not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv") + + job_id = str(uuid4()) + contents = await file.read() + + meta_data = { + "tenant_id": tenant_id, + "company_id": company_id, + "user_id": current_user.get("id"), + "template_id": "boms", + } + + try: + r = _get_redis() + r.set( + f"{BOM_IMPORT_FILE_PREFIX}{job_id}", + base64.b64encode(contents), + ex=BOM_IMPORT_REDIS_TTL, + ) + r.set( + f"{BOM_IMPORT_META_PREFIX}{job_id}", + json.dumps(meta_data).encode("utf-8"), + ex=BOM_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.error(f"BOMs import: Redis store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") + + try: + upload_dir = os.path.join(os.getcwd(), "uploads", "temp") + os.makedirs(upload_dir, exist_ok=True) + with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f: + f.write(contents) + with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f: + json.dump(meta_data, f) + except Exception as e: + logger.warning(f"BOMs import: local file save failed: {e}") + + scan_file.apply_async(args=[job_id], task_id=job_id) + + return ImportJobResponse( + job_id=job_id, + status="queued", + message="Archivo subido. Escaneo iniciado.", + ) + + +@router.get("/{job_id}/status") +async def get_import_status(job_id: str): + task_result = celery_app.AsyncResult(job_id) + + if task_result.state == "PENDING": + return {"status": "processing", "progress": 0} + if task_result.state == "PROGRESS": + info = (task_result.info or {}) + return { + "status": "processing", + "progress": info.get("current", 0), + "total": info.get("total", 0), + } + if task_result.state == "SUCCESS": + result = task_result.result + if isinstance(result, dict) and "status" in result: + return result + return {"status": "finished", "result": result} + + result = getattr(task_result, "result", None) + if isinstance(result, dict) and result.get("status") in ("finished", "warning"): + return result + + logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state) + err_msg = None + tb = getattr(task_result, "traceback", None) + if tb and isinstance(tb, str): + lines = [l.strip() for l in tb.strip().split("\n") if l.strip()] + if lines: + err_msg = lines[-1] + if not err_msg: + try: + exc = task_result.get(propagate=False) + if exc is not None: + err_msg = str(exc) + except Exception: + pass + if not err_msg and result is not None: + if not isinstance(result, dict): + err_msg = str(result) + elif result.get("error") or result.get("message"): + err_msg = result.get("error") or result.get("message") + return {"status": "failed", "error": err_msg or "Task failed"} + + +@router.post("/{job_id}/commit") +async def commit_import_job(job_id: str): + task = insert_valid_rows.delay(job_id) + return { + "status": "committing", + "message": "Inserción iniciada.", + "commit_job_id": task.id, + } diff --git a/backend/api/v1/modules/a76/boms/imports/schemas.py b/backend/api/v1/modules/a76/boms/imports/schemas.py new file mode 100644 index 00000000..7827dae6 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/schemas.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional + + +class ImportJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class ImportJobStatus(BaseModel): + status: str + job_id: str + total_rows: Optional[int] = 0 + error_count: Optional[int] = 0 + valid_rows: Optional[int] = 0 + error: Optional[str] = None + inserted: Optional[int] = 0 + skipped_invalid: Optional[int] = 0 + skipped_missing_fk: Optional[int] = 0 + skipped_duplicate: Optional[int] = 0 + skipped_details: Optional[list] = None diff --git a/backend/api/v1/modules/a76/boms/imports/tasks.py b/backend/api/v1/modules/a76/boms/imports/tasks.py new file mode 100644 index 00000000..ac72ad0d --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/tasks.py @@ -0,0 +1,430 @@ +""" +Tareas Celery para importación CSV de BOMs. +Flujo: scan_file (validación) → insert_valid_rows (commit). +Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista. +""" +import os +import base64 +import csv +import json +import logging +import re +import unicodedata +from decimal import Decimal, InvalidOperation +from typing import Dict, Any, Optional, List, Set + +from core.celery_app import celery_app +from core.database import CoreSessionLocal + +from .template_config import row_from_template + +logger = logging.getLogger(__name__) + +BOM_IMPORT_FILE_PREFIX = "bom_import_file:" +BOM_IMPORT_META_PREFIX = "bom_import_meta:" +BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:" +BOM_IMPORT_REDIS_TTL = 3600 + + +def _get_redis(): + import redis + url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) + return redis.Redis.from_url(url, decode_responses=False) + + +def _worker_upload_dir() -> str: + return os.path.join(os.getcwd(), "uploads", "temp") + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + r = _get_redis() + data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}") + if not data: + return None + try: + raw = base64.b64decode(data) + except Exception as e: + logger.warning(f"BOMs import: failed to decode file from Redis: {e}") + return None + upload_dir = _worker_upload_dir() + os.makedirs(upload_dir, exist_ok=True) + file_path = os.path.join(upload_dir, f"bom_{job_id}.csv") + with open(file_path, "wb") as f: + f.write(raw) + return file_path + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + r = _get_redis() + data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}") + if not data: + return False + try: + meta = json.loads(data.decode("utf-8")) + except Exception as e: + logger.warning(f"BOMs import: failed to decode meta from Redis: {e}") + return False + meta_path = file_path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f) + return True + + +def _delete_import_from_redis(job_id: str) -> None: + try: + r = _get_redis() + r.delete( + f"{BOM_IMPORT_FILE_PREFIX}{job_id}", + f"{BOM_IMPORT_META_PREFIX}{job_id}", + f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", + ) + except Exception as e: + logger.warning(f"BOMs import: failed to delete Redis keys: {e}") + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def _validate_row_bom( + row: Dict[str, Any], + line_num: int, + valid_part_numbers: Optional[Set[str]] = None, +) -> Optional[Dict[str, Any]]: + """Valida una fila BOM según columnas del template. FK opcional a a76.parts.""" + parent = (row.get("NUMPARTE_PADRE") or "").strip() + if not parent: + return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"} + if len(parent) > 70: + return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"} + + component = (row.get("NUMPARTE_COMPONENTE") or "").strip() + if not component: + return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"} + if len(component) > 70: + return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"} + + qty = row.get("CANTIDAD") + if qty is None or qty == "": + return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"} + try: + val = Decimal(str(qty)) + if val < 0: + return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"} + except (InvalidOperation, ValueError, TypeError): + return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"} + + uom = (row.get("UNIMED") or "").strip() + if uom and len(uom) > 10: + return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"} + + def _optional_number(val: Any) -> bool: + if val is None: + return True + s = re.sub(r"\s+", "", str(val).strip()) + if not s: + return True + try: + float(s.replace(",", ".")) + return True + except (ValueError, TypeError): + return False + + version_bom = row.get("VERSION_BOM") + if not _optional_number(version_bom): + return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"} + + version_bill = row.get("VERSION_BILL") + if not _optional_number(version_bill): + return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"} + + # Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas) + if valid_part_numbers is not None and len(valid_part_numbers) > 0: + if parent not in valid_part_numbers: + return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"} + if component not in valid_part_numbers: + return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"} + + return None + + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, config: str = None): + logger.info(f"BOMs import: starting scan for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "") + os.makedirs(error_dir, exist_ok=True) + error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") + + total_rows = 0 + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + total_rows = sum(1 for _ in f) - 1 + except Exception as e: + return {"status": "failed", "error": str(e)} + + meta_path = file_path.replace(".csv", ".meta.json") + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + except Exception as e: + logger.warning(f"BOMs import: failed to read meta: {e}") + + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_part_numbers: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.parts.models import Part + for p in ( + session.query(Part.part_number) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + valid_part_numbers.add(p[0]) + except Exception as e: + logger.warning(f"BOMs import: could not load parts for FK validation: {e}") + + error_count = 0 + processed_rows = 0 + errors_detail: List[Dict[str, Any]] = [] + + try: + with open(file_path, "r", encoding="utf-8-sig") as f_in, open( + error_path, "w", encoding="utf-8" + ) as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i % 500 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + error_count += 1 + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append( + {"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")} + ) + processed_rows += 1 + + except Exception as e: + logger.error(f"BOMs import scan failed: {e}") + return {"status": "failed", "error": str(e)} + + error_lines_list = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + except Exception: + pass + if error_lines_list: + r = _get_redis() + r.set( + f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}", + json.dumps(error_lines_list).encode("utf-8"), + ex=BOM_IMPORT_REDIS_TTL, + ) + except Exception as e: + logger.warning(f"BOMs import: failed to store error lines in Redis: {e}") + + return { + "status": "waiting_confirmation", + "job_id": job_id, + "total_rows": processed_rows, + "error_count": error_count, + "valid_rows": processed_rows - error_count, + "errors": errors_detail, + } + + +def _decimal_or_none(val: Any) -> Optional[Decimal]: + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError, TypeError): + return None + + +def _int_or_none(val: Any) -> Optional[int]: + if val is None or val == "": + return None + try: + return int(val) + except (ValueError, TypeError): + return None + + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str): + logger.info(f"BOMs import: starting commit for job {job_id}") + + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv") + if not os.path.exists(alt_path): + return { + "status": "failed", + "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.", + } + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + base_dir = os.path.dirname(file_path) + error_dir = base_dir.replace("temp", "errors") + error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl") + + error_lines = set() + try: + r = _get_redis() + raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}") + if raw: + error_lines = set(json.loads(raw.decode("utf-8"))) + except Exception as e: + logger.debug(f"BOMs import: could not load error lines from Redis: {e}") + if not error_lines and os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + error_lines.add(err["line"]) + except Exception: + pass + + meta_path = file_path.replace(".csv", ".meta.json") + tenant_id = None + company_id = None + meta = {} + if os.path.exists(meta_path): + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) or {} + tenant_id = meta.get("tenant_id") + company_id = meta.get("company_id") + except Exception: + pass + + if not tenant_id or not company_id: + return {"status": "failed", "error": "Falta contexto (tenant/company)"} + + valid_part_numbers: Set[str] = set() + try: + with CoreSessionLocal() as session: + from api.v1.modules.a76.parts.models import Part + for p in ( + session.query(Part.part_number) + .filter( + Part.tenant_id == tenant_id, + Part.company_id == company_id, + ) + .all() + ): + valid_part_numbers.add(p[0]) + except Exception as e: + logger.warning(f"BOMs import: could not load parts: {e}") + + inserted_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + valid_count = 0 + + try: + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, normalize_header) + err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers) + if err: + skipped_invalid += 1 + skipped_details.append( + {"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"} + ) + continue + + valid_count += 1 + # Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas. + # Cuando exista la tabla de destino, aquí se hará insert/update. + + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": skipped_details, + } + if valid_count > 0 and inserted_count == 0: + response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada." + + except Exception as e: + logger.error(f"BOMs import task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + response = { + "status": "failed", + "error": str(e), + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": skipped_details, + } + + try: + if file_path and os.path.exists(file_path): + os.remove(file_path) + if os.path.exists(error_path): + os.remove(error_path) + if os.path.exists(meta_path): + os.remove(meta_path) + _delete_import_from_redis(job_id) + except Exception as cleanup_err: + logger.warning(f"BOMs import cleanup failed: {cleanup_err}") + + return response diff --git a/backend/api/v1/modules/a76/boms/imports/template_config.py b/backend/api/v1/modules/a76/boms/imports/template_config.py new file mode 100644 index 00000000..1d4efa48 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/imports/template_config.py @@ -0,0 +1,42 @@ +""" +Configuración de plantilla CSV para BOMs (EstructuraBOMS.xlsx). +Placeholders hasta tener el XLS definitivo; ajustar canónicos y aliases según el archivo. +""" + +from typing import Dict, List, Any + +TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { + "boms": [ + {"canonical": "NUMPARTE_PADRE", "aliases": ["PARTE PADRE", "PART NUMBER", "PARENT PART", "NUM PARTE PADRE"]}, + {"canonical": "NUMPARTE_COMPONENTE", "aliases": ["PARTE COMPONENTE", "COMPONENT PART", "NUM PARTE COMPONENTE"]}, + {"canonical": "CANTIDAD", "aliases": ["QTY", "QUANTITY", "CANT"]}, + {"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]}, + {"canonical": "VERSION_BOM", "aliases": ["VERSION BOM", "BOM VERSION", "VERSIONBOM"]}, + {"canonical": "VERSION_BILL", "aliases": ["VERSION BILL", "BILL VERSION", "VERSIONBILL"]}, + ], +} + + +def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]: + cols = TEMPLATE_COLUMNS.get("boms") + if not cols: + return {} + lookup: Dict[str, str] = {} + for item in cols: + canonical = item["canonical"] + lookup[normalize_header_fn(canonical)] = canonical + for alias in item.get("aliases") or []: + lookup[normalize_header_fn(alias)] = canonical + return lookup + + +def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]: + lookup = build_normalized_lookup(normalize_header_fn) + if not lookup: + return {normalize_header_fn(k): v for k, v in row.items()} + out: Dict[str, Any] = {} + for csv_header, value in row.items(): + key_norm = normalize_header_fn(csv_header) + if key_norm in lookup: + out[lookup[key_norm]] = value + return out diff --git a/backend/api/v1/modules/a76/boms/routes.py b/backend/api/v1/modules/a76/boms/routes.py new file mode 100644 index 00000000..ddd64d15 --- /dev/null +++ b/backend/api/v1/modules/a76/boms/routes.py @@ -0,0 +1,13 @@ +""" +Endpoints para importación CSV de BOMs. +Mismo patrón que parts y classes: upload → scan → status → commit. +""" + +from fastapi import APIRouter + +from .imports.routes import router as imports_router + +router = APIRouter() + +# CSV import (upload → scan → status → commit) +router.include_router(imports_router, prefix="/imports", tags=["a76 / boms / csv_import"]) diff --git a/backend/api/v1/modules/a76/router.py b/backend/api/v1/modules/a76/router.py index 5f5d23bd..d7057536 100644 --- a/backend/api/v1/modules/a76/router.py +++ b/backend/api/v1/modules/a76/router.py @@ -23,6 +23,7 @@ from .transportation.drivers.routes import router as drivers_router from .doc_types_dig.routes import router as doc_types_dig_router from .fraction_rule_octave.routes import router as fraction_rule_octave_router from .parts import router as parts_router +from .boms import router as boms_router from .pedmientos.router import router as pedimentos_router from .permission_rule_oct.routes import router as permission_rule_oct_router from .transportation.trailers.routes import router as trailers_router @@ -59,6 +60,7 @@ router.include_router(pedimentos_router, prefix="/a76") router.include_router(client_and_provider_router, prefix="/a76", tags=["a76 / clients_and_providers"]) router.include_router(classes_router, prefix="/a76/classes", tags=["a76 / classes"]) router.include_router(parts_router, prefix="/a76", tags=["a76 / parts"]) +router.include_router(boms_router, prefix="/a76/boms", tags=["a76 / boms"]) router.include_router(permission_rule_oct_router, prefix="/a76", tags=["a76 / permission_rule_oct"]) router.include_router(fraction_rule_octave_router, prefix="/a76", tags=["a76 / fraction_rule_octave"]) router.include_router(country_rule_oct_router, prefix="/a76", tags=["a76 / country_rule_oct"]) diff --git a/backend/core/celery_app.py b/backend/core/celery_app.py index a856f339..5f074b4a 100644 --- a/backend/core/celery_app.py +++ b/backend/core/celery_app.py @@ -40,6 +40,7 @@ celery_app.conf.update( "api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks", "api.v1.modules.a76.classes.imports.tasks", "api.v1.modules.a76.parts.imports.tasks", + "api.v1.modules.a76.boms.imports.tasks", "api.v1.modules.a76.transportation.vehicles.imports.tasks", "api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task", "api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task",