""" Tareas Celery para importación CSV de Pedimentos. Flujo: scan_file (validación) → insert_valid_rows (commit). Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader), fk_loader, validators, mappers. Create/update delegan en PedimentosService; defaults de fechas (pedimento_dates) y merge vs replace están alineados con el servicio para paridad con el flujo API. """ import json import logging import os from datetime import datetime from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal from ..common import storage as common_storage from ..common import normalize as common_normalize from ..common import meta as common_meta from ..common import responses as common_responses from ..common import csv_reader as common_csv_reader from .template_config import ( row_from_template, is_clarion_layout, parse_pedimento_col_a, detect_headers_or_data, ) from .validators import validate_row_pedimento from .common.mappers import row_to_pedimento_data, row_to_pedimento_data_merge_existing from .common.fk_loader import load_pedimentos_fk_sets, pedimento_key_from_parsed logger = logging.getLogger(__name__) JOB_TYPE = "ped" TEMPLATE_ID = "pedimentos" # Para routes.py PED_IMPORT_FILE_PREFIX = "ped_import_file:" PED_IMPORT_META_PREFIX = "ped_import_meta:" PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" PED_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _read_plan_for_pedimentos(fieldnames): if fieldnames: return common_csv_reader.CsvReadPlan(header_mode="headerless", fieldnames=fieldnames) return common_csv_reader.CsvReadPlan(header_mode="header") def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} if os.path.getsize(file_path) == 0: return {"status": "failed", "error": "El archivo está vacío. Verifica que el CSV tenga contenido."} common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) fieldnames, has_header = detect_headers_or_data( file_path, common_normalize.normalize_header, parse_pedimento_col_a, ) read_plan = _read_plan_for_pedimentos(fieldnames) try: total_rows = common_csv_reader.count_csv_rows(file_path, has_header=has_header, read_plan=read_plan) except Exception as e: return {"status": "failed", "error": str(e)} try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) or {} actualizar = meta.get("actualizar", False) date_format_preference = meta.get("dateFormat") or meta.get("date_format") try: with CoreSessionLocal() as session: ( valid_client_ids, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, short_name_to_id, ) = load_pedimentos_fk_sets(session, tenant_id, company_id) except Exception as e: logger.error("Pedimentos import: failed to load FK sets: %s", e) return {"status": "failed", "error": "No se pudo cargar catálogos"} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows_with_plan(file_path, read_plan=read_plan): if progress_callback: progress_callback(i, total_rows, error_count) row_norm = _norm_row(row) row_errors = validate_row_pedimento( row_norm, i, short_name_to_id, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, actualizar=actualizar, raw_row=row, date_format_preference=date_format_preference, ) if row_errors: has_blocking_error = any(not e.get("warning", False) for e in row_errors) if has_blocking_error: error_count += 1 error_lines_list.append(i) for e in row_errors: if not e.get("warning", False): f_err.write(json.dumps(e) + "\n") if len(errors_detail) < 500: for e in row_errors: if len(errors_detail) >= 500: break errors_detail.append({ "line": e["line"], "col": e.get("col", ""), "msg": e.get("msg", ""), "solution": e.get("solution", ""), "warning": bool(e.get("warning", False)), }) processed_rows += 1 if error_lines_list: common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.error("Pedimentos import scan failed: %s", e) return {"status": "failed", "error": str(e)} return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): logger.info("Pedimentos import: starting scan for job %s", job_id) def on_progress(current: int, total: int, errors: int) -> None: self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) return _do_scan(job_id, progress_callback=on_progress) def _do_commit(job_id: str) -> Dict[str, Any]: file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") if not file_path: alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) or {} actualizar = meta.get("actualizar", False) date_format_preference = meta.get("dateFormat") or meta.get("date_format") try: with CoreSessionLocal() as session: ( valid_client_ids, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, short_name_to_id, ) = load_pedimentos_fk_sets(session, tenant_id, company_id) except Exception as e: logger.error("Pedimentos import: failed to load FK sets: %s", e) return {"status": "failed", "error": "No se pudo cargar catálogos"} from api.v1.modules.a76.pedmientos.dtos.pedimentos import ( PedimentosCreate, PedimentosUpdate, ) from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos inserted_count = 0 updated_count = 0 skipped_invalid = 0 skipped_missing_fk = 0 skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] meta_path = common_meta.get_meta_path(file_path) fieldnames_commit, _ = detect_headers_or_data( file_path, common_normalize.normalize_header, parse_pedimento_col_a, ) read_plan_commit = _read_plan_for_pedimentos(fieldnames_commit) def _key_from_row(r: Dict[str, Any]) -> Optional[str]: if is_clarion_layout(r): año = (r.get("AÑO") or "").strip() patente = (r.get("PATENTE") or "").strip() numero = (r.get("NUMERO") or "").strip() if año and patente and numero: adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] return pedimento_key_from_parsed(año[:2], adu, patente[:4], numero[:7]) parsed = parse_pedimento_col_a((r.get("PEDIMENTO") or "").strip()) if not parsed: return None y, lic, num = parsed adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] return pedimento_key_from_parsed(y, adu, lic, num) y = (r.get("AÑO") or "").strip()[:2] adu = (r.get("ADUANA_SECCION_CRUCE") or r.get("ADUANA") or "").strip()[:3] lic = (r.get("PATENTE") or "").strip()[:4] num = (r.get("NUMERO") or "").strip()[:7] return pedimento_key_from_parsed(y, adu, lic, num) try: with CoreSessionLocal() as session: for i, row in common_csv_reader.iter_csv_rows_with_plan(file_path, read_plan=read_plan_commit): if i in error_lines: continue row_norm = _norm_row(row) row_errors = validate_row_pedimento( row_norm, i, short_name_to_id, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, actualizar=actualizar, raw_row=row, date_format_preference=date_format_preference, ) if row_errors and any(not e.get("warning", False) for e in row_errors): skipped_invalid += 1 blocking_errors = [e for e in row_errors if not e.get("warning", False)] for blocking in blocking_errors: skipped_details.append({ "line": i, "reason": f"{blocking.get('col', '')}: {blocking.get('msg', '')}", "solution": blocking.get("solution", ""), }) continue key = _key_from_row(row_norm) try: # Si el pedimento ya existe: actualizar (merge) o reemplazar (crear). Si no existe: crear. if key and key in existing_pedimento_keys: existing = ( session.query(Pedimentos) .filter( Pedimentos.tenant_id == tenant_id, Pedimentos.company_id == company_id, Pedimentos.year == key.split("|")[0], Pedimentos.customs_office == key.split("|")[1], Pedimentos.license == key.split("|")[2], Pedimentos.pedimento_number == key.split("|")[3], ) .first() ) if existing: if actualizar: # Modo actualizar: merge con existente existing_dict = { "year": existing.year, "customs_office": existing.customs_office, "license": existing.license, "pedimento_number": existing.pedimento_number, "client_id": existing.client_id, "pedimento_code": existing.pedimento_code, "regime": existing.regime, "operation_type": existing.operation_type, "pedimento_type": existing.pedimento_type, "status": existing.status, "usd_value": existing.usd_value, "paid_price": existing.paid_price, "gross_weight": existing.gross_weight, "exchange_rate": existing.exchange_rate, "observations": existing.observations, } if existing.pedimento_dates: existing_dict["pedimento_dates"] = { "entry_date": existing.pedimento_dates.entry_date, "end_date": existing.pedimento_dates.end_date, "start_date": getattr( existing.pedimento_dates, "start_date", existing.pedimento_dates.entry_date ), "payment_date": getattr( existing.pedimento_dates, "payment_date", existing.pedimento_dates.entry_date ), } data = row_to_pedimento_data_merge_existing( row_norm, existing_dict, short_name_to_id, date_format_preference, ) else: # Modo crear: reemplazar con datos del CSV data = row_to_pedimento_data( row_norm, short_name_to_id, date_format_preference ) if "pedimento_dates" not in data or data.get("pedimento_dates") is None: data["pedimento_dates"] = PedimentoDatesCreate( entry_date=datetime.now(), end_date=datetime.now(), ) update_data = {k: v for k, v in data.items() if k != "pedimento_dates"} if data.get("pedimento_dates"): update_data["pedimento_dates"] = PedimentoDatesCreate( **data["pedimento_dates"] ) update_schema = PedimentosUpdate(**update_data) PedimentosService.update( session, existing.id, tenant_id, update_schema, company_id ) updated_count += 1 else: skipped_invalid += 1 skipped_details.append( { "line": i, "reason": "Pedimento no encontrado", "solution": "Verifica que el pedimento exista en el sistema usando las columnas PEDIMENTO (o AÑO/PATENTE/NUMERO) y ADUANA_SECCION_CRUCE; si aplica, cambia a modo creación.", } ) else: # No existe: crear (ambos modos) data = row_to_pedimento_data( row_norm, short_name_to_id, date_format_preference ) if "pedimento_dates" not in data or data.get("pedimento_dates") is None: data["pedimento_dates"] = PedimentoDatesCreate( entry_date=datetime.now(), end_date=datetime.now(), ) create_data = PedimentosCreate(**data) PedimentosService.create(session, create_data, tenant_id, company_id) inserted_count += 1 except ValueError as ve: if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): skipped_duplicate += 1 skipped_details.append( { "line": i, "reason": str(ve), "solution": "Si deseas actualizar registros existentes, activa la opción de actualizar en la importación; si no, corrige el pedimento para evitar duplicados.", } ) else: skipped_invalid += 1 skipped_details.append( { "line": i, "reason": str(ve), "solution": "Revisa la línea del CSV y el valor que falló la validación; corrige el dato y vuelve a intentar.", } ) except Exception as e: logger.warning("Pedimentos import line %s: %s", i, e) skipped_invalid += 1 skipped_details.append( { "line": i, "reason": str(e), "solution": "Revisa la línea del CSV y los catálogos/fechas relacionados para resolver el error.", } ) except Exception as e: logger.exception("Pedimentos import task failed") return {"status": "failed", "error": str(e)} common_storage.cleanup_import_job( JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path, ) total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate critical_reference_gaps = skipped_invalid + skipped_missing_fk reference_state_ready = critical_reference_gaps == 0 if inserted_count == 0 and updated_count == 0 and total_skipped > 0: reasons = "; ".join( f"Línea {d.get('line', '?')}: {d.get('reason', '')}" for d in skipped_details[:5] ) if len(skipped_details) > 5: reasons += f" (+{len(skipped_details) - 5} más)" return { "status": "warning", "inserted": 0, "updated": 0, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, "critical_reference_gaps": critical_reference_gaps, "reference_state_ready": reference_state_ready, "message": f"No se insertaron registros. {total_skipped} rechazados. Motivos: {reasons}", } if inserted_count == 0 and updated_count == 0: return { "status": "failed", "error": "No hay registros válidos en el archivo CSV", "inserted": 0, "updated": 0, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, "critical_reference_gaps": critical_reference_gaps, "reference_state_ready": reference_state_ready, } return { "status": "finished", "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, "critical_reference_gaps": critical_reference_gaps, "reference_state_ready": reference_state_ready, } @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): logger.info("Pedimentos import: starting commit for job %s", job_id) return _do_commit(job_id)