""" Tareas Celery para importación CSV de Pedimentos. Flujo: scan_file (validación) → insert_valid_rows (commit). Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader), fk_loader, validators, mappers. """ import json import logging import os from datetime import datetime from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal from ..common import storage as common_storage from ..common import normalize as common_normalize from ..common import meta as common_meta from ..common import responses as common_responses from ..common import csv_reader as common_csv_reader from .template_config import ( row_from_template, is_clarion_layout, parse_pedimento_col_a, detect_headers_or_data, ) from .validators import validate_row_pedimento from .common.mappers import row_to_pedimento_data, row_to_pedimento_data_merge_existing from .common.fk_loader import load_pedimentos_fk_sets, pedimento_key_from_parsed logger = logging.getLogger(__name__) JOB_TYPE = "ped" TEMPLATE_ID = "pedimentos" # Para routes.py PED_IMPORT_FILE_PREFIX = "ped_import_file:" PED_IMPORT_META_PREFIX = "ped_import_meta:" PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:" PED_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]: return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID) def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} if os.path.getsize(file_path) == 0: return {"status": "failed", "error": "El archivo está vacío. Verifica que el CSV tenga contenido."} common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) fieldnames, has_header = detect_headers_or_data( file_path, common_normalize.normalize_header, parse_pedimento_col_a, ) try: total_rows = common_csv_reader.count_csv_rows(file_path, has_header=has_header) except Exception as e: return {"status": "failed", "error": str(e)} try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) or {} actualizar = meta.get("actualizar", False) date_format_preference = meta.get("dateFormat") or meta.get("date_format") try: with CoreSessionLocal() as session: ( valid_client_ids, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, short_name_to_id, ) = load_pedimentos_fk_sets(session, tenant_id, company_id) except Exception as e: logger.error("Pedimentos import: failed to load FK sets: %s", e) return {"status": "failed", "error": "No se pudo cargar catálogos"} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames): if progress_callback and i % 500 == 0: progress_callback(i, total_rows, error_count) row_norm = _norm_row(row) err = validate_row_pedimento( row_norm, i, short_name_to_id, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, actualizar=actualizar, raw_row=row, date_format_preference=date_format_preference, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: errors_detail.append({ "line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", ""), }) processed_rows += 1 if error_lines_list: common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.error("Pedimentos import scan failed: %s", e) return {"status": "failed", "error": str(e)} return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): logger.info("Pedimentos import: starting scan for job %s", job_id) def on_progress(current: int, total: int, errors: int) -> None: self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) return _do_scan(job_id, progress_callback=on_progress) def _do_commit(job_id: str) -> Dict[str, Any]: file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import") if not file_path: alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) or {} actualizar = meta.get("actualizar", False) date_format_preference = meta.get("dateFormat") or meta.get("date_format") try: with CoreSessionLocal() as session: ( valid_client_ids, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, short_name_to_id, ) = load_pedimentos_fk_sets(session, tenant_id, company_id) except Exception as e: logger.error("Pedimentos import: failed to load FK sets: %s", e) return {"status": "failed", "error": "No se pudo cargar catálogos"} from api.v1.modules.a76.pedmientos.dtos.pedimentos import ( PedimentosCreate, PedimentosUpdate, ) from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos inserted_count = 0 updated_count = 0 skipped_invalid = 0 skipped_missing_fk = 0 skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] meta_path = common_meta.get_meta_path(file_path) fieldnames_commit, _ = detect_headers_or_data( file_path, common_normalize.normalize_header, parse_pedimento_col_a, ) def _key_from_row(r: Dict[str, Any]) -> Optional[str]: if is_clarion_layout(r): año = (r.get("AÑO") or "").strip() patente = (r.get("PATENTE") or "").strip() numero = (r.get("NUMERO") or "").strip() if año and patente and numero: adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] return pedimento_key_from_parsed(año[:2], adu, patente[:4], numero[:7]) parsed = parse_pedimento_col_a((r.get("PEDIMENTO") or "").strip()) if not parsed: return None y, lic, num = parsed adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3] return pedimento_key_from_parsed(y, adu, lic, num) y = (r.get("AÑO") or "").strip()[:2] adu = (r.get("ADUANA_SECCION_CRUCE") or r.get("ADUANA") or "").strip()[:3] lic = (r.get("PATENTE") or "").strip()[:4] num = (r.get("NUMERO") or "").strip()[:7] return pedimento_key_from_parsed(y, adu, lic, num) try: with CoreSessionLocal() as session: for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames_commit): if i in error_lines: continue row_norm = _norm_row(row) err = validate_row_pedimento( row_norm, i, short_name_to_id, valid_regimes, valid_pedimento_codes, valid_clave_regimen_tipo, valid_aduana_seccion, existing_pedimento_keys, valid_anexo22_claves, valid_patentes, actualizar=actualizar, raw_row=row, date_format_preference=date_format_preference, ) if err: skipped_invalid += 1 skipped_details.append({ "line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}", }) continue key = _key_from_row(row_norm) try: # Si el pedimento ya existe: actualizar (merge) o reemplazar (crear). Si no existe: crear. if key and key in existing_pedimento_keys: existing = ( session.query(Pedimentos) .filter( Pedimentos.tenant_id == tenant_id, Pedimentos.company_id == company_id, Pedimentos.year == key.split("|")[0], Pedimentos.customs_office == key.split("|")[1], Pedimentos.license == key.split("|")[2], Pedimentos.pedimento_number == key.split("|")[3], ) .first() ) if existing: if actualizar: # Modo actualizar: merge con existente existing_dict = { "year": existing.year, "customs_office": existing.customs_office, "license": existing.license, "pedimento_number": existing.pedimento_number, "client_id": existing.client_id, "pedimento_code": existing.pedimento_code, "regime": existing.regime, "operation_type": existing.operation_type, "pedimento_type": existing.pedimento_type, "status": existing.status, "usd_value": existing.usd_value, "paid_price": existing.paid_price, "gross_weight": existing.gross_weight, "exchange_rate": existing.exchange_rate, "observations": existing.observations, } if existing.pedimento_dates: existing_dict["pedimento_dates"] = { "entry_date": existing.pedimento_dates.entry_date, "end_date": existing.pedimento_dates.end_date, "start_date": getattr( existing.pedimento_dates, "start_date", existing.pedimento_dates.entry_date ), "payment_date": getattr( existing.pedimento_dates, "payment_date", existing.pedimento_dates.entry_date ), } data = row_to_pedimento_data_merge_existing( row_norm, existing_dict, short_name_to_id, date_format_preference, ) else: # Modo crear: reemplazar con datos del CSV data = row_to_pedimento_data( row_norm, short_name_to_id, date_format_preference ) if "pedimento_dates" not in data or data.get("pedimento_dates") is None: data["pedimento_dates"] = PedimentoDatesCreate( entry_date=datetime.now(), end_date=datetime.now(), ) update_data = {k: v for k, v in data.items() if k != "pedimento_dates"} if data.get("pedimento_dates"): update_data["pedimento_dates"] = PedimentoDatesCreate( **data["pedimento_dates"] ) update_schema = PedimentosUpdate(**update_data) PedimentosService.update( session, existing.id, tenant_id, update_schema, company_id ) updated_count += 1 else: skipped_invalid += 1 skipped_details.append({"line": i, "reason": "Pedimento no encontrado"}) else: # No existe: crear (ambos modos) data = row_to_pedimento_data( row_norm, short_name_to_id, date_format_preference ) if "pedimento_dates" not in data or data.get("pedimento_dates") is None: data["pedimento_dates"] = PedimentoDatesCreate( entry_date=datetime.now(), end_date=datetime.now(), ) create_data = PedimentosCreate(**data) PedimentosService.create(session, create_data, tenant_id, company_id) inserted_count += 1 except ValueError as ve: if "Ya existe" in str(ve) or "duplicate" in str(ve).lower(): skipped_duplicate += 1 skipped_details.append({"line": i, "reason": str(ve)}) else: skipped_invalid += 1 skipped_details.append({"line": i, "reason": str(ve)}) except Exception as e: logger.warning("Pedimentos import line %s: %s", i, e) skipped_invalid += 1 skipped_details.append({"line": i, "reason": str(e)}) except Exception as e: logger.exception("Pedimentos import task failed") return {"status": "failed", "error": str(e)} common_storage.cleanup_import_job( JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path, ) total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate if inserted_count == 0 and updated_count == 0 and total_skipped > 0: reasons = "; ".join( f"Línea {d.get('line', '?')}: {d.get('reason', '')}" for d in skipped_details[:5] ) if len(skipped_details) > 5: reasons += f" (+{len(skipped_details) - 5} más)" return { "status": "warning", "inserted": 0, "updated": 0, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, "message": f"No se insertaron registros. {total_skipped} rechazados. Motivos: {reasons}", } if inserted_count == 0 and updated_count == 0: return { "status": "failed", "error": "No hay registros válidos en el archivo CSV", "inserted": 0, "updated": 0, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, } return { "status": "finished", "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_missing_fk": skipped_missing_fk, "skipped_duplicate": skipped_duplicate, "skipped_details": skipped_details, } @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): logger.info("Pedimentos import: starting commit for job %s", job_id) return _do_commit(job_id)