469 lines
21 KiB
Python
469 lines
21 KiB
Python
"""
|
|
Tareas Celery para importación CSV de Pedimentos.
|
|
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
|
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader), fk_loader, validators, mappers.
|
|
Create/update delegan en PedimentosService; defaults de fechas (pedimento_dates) y merge vs replace
|
|
están alineados con el servicio para paridad con el flujo API.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import datetime
|
|
from typing import Dict, Any, Optional, List
|
|
|
|
from core.celery_app import celery_app
|
|
from core.database import CoreSessionLocal
|
|
|
|
from ..common import storage as common_storage
|
|
from ..common import normalize as common_normalize
|
|
from ..common import meta as common_meta
|
|
from ..common import responses as common_responses
|
|
from ..common import csv_reader as common_csv_reader
|
|
from .template_config import (
|
|
row_from_template,
|
|
is_clarion_layout,
|
|
parse_pedimento_col_a,
|
|
detect_headers_or_data,
|
|
)
|
|
from .validators import validate_row_pedimento
|
|
from .common.mappers import row_to_pedimento_data, row_to_pedimento_data_merge_existing
|
|
from .common.fk_loader import load_pedimentos_fk_sets, pedimento_key_from_parsed
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
JOB_TYPE = "ped"
|
|
TEMPLATE_ID = "pedimentos"
|
|
|
|
# Para routes.py
|
|
PED_IMPORT_FILE_PREFIX = "ped_import_file:"
|
|
PED_IMPORT_META_PREFIX = "ped_import_meta:"
|
|
PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:"
|
|
PED_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
|
|
|
|
|
def _read_plan_for_pedimentos(fieldnames):
|
|
if fieldnames:
|
|
return common_csv_reader.CsvReadPlan(header_mode="headerless", fieldnames=fieldnames)
|
|
return common_csv_reader.CsvReadPlan(header_mode="header")
|
|
|
|
|
|
def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
|
return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID)
|
|
|
|
|
|
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
|
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import")
|
|
if not file_path:
|
|
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
|
if os.path.getsize(file_path) == 0:
|
|
return {"status": "failed", "error": "El archivo está vacío. Verifica que el CSV tenga contenido."}
|
|
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import")
|
|
|
|
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
|
|
|
fieldnames, has_header = detect_headers_or_data(
|
|
file_path,
|
|
common_normalize.normalize_header,
|
|
parse_pedimento_col_a,
|
|
)
|
|
read_plan = _read_plan_for_pedimentos(fieldnames)
|
|
try:
|
|
total_rows = common_csv_reader.count_csv_rows(file_path, has_header=has_header, read_plan=read_plan)
|
|
except Exception as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
try:
|
|
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
|
except ValueError as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
meta = common_meta.load_meta(file_path) or {}
|
|
actualizar = meta.get("actualizar", False)
|
|
date_format_preference = meta.get("dateFormat") or meta.get("date_format")
|
|
|
|
try:
|
|
with CoreSessionLocal() as session:
|
|
(
|
|
valid_client_ids,
|
|
valid_regimes,
|
|
valid_pedimento_codes,
|
|
valid_clave_regimen_tipo,
|
|
valid_aduana_seccion,
|
|
existing_pedimento_keys,
|
|
valid_anexo22_claves,
|
|
valid_patentes,
|
|
short_name_to_id,
|
|
) = load_pedimentos_fk_sets(session, tenant_id, company_id)
|
|
except Exception as e:
|
|
logger.error("Pedimentos import: failed to load FK sets: %s", e)
|
|
return {"status": "failed", "error": "No se pudo cargar catálogos"}
|
|
|
|
error_count = 0
|
|
processed_rows = 0
|
|
errors_detail: List[Dict[str, Any]] = []
|
|
error_lines_list: List[int] = []
|
|
|
|
try:
|
|
with open(error_path, "w", encoding="utf-8") as f_err:
|
|
for i, row in common_csv_reader.iter_csv_rows_with_plan(file_path, read_plan=read_plan):
|
|
if progress_callback:
|
|
progress_callback(i, total_rows, error_count)
|
|
|
|
row_norm = _norm_row(row)
|
|
row_errors = validate_row_pedimento(
|
|
row_norm,
|
|
i,
|
|
short_name_to_id,
|
|
valid_regimes,
|
|
valid_pedimento_codes,
|
|
valid_clave_regimen_tipo,
|
|
valid_aduana_seccion,
|
|
existing_pedimento_keys,
|
|
valid_anexo22_claves,
|
|
valid_patentes,
|
|
actualizar=actualizar,
|
|
raw_row=row,
|
|
date_format_preference=date_format_preference,
|
|
)
|
|
if row_errors:
|
|
has_blocking_error = any(not e.get("warning", False) for e in row_errors)
|
|
if has_blocking_error:
|
|
error_count += 1
|
|
error_lines_list.append(i)
|
|
for e in row_errors:
|
|
if not e.get("warning", False):
|
|
f_err.write(json.dumps(e) + "\n")
|
|
if len(errors_detail) < 500:
|
|
for e in row_errors:
|
|
if len(errors_detail) >= 500:
|
|
break
|
|
errors_detail.append({
|
|
"line": e["line"],
|
|
"col": e.get("col", ""),
|
|
"msg": e.get("msg", ""),
|
|
"solution": e.get("solution", ""),
|
|
"warning": bool(e.get("warning", False)),
|
|
})
|
|
processed_rows += 1
|
|
|
|
if error_lines_list:
|
|
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
|
except Exception as e:
|
|
logger.error("Pedimentos import scan failed: %s", e)
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
|
|
|
|
|
@celery_app.task(bind=True)
|
|
def scan_file(self, job_id: str, config: str = None):
|
|
logger.info("Pedimentos import: starting scan for job %s", job_id)
|
|
|
|
def on_progress(current: int, total: int, errors: int) -> None:
|
|
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
|
|
|
return _do_scan(job_id, progress_callback=on_progress)
|
|
|
|
|
|
def _do_commit(job_id: str) -> Dict[str, Any]:
|
|
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import")
|
|
if not file_path:
|
|
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
|
if not os.path.exists(alt_path):
|
|
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
|
file_path = alt_path
|
|
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import")
|
|
|
|
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
|
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
|
|
|
try:
|
|
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
|
except ValueError as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
meta = common_meta.load_meta(file_path) or {}
|
|
actualizar = meta.get("actualizar", False)
|
|
date_format_preference = meta.get("dateFormat") or meta.get("date_format")
|
|
|
|
try:
|
|
with CoreSessionLocal() as session:
|
|
(
|
|
valid_client_ids,
|
|
valid_regimes,
|
|
valid_pedimento_codes,
|
|
valid_clave_regimen_tipo,
|
|
valid_aduana_seccion,
|
|
existing_pedimento_keys,
|
|
valid_anexo22_claves,
|
|
valid_patentes,
|
|
short_name_to_id,
|
|
) = load_pedimentos_fk_sets(session, tenant_id, company_id)
|
|
except Exception as e:
|
|
logger.error("Pedimentos import: failed to load FK sets: %s", e)
|
|
return {"status": "failed", "error": "No se pudo cargar catálogos"}
|
|
|
|
from api.v1.modules.a76.pedmientos.dtos.pedimentos import (
|
|
PedimentosCreate,
|
|
PedimentosUpdate,
|
|
)
|
|
from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate
|
|
from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService
|
|
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
|
|
|
|
inserted_count = 0
|
|
updated_count = 0
|
|
skipped_invalid = 0
|
|
skipped_missing_fk = 0
|
|
skipped_duplicate = 0
|
|
skipped_details: List[Dict[str, Any]] = []
|
|
meta_path = common_meta.get_meta_path(file_path)
|
|
|
|
fieldnames_commit, _ = detect_headers_or_data(
|
|
file_path,
|
|
common_normalize.normalize_header,
|
|
parse_pedimento_col_a,
|
|
)
|
|
read_plan_commit = _read_plan_for_pedimentos(fieldnames_commit)
|
|
|
|
def _key_from_row(r: Dict[str, Any]) -> Optional[str]:
|
|
if is_clarion_layout(r):
|
|
año = (r.get("AÑO") or "").strip()
|
|
patente = (r.get("PATENTE") or "").strip()
|
|
numero = (r.get("NUMERO") or "").strip()
|
|
if año and patente and numero:
|
|
adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3]
|
|
return pedimento_key_from_parsed(año[:2], adu, patente[:4], numero[:7])
|
|
parsed = parse_pedimento_col_a((r.get("PEDIMENTO") or "").strip())
|
|
if not parsed:
|
|
return None
|
|
y, lic, num = parsed
|
|
adu = (r.get("ADUANA_SECCION_CRUCE") or "").strip()[:3]
|
|
return pedimento_key_from_parsed(y, adu, lic, num)
|
|
y = (r.get("AÑO") or "").strip()[:2]
|
|
adu = (r.get("ADUANA_SECCION_CRUCE") or r.get("ADUANA") or "").strip()[:3]
|
|
lic = (r.get("PATENTE") or "").strip()[:4]
|
|
num = (r.get("NUMERO") or "").strip()[:7]
|
|
return pedimento_key_from_parsed(y, adu, lic, num)
|
|
|
|
try:
|
|
with CoreSessionLocal() as session:
|
|
for i, row in common_csv_reader.iter_csv_rows_with_plan(file_path, read_plan=read_plan_commit):
|
|
if i in error_lines:
|
|
continue
|
|
|
|
row_norm = _norm_row(row)
|
|
row_errors = validate_row_pedimento(
|
|
row_norm,
|
|
i,
|
|
short_name_to_id,
|
|
valid_regimes,
|
|
valid_pedimento_codes,
|
|
valid_clave_regimen_tipo,
|
|
valid_aduana_seccion,
|
|
existing_pedimento_keys,
|
|
valid_anexo22_claves,
|
|
valid_patentes,
|
|
actualizar=actualizar,
|
|
raw_row=row,
|
|
date_format_preference=date_format_preference,
|
|
)
|
|
if row_errors and any(not e.get("warning", False) for e in row_errors):
|
|
skipped_invalid += 1
|
|
blocking_errors = [e for e in row_errors if not e.get("warning", False)]
|
|
for blocking in blocking_errors:
|
|
skipped_details.append({
|
|
"line": i,
|
|
"reason": f"{blocking.get('col', '')}: {blocking.get('msg', '')}",
|
|
"solution": blocking.get("solution", ""),
|
|
})
|
|
continue
|
|
|
|
key = _key_from_row(row_norm)
|
|
try:
|
|
# Si el pedimento ya existe: actualizar (merge) o reemplazar (crear). Si no existe: crear.
|
|
if key and key in existing_pedimento_keys:
|
|
existing = (
|
|
session.query(Pedimentos)
|
|
.filter(
|
|
Pedimentos.tenant_id == tenant_id,
|
|
Pedimentos.company_id == company_id,
|
|
Pedimentos.year == key.split("|")[0],
|
|
Pedimentos.customs_office == key.split("|")[1],
|
|
Pedimentos.license == key.split("|")[2],
|
|
Pedimentos.pedimento_number == key.split("|")[3],
|
|
)
|
|
.first()
|
|
)
|
|
if existing:
|
|
if actualizar:
|
|
# Modo actualizar: merge con existente
|
|
existing_dict = {
|
|
"year": existing.year,
|
|
"customs_office": existing.customs_office,
|
|
"license": existing.license,
|
|
"pedimento_number": existing.pedimento_number,
|
|
"client_id": existing.client_id,
|
|
"pedimento_code": existing.pedimento_code,
|
|
"regime": existing.regime,
|
|
"operation_type": existing.operation_type,
|
|
"pedimento_type": existing.pedimento_type,
|
|
"status": existing.status,
|
|
"usd_value": existing.usd_value,
|
|
"paid_price": existing.paid_price,
|
|
"gross_weight": existing.gross_weight,
|
|
"exchange_rate": existing.exchange_rate,
|
|
"observations": existing.observations,
|
|
}
|
|
if existing.pedimento_dates:
|
|
existing_dict["pedimento_dates"] = {
|
|
"entry_date": existing.pedimento_dates.entry_date,
|
|
"end_date": existing.pedimento_dates.end_date,
|
|
"start_date": getattr(
|
|
existing.pedimento_dates, "start_date", existing.pedimento_dates.entry_date
|
|
),
|
|
"payment_date": getattr(
|
|
existing.pedimento_dates, "payment_date", existing.pedimento_dates.entry_date
|
|
),
|
|
}
|
|
data = row_to_pedimento_data_merge_existing(
|
|
row_norm,
|
|
existing_dict,
|
|
short_name_to_id,
|
|
date_format_preference,
|
|
)
|
|
else:
|
|
# Modo crear: reemplazar con datos del CSV
|
|
data = row_to_pedimento_data(
|
|
row_norm, short_name_to_id, date_format_preference
|
|
)
|
|
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
|
data["pedimento_dates"] = PedimentoDatesCreate(
|
|
entry_date=datetime.now(),
|
|
end_date=datetime.now(),
|
|
)
|
|
update_data = {k: v for k, v in data.items() if k != "pedimento_dates"}
|
|
if data.get("pedimento_dates"):
|
|
update_data["pedimento_dates"] = PedimentoDatesCreate(
|
|
**data["pedimento_dates"]
|
|
)
|
|
update_schema = PedimentosUpdate(**update_data)
|
|
PedimentosService.update(
|
|
session, existing.id, tenant_id, update_schema, company_id
|
|
)
|
|
updated_count += 1
|
|
else:
|
|
skipped_invalid += 1
|
|
skipped_details.append(
|
|
{
|
|
"line": i,
|
|
"reason": "Pedimento no encontrado",
|
|
"solution": "Verifica que el pedimento exista en el sistema usando las columnas PEDIMENTO (o AÑO/PATENTE/NUMERO) y ADUANA_SECCION_CRUCE; si aplica, cambia a modo creación.",
|
|
}
|
|
)
|
|
else:
|
|
# No existe: crear (ambos modos)
|
|
data = row_to_pedimento_data(
|
|
row_norm, short_name_to_id, date_format_preference
|
|
)
|
|
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
|
data["pedimento_dates"] = PedimentoDatesCreate(
|
|
entry_date=datetime.now(),
|
|
end_date=datetime.now(),
|
|
)
|
|
create_data = PedimentosCreate(**data)
|
|
PedimentosService.create(session, create_data, tenant_id, company_id)
|
|
inserted_count += 1
|
|
except ValueError as ve:
|
|
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
|
|
skipped_duplicate += 1
|
|
skipped_details.append(
|
|
{
|
|
"line": i,
|
|
"reason": str(ve),
|
|
"solution": "Si deseas actualizar registros existentes, activa la opción de actualizar en la importación; si no, corrige el pedimento para evitar duplicados.",
|
|
}
|
|
)
|
|
else:
|
|
skipped_invalid += 1
|
|
skipped_details.append(
|
|
{
|
|
"line": i,
|
|
"reason": str(ve),
|
|
"solution": "Revisa la línea del CSV y el valor que falló la validación; corrige el dato y vuelve a intentar.",
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.warning("Pedimentos import line %s: %s", i, e)
|
|
skipped_invalid += 1
|
|
skipped_details.append(
|
|
{
|
|
"line": i,
|
|
"reason": str(e),
|
|
"solution": "Revisa la línea del CSV y los catálogos/fechas relacionados para resolver el error.",
|
|
}
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.exception("Pedimentos import task failed")
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
common_storage.cleanup_import_job(
|
|
JOB_TYPE, job_id,
|
|
file_path=file_path,
|
|
error_path=error_path,
|
|
meta_path=meta_path,
|
|
)
|
|
|
|
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
|
critical_reference_gaps = skipped_invalid + skipped_missing_fk
|
|
reference_state_ready = critical_reference_gaps == 0
|
|
if inserted_count == 0 and updated_count == 0 and total_skipped > 0:
|
|
reasons = "; ".join(
|
|
f"Línea {d.get('line', '?')}: {d.get('reason', '')}" for d in skipped_details[:5]
|
|
)
|
|
if len(skipped_details) > 5:
|
|
reasons += f" (+{len(skipped_details) - 5} más)"
|
|
return {
|
|
"status": "warning",
|
|
"inserted": 0,
|
|
"updated": 0,
|
|
"skipped_invalid": skipped_invalid,
|
|
"skipped_missing_fk": skipped_missing_fk,
|
|
"skipped_duplicate": skipped_duplicate,
|
|
"skipped_details": skipped_details,
|
|
"critical_reference_gaps": critical_reference_gaps,
|
|
"reference_state_ready": reference_state_ready,
|
|
"message": f"No se insertaron registros. {total_skipped} rechazados. Motivos: {reasons}",
|
|
}
|
|
if inserted_count == 0 and updated_count == 0:
|
|
return {
|
|
"status": "failed",
|
|
"error": "No hay registros válidos en el archivo CSV",
|
|
"inserted": 0,
|
|
"updated": 0,
|
|
"skipped_invalid": skipped_invalid,
|
|
"skipped_missing_fk": skipped_missing_fk,
|
|
"skipped_duplicate": skipped_duplicate,
|
|
"skipped_details": skipped_details,
|
|
"critical_reference_gaps": critical_reference_gaps,
|
|
"reference_state_ready": reference_state_ready,
|
|
}
|
|
return {
|
|
"status": "finished",
|
|
"inserted": inserted_count,
|
|
"updated": updated_count,
|
|
"skipped_invalid": skipped_invalid,
|
|
"skipped_missing_fk": skipped_missing_fk,
|
|
"skipped_duplicate": skipped_duplicate,
|
|
"skipped_details": skipped_details,
|
|
"critical_reference_gaps": critical_reference_gaps,
|
|
"reference_state_ready": reference_state_ready,
|
|
}
|
|
|
|
|
|
@celery_app.task(bind=True)
|
|
def insert_valid_rows(self, job_id: str):
|
|
logger.info("Pedimentos import: starting commit for job %s", job_id)
|
|
return _do_commit(job_id)
|