feature/abstraccion-de-funcionalidades-y-manejo-por-tareas-en-comun
This commit is contained in:
hreyes
@@ -0,0 +1 @@
|
||||
# common validators, mappers, fk_loader for boms CSV import
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (BOMs).
|
||||
"""
|
||||
import re
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_decimal_required_min(
|
||||
row: Dict[str, Any], col: str, line_num: int, min_val: Decimal
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
try:
|
||||
v = Decimal(str(val))
|
||||
if v < min_val:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def check_optional_number(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
if not _optional_number(row.get(col)):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
allowed: Optional[set],
|
||||
msg: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or allowed is None or len(allowed) == 0:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación de import CSV de BOMs.
|
||||
"""
|
||||
from typing import Set
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_boms_fk_sets(tenant_id: int, company_id: int) -> Set[str]:
|
||||
"""Carga valid_part_numbers (Part.part_number por tenant/company)."""
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("BOMs import: could not load parts: %s", e)
|
||||
return valid_part_numbers
|
||||
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para BOM (para cuando exista tabla BOM).
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).replace(",", "."))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_bom_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_part_numbers: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para BOM.
|
||||
Para uso futuro cuando exista tabla BOM.
|
||||
"""
|
||||
parent = _str_or_none(row_norm.get("NUMPARTE_PADRE"), 70)
|
||||
component = _str_or_none(row_norm.get("NUMPARTE_COMPONENTE"), 70)
|
||||
quantity = _decimal_or_none(row_norm.get("CANTIDAD"))
|
||||
uom = _str_or_none(row_norm.get("UNIMED"), 10)
|
||||
version_bom = _decimal_or_none(row_norm.get("VERSION_BOM"))
|
||||
version_bill = _decimal_or_none(row_norm.get("VERSION_BILL"))
|
||||
return {
|
||||
"parent_part_number": parent,
|
||||
"component_part_number": component,
|
||||
"quantity": quantity,
|
||||
"uom": uom,
|
||||
"version_bom": version_bom,
|
||||
"version_bill": version_bill,
|
||||
}
|
||||
@@ -1,430 +1,168 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista.
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y cuenta filas válidas.
|
||||
Usa layouts_csv.common y common.fk_loader, validators.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_bom
|
||||
from .common.fk_loader import load_boms_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "bom"
|
||||
|
||||
# Para routes.py
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"bom_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila BOM según columnas del template. FK opcional a a76.parts."""
|
||||
parent = (row.get("NUMPARTE_PADRE") or "").strip()
|
||||
if not parent:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"}
|
||||
if len(parent) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
component = (row.get("NUMPARTE_COMPONENTE") or "").strip()
|
||||
if not component:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"}
|
||||
if len(component) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
qty = row.get("CANTIDAD")
|
||||
if qty is None or qty == "":
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"}
|
||||
try:
|
||||
val = Decimal(str(qty))
|
||||
if val < 0:
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 10:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
version_bom = row.get("VERSION_BOM")
|
||||
if not _optional_number(version_bom):
|
||||
return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"}
|
||||
|
||||
version_bill = row.get("VERSION_BILL")
|
||||
if not _optional_number(version_bill):
|
||||
return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"}
|
||||
|
||||
# Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas)
|
||||
if valid_part_numbers is not None and len(valid_part_numbers) > 0:
|
||||
if parent not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"}
|
||||
if component not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"}
|
||||
|
||||
return None
|
||||
BOM_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"BOMs import: starting scan for job {job_id}")
|
||||
logger.info("BOMs import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts for FK validation: {e}")
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import scan failed: {e}")
|
||||
logger.error("BOMs import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"BOMs import: starting commit for job {job_id}")
|
||||
logger.info("BOMs import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"BOMs import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts: {e}")
|
||||
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
valid_count = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
# Cuando exista la tabla de destino, aquí se hará insert/update.
|
||||
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, 0, 0, skipped_details,
|
||||
)
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
|
||||
response["message"] = (
|
||||
f"WIP: {valid_count} filas válidas. "
|
||||
"La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
logger.exception("BOMs import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"BOMs import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_bom
|
||||
|
||||
__all__ = ["validate_row_bom"]
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de BOMs.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_max_length,
|
||||
check_decimal_required_min,
|
||||
check_optional_number,
|
||||
check_in_set,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_required_parent(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "NUMPARTE_PADRE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_required_component(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "NUMPARTE_COMPONENTE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_quantity(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_decimal_required_min(row, "CANTIDAD", line_num, Decimal("0"))
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "UNIMED", 10, line_num)
|
||||
|
||||
|
||||
def validate_row_optional_numbers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_optional_number(row, "VERSION_BOM", line_num)
|
||||
if err:
|
||||
return err
|
||||
return check_optional_number(row, "VERSION_BILL", line_num)
|
||||
|
||||
|
||||
def validate_row_fks(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not valid_part_numbers or len(valid_part_numbers) == 0:
|
||||
return None
|
||||
err = check_in_set(
|
||||
row, "NUMPARTE_PADRE", line_num,
|
||||
valid_part_numbers, "Parte padre no existe en catálogo",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return check_in_set(
|
||||
row, "NUMPARTE_COMPONENTE", line_num,
|
||||
valid_part_numbers, "Parte componente no existe en catálogo",
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila BOM.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required_parent,
|
||||
validate_row_required_component,
|
||||
validate_row_quantity,
|
||||
validate_row_lengths,
|
||||
validate_row_optional_numbers,
|
||||
validate_row_fks,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de BOMs.
|
||||
Encadena: requeridos (padre, componente, cantidad) → longitudes → opcionales numéricos → FKs.
|
||||
"""
|
||||
err = validate_row_required_parent(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_required_component(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_quantity(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_optional_numbers(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fks(row, line_num, valid_part_numbers)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers, fk_loader for classes CSV import
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (clases de materiales).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_int_range(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
min_val: int,
|
||||
max_val: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; devuelve error si no es int o está fuera de rango."""
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
v = int(val)
|
||||
if v < min_val or v > max_val:
|
||||
return {"line": line_num, "col": col, "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número entero"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
allowed: Optional[set],
|
||||
msg: str = "No existe en el catálogo",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor y allowed no es None."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or allowed is None:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
@@ -0,0 +1,37 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación/mapeo de import CSV de clases de materiales.
|
||||
"""
|
||||
from typing import Set, Tuple
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_classes_fk_sets(
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Tuple[Set[str], Set[str]]:
|
||||
"""
|
||||
Carga valid_material_keys (MaterialType.key) y valid_uom_codes (UnitOfMeasure.code).
|
||||
Devuelve (valid_material_keys, valid_uom_codes).
|
||||
"""
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("Classes import: could not load FK sets: %s", e)
|
||||
return valid_material_keys, valid_uom_codes
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Class (clases de materiales).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def row_to_class_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_material_keys: Set[str],
|
||||
valid_uom_codes: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para Class.
|
||||
Ajusta material_key y unit_of_measure a None si no están en los conjuntos.
|
||||
"""
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
return {
|
||||
"class_code": class_code,
|
||||
"description_es": desc_es,
|
||||
"description_en": desc_en,
|
||||
"material_key": material_key,
|
||||
"unit_of_measure": unit_of_measure,
|
||||
"fraction": fraction,
|
||||
"us_fraction": us_fraction,
|
||||
"sub_key": sub_key,
|
||||
"physical_review": physical_review,
|
||||
"iva_exempt_fraction": iva_exempt_fraction,
|
||||
}
|
||||
@@ -1,528 +1,237 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_class
|
||||
from .common.mappers import row_to_class_data
|
||||
from .common.fk_loader import load_classes_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cls"
|
||||
|
||||
# Para routes.py
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
if not class_code:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
|
||||
if len(class_code) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
material_key = (row.get("CLAVEMAT") or "").strip()
|
||||
if material_key:
|
||||
if len(material_key) > 10:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
|
||||
if valid_material_keys is not None and material_key not in valid_material_keys:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom:
|
||||
if len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
if valid_uom_codes is not None and uom not in valid_uom_codes:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 20:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
sub_key = (row.get("CLAVESUB") or "").strip()
|
||||
if sub_key and len(sub_key) > 5:
|
||||
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
|
||||
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if iva_exempt and len(iva_exempt) > 4:
|
||||
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
rev_fisica = row.get("REVFISICA")
|
||||
if rev_fisica is not None and rev_fisica != "":
|
||||
try:
|
||||
v = int(rev_fisica)
|
||||
if v < -32768 or v > 32767:
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
|
||||
|
||||
return None
|
||||
CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Classes import: starting scan for job {job_id}")
|
||||
logger.info("Classes import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_material_keys, valid_uom_codes = load_classes_fk_sets(tenant_id, company_id)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_class(
|
||||
row_norm,
|
||||
i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import scan failed: {e}")
|
||||
logger.error("Classes import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Classes import: starting commit for job {job_id}")
|
||||
logger.info("Classes import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
valid_material_keys, valid_uom_codes = load_classes_fk_sets(tenant_id, company_id)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code: Dict[str, Class] = {}
|
||||
for c in (
|
||||
session.query(Class)
|
||||
.filter(
|
||||
existing_by_code = {
|
||||
c.class_code: c
|
||||
for c in session.query(Class).filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
).all()
|
||||
}
|
||||
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_class(
|
||||
row_norm,
|
||||
i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_code[c.class_code] = c
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes)
|
||||
class_code = data.get("class_code")
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
existing = existing_by_code.get(class_code)
|
||||
if existing:
|
||||
existing.description_es = data["description_es"]
|
||||
existing.description_en = data["description_en"]
|
||||
existing.material_key = data["material_key"]
|
||||
existing.unit_of_measure = data["unit_of_measure"]
|
||||
existing.fraction = data["fraction"]
|
||||
existing.us_fraction = data["us_fraction"]
|
||||
existing.sub_key = data["sub_key"]
|
||||
existing.physical_review = data["physical_review"]
|
||||
existing.iva_exempt_fraction = data["iva_exempt_fraction"]
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
**data,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
if existing:
|
||||
existing.description_es = desc_es
|
||||
existing.description_en = desc_en
|
||||
existing.material_key = material_key
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.sub_key = sub_key
|
||||
existing.physical_review = physical_review
|
||||
existing.iva_exempt_fraction = iva_exempt_fraction
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
class_code=class_code,
|
||||
description_es=desc_es,
|
||||
description_en=desc_en,
|
||||
material_key=material_key,
|
||||
unit_of_measure=unit_of_measure,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
sub_key=sub_key,
|
||||
physical_review=physical_review,
|
||||
iva_exempt_fraction=iva_exempt_fraction,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Classes import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
logger.error("Classes import DB error: %s", db_err)
|
||||
return common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error=str(db_err),
|
||||
)
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
response = common_responses.commit_result(
|
||||
"warning", 0, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
message=f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
)
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
error="No hay registros válidos en el archivo CSV",
|
||||
)
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
logger.exception("Classes import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_class
|
||||
|
||||
__all__ = ["validate_row_class"]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de clases de materiales.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_max_length,
|
||||
check_int_range,
|
||||
check_in_set,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "CLASE", 8, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("DESCRIPCIONE", 500),
|
||||
("DESCRIPCIONI", 500),
|
||||
("CLAVEMAT", 10),
|
||||
("UNIMED", 5),
|
||||
("FRACCION", 20),
|
||||
("FRACCIONAME", 16),
|
||||
("CLAVESUB", 5),
|
||||
("FRACCIONEXENTAIVA", 4),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_int_range(row, "REVFISICA", line_num, -32768, 32767)
|
||||
|
||||
|
||||
def validate_row_fks(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]],
|
||||
valid_uom_codes: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
err = check_in_set(
|
||||
row, "CLAVEMAT", line_num, valid_material_keys, "Tipo de material no existe"
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_in_set(
|
||||
row, "UNIMED", line_num, valid_uom_codes, "Unidad de medida no existe"
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila de clase de material.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required,
|
||||
validate_row_lengths,
|
||||
validate_row_types,
|
||||
validate_row_fks,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de clases de materiales.
|
||||
Encadena: requeridos → longitudes → tipos → FKs.
|
||||
"""
|
||||
err = validate_row_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
err = validate_row_types(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
err = validate_row_fks(row, line_num, valid_material_keys, valid_uom_codes)
|
||||
if err:
|
||||
return err
|
||||
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for clients_and_providers)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de clientes y proveedores.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum
|
||||
|
||||
RFC_MAX = 30
|
||||
NAME_MAX = 256
|
||||
SHORT_NAME_MAX = 10
|
||||
CURP_MAX = 19
|
||||
|
||||
|
||||
def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def check_tipo_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if not tipo_raw:
|
||||
return None
|
||||
if parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def parse_active(val: Optional[str]) -> bool:
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para ClientProvider y opcional ClientProviderAddress.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum
|
||||
|
||||
from .common_validators import (
|
||||
parse_client_or_provider,
|
||||
parse_active,
|
||||
)
|
||||
|
||||
MAX_LEN = {
|
||||
"rfc": 30,
|
||||
"name": 256,
|
||||
"short_name": 10,
|
||||
"curp": 19,
|
||||
"responsible": 80,
|
||||
"position": 30,
|
||||
"incoterm": 19,
|
||||
"email": 100,
|
||||
"phone": 30,
|
||||
"address": 100,
|
||||
"postal_code": 15,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"contact": 50,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_client_provider_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
|
||||
"""
|
||||
Mapea fila normalizada a datos para ClientProvider y opcional ClientProviderAddress.
|
||||
Devuelve (cp_data, address_data_or_none). address_data es para crear después del flush (necesita client_id).
|
||||
"""
|
||||
rfc = _str_or_none(row_norm.get("RFC"), MAX_LEN["rfc"])
|
||||
if not rfc:
|
||||
return ({}, None)
|
||||
|
||||
client_or_provider = parse_client_or_provider(row_norm.get("TIPO")) or ClientOrProviderEnum.BOTH
|
||||
|
||||
cp_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"rfc": rfc,
|
||||
"name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]),
|
||||
"short_name": _str_or_none(row_norm.get("SHORT_NAME"), MAX_LEN["short_name"]),
|
||||
"curp": _str_or_none(row_norm.get("CURP"), MAX_LEN["curp"]),
|
||||
"client_or_provider": client_or_provider,
|
||||
"responsible": _str_or_none(row_norm.get("RESPONSABLE"), MAX_LEN["responsible"]),
|
||||
"position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]),
|
||||
"incoterm": _str_or_none(row_norm.get("INCOTERM"), MAX_LEN["incoterm"]),
|
||||
"is_active": parse_active(row_norm.get("ACTIVO")),
|
||||
}
|
||||
|
||||
email = _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"])
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"])
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"])
|
||||
if email or phone or address_str:
|
||||
address_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"streets": address_str,
|
||||
"postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"phone": phone,
|
||||
"email": email,
|
||||
"contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]),
|
||||
}
|
||||
return (cp_data, address_data)
|
||||
return (cp_data, None)
|
||||
@@ -1,340 +1,128 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientOrProviderEnum,
|
||||
)
|
||||
from .validators import validate_row_client_provider
|
||||
from .common.mappers import row_to_client_provider_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
|
||||
JOB_TYPE = "cp"
|
||||
|
||||
# Para routes.py
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
CP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active(val: Optional[str]) -> bool:
|
||||
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
|
||||
rfc = (row.get("RFC") or "").strip()
|
||||
if not rfc:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
|
||||
if len(rfc) > 30:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
|
||||
name = (row.get("NOMBRE") or "").strip()
|
||||
if len(name) > 256:
|
||||
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
if short_name and len(short_name) > 10:
|
||||
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
curp = (row.get("CURP") or "").strip()
|
||||
if curp and len(curp) > 19:
|
||||
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CP import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import scan failed: {e}")
|
||||
logger.error("CP import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
|
||||
Upsert por (tenant_id, company_id, rfc).
|
||||
"""
|
||||
logger.info(f"CP import: starting commit for job {job_id}")
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("CP import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CP import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
@@ -347,154 +135,108 @@ def insert_valid_rows(self, job_id: str):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
cp_data, address_data = row_to_client_provider_data(row_norm, tenant_id, company_id)
|
||||
if not cp_data or not cp_data.get("rfc"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
rfc = _str_or_none(row_norm.get("RFC"), 30)
|
||||
if not rfc:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
|
||||
if client_or_provider is None:
|
||||
client_or_provider = ClientOrProviderEnum.BOTH
|
||||
|
||||
is_active = _parse_active(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
|
||||
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
|
||||
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
|
||||
existing.client_or_provider = client_or_provider
|
||||
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(
|
||||
rfc = cp_data["rfc"]
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
for k, v in cp_data.items():
|
||||
if k not in ("tenant_id", "company_id", "rfc"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(**cp_data)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
if address_data:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
rfc=rfc,
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 256),
|
||||
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
|
||||
curp=_str_or_none(row_norm.get("CURP"), 19),
|
||||
client_or_provider=client_or_provider,
|
||||
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
|
||||
is_active=is_active,
|
||||
streets=address_data.get("streets"),
|
||||
postal_code=address_data.get("postal_code"),
|
||||
city=address_data.get("city"),
|
||||
state=address_data.get("state"),
|
||||
country=address_data.get("country"),
|
||||
phone=address_data.get("phone"),
|
||||
email=address_data.get("email"),
|
||||
contact=address_data.get("contact"),
|
||||
)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
|
||||
# Opcional: crear dirección si hay email/teléfono/dirección
|
||||
email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
|
||||
if email or phone or address_str:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_str,
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
phone=phone,
|
||||
email=email,
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
|
||||
)
|
||||
session.add(addr)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CP import DB error: {db_err}")
|
||||
logger.error("CP import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("CP import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CP import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("CP import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_client_provider
|
||||
|
||||
__all__ = ["validate_row_client_provider"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de clientes y proveedores.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
RFC_MAX,
|
||||
NAME_MAX,
|
||||
SHORT_NAME_MAX,
|
||||
CURP_MAX,
|
||||
check_required_max,
|
||||
check_max_length,
|
||||
check_tipo_client_provider,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_client_provider_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_required_max(row, "RFC", RFC_MAX, line_num)
|
||||
|
||||
|
||||
def validate_row_client_provider_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_max_length(row, "NOMBRE", NAME_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "SHORT_NAME", SHORT_NAME_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "CURP", CURP_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_client_provider_tipo(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_tipo_client_provider(row, line_num)
|
||||
|
||||
|
||||
def validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de clientes y proveedores.
|
||||
RFC requerido (max 30); TIPO opcional pero debe ser Cliente/Proveedor/Ambos; NOMBRE/SHORT_NAME/CURP longitudes.
|
||||
"""
|
||||
err = validate_row_client_provider_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_client_provider_tipo(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_client_provider_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila cliente/proveedor.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import validate_row_client_provider
|
||||
|
||||
__all__ = ["validate_row_client_provider"]
|
||||
@@ -0,0 +1 @@
|
||||
# Shared utilities for layouts_csv imports (storage, normalize, csv, meta, responses)
|
||||
28
backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py
Normal file
28
backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py
Normal file
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
Lectura de CSV con detección de delimitador (compartida por layouts_csv).
|
||||
"""
|
||||
import csv
|
||||
from typing import Iterator, Tuple, Dict, Any
|
||||
|
||||
|
||||
def iter_csv_rows(file_path: str) -> Iterator[Tuple[int, Dict[str, Any]]]:
|
||||
"""
|
||||
Abre el CSV, detecta dialecto y devuelve (line_num, row_dict) por cada fila.
|
||||
line_num empieza en 1 (primera fila de datos).
|
||||
"""
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in enumerate(reader, start=1):
|
||||
yield i, row
|
||||
|
||||
|
||||
def count_csv_rows(file_path: str) -> int:
|
||||
"""Cuenta filas del CSV (sin contar cabecera)."""
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
return sum(1 for _ in f) - 1
|
||||
35
backend/api/v1/modules/a76/layouts_csv/common/meta.py
Normal file
35
backend/api/v1/modules/a76/layouts_csv/common/meta.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Carga y guardado de meta (tenant_id, company_id) para imports CSV.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, Tuple
|
||||
|
||||
|
||||
def load_meta(file_path: str) -> Dict[str, Any]:
|
||||
"""Carga meta desde archivo .meta.json asociado al CSV. Devuelve dict vacío si no existe o falla."""
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if not os.path.exists(meta_path):
|
||||
return {}
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def require_tenant_context(file_path: str) -> Tuple[int, int]:
|
||||
"""
|
||||
Obtiene tenant_id y company_id del meta. Lanza ValueError si faltan.
|
||||
"""
|
||||
meta = load_meta(file_path)
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
raise ValueError("Falta contexto (tenant/company)")
|
||||
return int(tenant_id), int(company_id)
|
||||
|
||||
|
||||
def get_meta_path(file_path: str) -> str:
|
||||
"""Ruta del archivo .meta.json para un CSV."""
|
||||
return file_path.replace(".csv", ".meta.json")
|
||||
16
backend/api/v1/modules/a76/layouts_csv/common/normalize.py
Normal file
16
backend/api/v1/modules/a76/layouts_csv/common/normalize.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""
|
||||
Normalización de cabeceras CSV (compartida por todos los módulos layouts_csv).
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
"""Normaliza nombre de columna: NFKD, mayúsculas, sin acentos, espacios colapsados."""
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
47
backend/api/v1/modules/a76/layouts_csv/common/responses.py
Normal file
47
backend/api/v1/modules/a76/layouts_csv/common/responses.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Helpers para construir respuestas de scan y commit (formato unificado).
|
||||
"""
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
def scan_result(
|
||||
job_id: str,
|
||||
processed_rows: int,
|
||||
error_count: int,
|
||||
errors_detail: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Respuesta de scan_file (waiting_confirmation)."""
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def commit_result(
|
||||
status: str,
|
||||
inserted: int,
|
||||
skipped_invalid: int,
|
||||
skipped_missing_fk: int,
|
||||
skipped_duplicate: int,
|
||||
skipped_details: List[Dict[str, Any]],
|
||||
message: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Respuesta de insert_valid_rows (finished / warning / failed)."""
|
||||
out = {
|
||||
"status": status,
|
||||
"inserted": inserted,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if message:
|
||||
out["message"] = message
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
171
backend/api/v1/modules/a76/layouts_csv/common/storage.py
Normal file
171
backend/api/v1/modules/a76/layouts_csv/common/storage.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""
|
||||
Redis y rutas de archivos para imports CSV (compartido por módulos layouts_csv).
|
||||
Cada módulo usa un job_type (ej. "part", "cls", "bom") para prefijos y nombres de archivo.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, Set, List
|
||||
|
||||
from core.paths import layout_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def upload_dir() -> str:
|
||||
"""Directorio temporal para CSV en el worker."""
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def error_dir() -> str:
|
||||
"""Directorio de archivos JSONL de errores."""
|
||||
return layout_path("imports", "errors")
|
||||
|
||||
|
||||
def storage_keys(job_type: str, job_id: str) -> tuple:
|
||||
"""Prefijos Redis para un job_type y job_id. Devuelve (file_key, meta_key, error_lines_key)."""
|
||||
# Facturas usa prefijo "import_" sin tipo (compatibilidad con rutas existentes)
|
||||
if job_type == "" or job_type == "invoice":
|
||||
prefix = "import_"
|
||||
else:
|
||||
prefix = f"{job_type}_import_"
|
||||
return (
|
||||
f"{prefix}file:{job_id}",
|
||||
f"{prefix}meta:{job_id}",
|
||||
f"{prefix}error_lines:{job_id}",
|
||||
)
|
||||
|
||||
|
||||
def file_path_for_job(job_type: str, job_id: str) -> str:
|
||||
"""Ruta local del archivo CSV para un job."""
|
||||
if job_type == "" or job_type == "invoice":
|
||||
return os.path.join(upload_dir(), f"{job_id}.csv")
|
||||
return os.path.join(upload_dir(), f"{job_type}_{job_id}.csv")
|
||||
|
||||
|
||||
def error_path_for_job(job_type: str, job_id: str) -> str:
|
||||
"""Ruta del archivo JSONL de errores para un job."""
|
||||
os.makedirs(error_dir(), exist_ok=True)
|
||||
if job_type == "" or job_type == "invoice":
|
||||
return os.path.join(error_dir(), f"{job_id}.jsonl")
|
||||
return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl")
|
||||
|
||||
|
||||
def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]:
|
||||
"""
|
||||
Descarga contenido del CSV desde Redis y lo escribe en disco.
|
||||
Devuelve la ruta del archivo o None si no hay datos o falla.
|
||||
"""
|
||||
file_key, _, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(file_key)
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e)
|
||||
return None
|
||||
path = file_path_for_job(job_type, job_id)
|
||||
os.makedirs(upload_dir(), exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(raw)
|
||||
return path
|
||||
|
||||
|
||||
def ensure_meta_from_redis(job_type: str, job_id: str, file_path: str, log_prefix: str = "") -> bool:
|
||||
"""Descarga meta desde Redis y la escribe en .meta.json. Devuelve True si hubo datos."""
|
||||
_, meta_key, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(meta_key)
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode meta from Redis: %s", log_prefix or job_type, e)
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def store_error_lines(job_type: str, job_id: str, line_numbers: List[int]) -> None:
|
||||
"""Guarda la lista de números de línea con error en Redis."""
|
||||
_, _, error_key = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(error_key, json.dumps(line_numbers).encode("utf-8"), ex=IMPORT_REDIS_TTL)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to store error lines in Redis: %s", e)
|
||||
|
||||
|
||||
def get_error_lines(job_type: str, job_id: str, error_path: str) -> Set[int]:
|
||||
"""
|
||||
Obtiene el conjunto de líneas con error: primero desde Redis, si está vacío desde el JSONL.
|
||||
"""
|
||||
_, _, error_key = storage_keys(job_type, job_id)
|
||||
lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(error_key)
|
||||
if raw:
|
||||
lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug("Could not load error lines from Redis: %s", e)
|
||||
if not lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
return lines
|
||||
|
||||
|
||||
def delete_import_from_redis(job_type: str, job_id: str) -> None:
|
||||
"""Borra claves Redis del import (file, meta, error_lines)."""
|
||||
file_key, meta_key, error_key = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(file_key, meta_key, error_key)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to delete import keys from Redis: %s", e)
|
||||
|
||||
|
||||
def cleanup_import_job(
|
||||
job_type: str,
|
||||
job_id: str,
|
||||
file_path: Optional[str] = None,
|
||||
error_path: Optional[str] = None,
|
||||
meta_path: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Elimina archivos locales y claves Redis del job."""
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove file %s: %s", file_path, e)
|
||||
if error_path and os.path.exists(error_path):
|
||||
try:
|
||||
os.remove(error_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove error file %s: %s", error_path, e)
|
||||
if meta_path and os.path.exists(meta_path):
|
||||
try:
|
||||
os.remove(meta_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove meta %s: %s", meta_path, e)
|
||||
delete_import_from_redis(job_type, job_id)
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for customs_brokers)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de agentes aduanales (clave, licencia).
|
||||
"""
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
BROKER_KEY_MAX = 5
|
||||
LICENSE_MAX = 4
|
||||
|
||||
|
||||
def check_required_broker_key(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > BROKER_KEY_MAX:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {BROKER_KEY_MAX} caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_license(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if not licencia:
|
||||
return None
|
||||
if len(licencia) > LICENSE_MAX or not licencia.isdigit():
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
return None
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para CustomsBroker.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
MAX_LEN = {
|
||||
"broker_key": 5,
|
||||
"type": 9,
|
||||
"name": 80,
|
||||
"address": 1500,
|
||||
"postal_code": 15,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"phone": 30,
|
||||
"fax": 30,
|
||||
"email": 100,
|
||||
"country": 3,
|
||||
"tax_id": 30,
|
||||
"personal_id": 20,
|
||||
"position": 30,
|
||||
"license": 4,
|
||||
"company": 200,
|
||||
"contact": 80,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _license_value(row_norm: Dict[str, Any]) -> Optional[str]:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
if not lic or not lic.isdigit():
|
||||
return None
|
||||
return lic[:4]
|
||||
|
||||
|
||||
def row_to_customs_broker_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for CustomsBroker model (create or update)."""
|
||||
clave = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["broker_key"])
|
||||
if not clave:
|
||||
return {}
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"broker_key": clave,
|
||||
"type": _str_or_none(row_norm.get("TIPO"), MAX_LEN["type"]),
|
||||
"name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]),
|
||||
"address": _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]),
|
||||
"postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"phone": _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]),
|
||||
"fax": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]),
|
||||
"email": _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"tax_id": _str_or_none(row_norm.get("RFC"), MAX_LEN["tax_id"]),
|
||||
"personal_id": _str_or_none(row_norm.get("PERSONAL_ID"), MAX_LEN["personal_id"]),
|
||||
"position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]),
|
||||
"license": _license_value(row_norm),
|
||||
"company": _str_or_none(row_norm.get("EMPRESA"), MAX_LEN["company"]),
|
||||
"contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]),
|
||||
}
|
||||
@@ -1,293 +1,122 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_customs_broker
|
||||
from .common.mappers import row_to_customs_broker_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con a76.imports)
|
||||
JOB_TYPE = "cb"
|
||||
|
||||
# Para routes.py
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
CB_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > 5:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CB import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import scan failed: {e}")
|
||||
logger.error("CB import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Guardar números de línea con error en Redis para insert_valid_rows
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
|
||||
"""
|
||||
logger.info(f"CB import: starting commit for job {job_id}")
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("CB import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CB import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
@@ -302,146 +131,90 @@ def insert_valid_rows(self, job_id: str):
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
data = row_to_customs_broker_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("broker_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
clave = (row_norm.get("CLAVE") or "").strip()[:5]
|
||||
if not clave:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
|
||||
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
|
||||
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
|
||||
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
|
||||
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
|
||||
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
|
||||
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
|
||||
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
|
||||
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
existing.license = lic[:4] if lic and lic.isdigit() else None
|
||||
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
|
||||
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
license_val = lic[:4] if lic and lic.isdigit() else None
|
||||
new_broker = CustomsBroker(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
broker_key=clave,
|
||||
type=_str_or_none(row_norm.get("TIPO"), 9),
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 80),
|
||||
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
|
||||
fax=_str_or_none(row_norm.get("FAX"), 30),
|
||||
email=_str_or_none(row_norm.get("EMAIL"), 100),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
tax_id=_str_or_none(row_norm.get("RFC"), 30),
|
||||
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
license=license_val,
|
||||
company=_str_or_none(row_norm.get("EMPRESA"), 200),
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
|
||||
)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
clave = data["broker_key"]
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
for k, v in data.items():
|
||||
if k not in ("tenant_id", "company_id", "broker_key"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
else:
|
||||
new_broker = CustomsBroker(**data)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CB import DB error: {db_err}")
|
||||
logger.error("CB import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("CB import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Limpieza
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CB import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("CB import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_customs_broker
|
||||
|
||||
__all__ = ["validate_row_customs_broker"]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de agentes aduanales.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_broker_key,
|
||||
check_optional_license,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de agentes aduanales.
|
||||
CLAVE requerida (max 5, alfanumérica); LICENCIA opcional (max 4 dígitos).
|
||||
"""
|
||||
err = check_required_broker_key(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_license(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila agente aduanal.
|
||||
"""
|
||||
from .common import validate_row_customs_broker
|
||||
|
||||
__all__ = ["validate_row_customs_broker"]
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers for drivers CSV import
|
||||
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (conductores).
|
||||
"""
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
MAX_LEN = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
}
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def check_int_positive(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
v = parse_int(row.get(col))
|
||||
if v is None:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser numerico"}
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser mayor a 0"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_birth_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or not str(val).strip():
|
||||
return None
|
||||
if parse_birth_date(val) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
return None
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Driver (conductores).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
MAX_LEN,
|
||||
parse_int,
|
||||
parse_birth_date,
|
||||
)
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_driver_data(
|
||||
row_norm: Dict[str, Any],
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""Mapea una fila normalizada del CSV a un diccionario para DriverCreateDTO."""
|
||||
transporter_key = _str_or_none(row_norm.get("TRANSPORTISTA"), MAX_LEN["transporter_key"])
|
||||
line = parse_int(row_norm.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
return {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row_norm.get("CLAVE CONDUCTOR"), MAX_LEN["driver_name"]),
|
||||
"license_number": _str_or_none(row_norm.get("LICENCIA"), MAX_LEN["license_number"]),
|
||||
"express_line_id": _str_or_none(row_norm.get("PERMISO LINEA EXPRESS"), MAX_LEN["express_line_id"]),
|
||||
"ace_id": _str_or_none(row_norm.get("IDENTIFICACION ACE"), MAX_LEN["ace_id"]),
|
||||
"birth_date": parse_birth_date(row_norm.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row_norm.get("SEXO"), MAX_LEN["gender"]),
|
||||
"birth_country": _str_or_none(row_norm.get("PAIS NACIMIENTO"), MAX_LEN["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row_norm.get("TRANSPORTA MAT. PELIGROSO?"), MAX_LEN["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row_norm.get("PERMISO MAT. PELIGROSO"), MAX_LEN["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row_norm.get("NOMBRE(S)"), MAX_LEN["first_name"]),
|
||||
"last_name": _str_or_none(row_norm.get("APELLIDO PATERNO"), MAX_LEN["last_name"]),
|
||||
"id_key1": _str_or_none(row_norm.get("FORMA IDENTIFICACION 1"), MAX_LEN["id_key1"]),
|
||||
"id_number1": _str_or_none(row_norm.get("NUM. IDENTIFICACION 1"), MAX_LEN["id_number1"]),
|
||||
"id_state1": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["id_state1"]),
|
||||
"id_country1": _str_or_none(row_norm.get("PAIS"), MAX_LEN["id_country1"]),
|
||||
"id_key2": _str_or_none(row_norm.get("FORMA IDENTIFICACION 2"), MAX_LEN["id_key2"]),
|
||||
"id_number2": _str_or_none(row_norm.get("NUM. IDENTIFICACION 2"), MAX_LEN["id_number2"]),
|
||||
"id_state2": _str_or_none(row_norm.get("ESTADO 2"), MAX_LEN["id_state2"]),
|
||||
"id_country2": _str_or_none(row_norm.get("PAIS 2"), MAX_LEN["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
@@ -1,29 +1,35 @@
|
||||
"""
|
||||
Tareas Celery para importacion CSV de Conductores.
|
||||
Flujo: scan_file (validacion) -> insert_valid_rows (commit).
|
||||
Tareas Celery para importación CSV de Conductores.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe) y clave de estado en Redis.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_driver
|
||||
from .common.mappers import row_to_driver_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "drv"
|
||||
|
||||
# Para routes.py
|
||||
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
|
||||
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
|
||||
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
|
||||
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
|
||||
DRV_IMPORT_REDIS_TTL = 3600
|
||||
DRV_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
@@ -32,147 +38,6 @@ def _get_redis():
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"drv_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
_MAX = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
"badge_number": 20,
|
||||
"class_type": 1,
|
||||
"unique_badge_number": 100,
|
||||
}
|
||||
|
||||
|
||||
def _parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
@@ -180,136 +45,33 @@ def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
unique.append(name if count == 1 else f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
transporter_key = (row.get("TRANSPORTISTA") or "").strip()
|
||||
if not transporter_key:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"}
|
||||
if len(transporter_key) > _MAX["transporter_key"]:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"}
|
||||
|
||||
line_val = _parse_int(row.get("LINEA"))
|
||||
if line_val is None:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"}
|
||||
if line_val <= 0:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"}
|
||||
|
||||
driver_name = (row.get("CLAVE CONDUCTOR") or "").strip()
|
||||
if driver_name and len(driver_name) > _MAX["driver_name"]:
|
||||
return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("LICENCIA", _MAX["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", _MAX["express_line_id"]),
|
||||
("IDENTIFICACION ACE", _MAX["ace_id"]),
|
||||
("SEXO", _MAX["gender"]),
|
||||
("PAIS NACIMIENTO", _MAX["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]),
|
||||
("NOMBRE(S)", _MAX["first_name"]),
|
||||
("APELLIDO PATERNO", _MAX["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", _MAX["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", _MAX["id_number1"]),
|
||||
("ESTADO", _MAX["id_state1"]),
|
||||
("PAIS", _MAX["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", _MAX["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", _MAX["id_number2"]),
|
||||
("ESTADO 2", _MAX["id_state2"]),
|
||||
("PAIS 2", _MAX["id_country2"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
|
||||
fecha = row.get("FECHA NACIMIENTO")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_birth_date(fecha) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA NACIMIENTO",
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"])
|
||||
line = _parse_int(row.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
data = {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]),
|
||||
"license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]),
|
||||
"express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]),
|
||||
"ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]),
|
||||
"birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row.get("SEXO"), _MAX["gender"]),
|
||||
"birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]),
|
||||
"last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]),
|
||||
"id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]),
|
||||
"id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]),
|
||||
"id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]),
|
||||
"id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]),
|
||||
"id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]),
|
||||
"id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]),
|
||||
"id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]),
|
||||
"id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
@@ -333,51 +95,27 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import scan failed: {e}")
|
||||
logger.error("Drivers import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
@@ -390,19 +128,16 @@ def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
logger.warning("Drivers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Drivers import: starting scan for job {job_id}")
|
||||
logger.info("Drivers import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
@@ -413,58 +148,27 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
logger.warning("Drivers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.transportation.drivers.services import DriverService
|
||||
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
|
||||
@@ -475,6 +179,7 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
@@ -497,21 +202,20 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
driver_key = (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-"
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": driver_key,
|
||||
"invoice": driver_key,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = _row_to_driver_dto(row_norm, tenant_id, company_id)
|
||||
data = row_to_driver_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("transporter_key") or data.get("line") is None:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
@@ -519,14 +223,12 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
key = f"{data['transporter_key']}:{data['line']}"
|
||||
if key in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
})
|
||||
continue
|
||||
seen_keys_in_file[key] = i
|
||||
|
||||
@@ -535,7 +237,10 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
)
|
||||
try:
|
||||
if existing:
|
||||
update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")}
|
||||
update_fields = {
|
||||
k: v for k, v in data.items()
|
||||
if k not in ("transporter_key", "line", "company_id", "tenant_id")
|
||||
}
|
||||
for field, value in update_fields.items():
|
||||
setattr(existing, field, value)
|
||||
session.add(existing)
|
||||
@@ -547,34 +252,33 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)}
|
||||
)
|
||||
skipped_details.append({
|
||||
"line": i, "driver_key": key, "invoice": key, "reason": str(db_err),
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Drivers import DB error: {db_err}")
|
||||
logger.error("Drivers import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("Drivers import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Drivers import cleanup failed: {cleanup_err}")
|
||||
r = _get_redis()
|
||||
r.delete(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to delete status key: %s", e)
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
@@ -620,13 +324,13 @@ def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
logger.warning("Drivers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Drivers import: starting commit for job {job_id}")
|
||||
logger.info("Drivers import: starting commit for job %s", job_id)
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
@@ -636,5 +340,5 @@ def insert_valid_rows(self, job_id: str):
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
logger.warning("Drivers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_driver
|
||||
|
||||
__all__ = ["validate_row_driver"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de conductores.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
MAX_LEN,
|
||||
check_max_length,
|
||||
check_int_positive,
|
||||
check_optional_birth_date,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_driver_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_max_length(
|
||||
row, "TRANSPORTISTA", MAX_LEN["transporter_key"], line_num, required=True
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return check_int_positive(row, "LINEA", line_num)
|
||||
|
||||
|
||||
def validate_row_driver_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("CLAVE CONDUCTOR", MAX_LEN["driver_name"]),
|
||||
("LICENCIA", MAX_LEN["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", MAX_LEN["express_line_id"]),
|
||||
("IDENTIFICACION ACE", MAX_LEN["ace_id"]),
|
||||
("SEXO", MAX_LEN["gender"]),
|
||||
("PAIS NACIMIENTO", MAX_LEN["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", MAX_LEN["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", MAX_LEN["hazardous_material_state"]),
|
||||
("NOMBRE(S)", MAX_LEN["first_name"]),
|
||||
("APELLIDO PATERNO", MAX_LEN["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", MAX_LEN["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", MAX_LEN["id_number1"]),
|
||||
("ESTADO", MAX_LEN["id_state1"]),
|
||||
("PAIS", MAX_LEN["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", MAX_LEN["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", MAX_LEN["id_number2"]),
|
||||
("ESTADO 2", MAX_LEN["id_state2"]),
|
||||
("PAIS 2", MAX_LEN["id_country2"]),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_driver_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_optional_birth_date(row, "FECHA NACIMIENTO", line_num)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila conductor.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import (
|
||||
validate_row_driver_required,
|
||||
validate_row_driver_lengths,
|
||||
validate_row_driver_date,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de conductores.
|
||||
Encadena: requeridos (TRANSPORTISTA, LINEA) → longitudes → fecha opcional.
|
||||
"""
|
||||
err = validate_row_driver_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_driver_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_driver_date(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for exchange_rate)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de tipos de cambio.
|
||||
"""
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
DATE_FORMATS: List[str] = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
CURRENCY_MAX = 7
|
||||
|
||||
|
||||
def parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def parse_decimal_positive(val: Optional[str]) -> Optional[Decimal]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
try:
|
||||
v = float(str(val).strip().replace(",", "."))
|
||||
if v <= 0:
|
||||
return None
|
||||
return Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def check_required_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
return None
|
||||
|
||||
|
||||
def check_required_value_positive(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para ExchangeRate.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
parse_date,
|
||||
parse_decimal_positive,
|
||||
)
|
||||
|
||||
CURRENCY_MAX = 7
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_exchange_rate_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for ExchangeRate model. Returns {} if FECHA or VALOR invalid."""
|
||||
parsed_date = parse_date(row_norm.get("FECHA"))
|
||||
value_decimal = parse_decimal_positive(row_norm.get("VALOR"))
|
||||
if not parsed_date or value_decimal is None:
|
||||
return {}
|
||||
local = _str_or_none(row_norm.get("MONEDA_LOCAL"), CURRENCY_MAX)
|
||||
if local:
|
||||
local = local.upper()
|
||||
foreign = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), CURRENCY_MAX)
|
||||
if foreign:
|
||||
foreign = foreign.upper()
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"date": parsed_date,
|
||||
"value": value_decimal,
|
||||
"local_currency": local,
|
||||
"foreign_currency": foreign,
|
||||
}
|
||||
@@ -1,322 +1,127 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_exchange_rate
|
||||
from .common.mappers import row_to_exchange_rate_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "er"
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
# Para routes.py
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
ER_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if _parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
|
||||
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
|
||||
if local_raw and len(local_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
|
||||
if foreign_raw and len(foreign_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"ER import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import scan failed: {e}")
|
||||
logger.error("ER import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
|
||||
"""
|
||||
logger.info(f"ER import: starting commit for job {job_id}")
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("ER import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"ER import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
@@ -332,134 +137,91 @@ def insert_valid_rows(self, job_id: str):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
data = row_to_exchange_rate_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("date"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA o VALOR no válidos"})
|
||||
continue
|
||||
|
||||
parsed_date = _parse_date(row_norm.get("FECHA"))
|
||||
if not parsed_date:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
|
||||
continue
|
||||
|
||||
try:
|
||||
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
|
||||
value_decimal = Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
|
||||
continue
|
||||
|
||||
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
|
||||
if local_currency:
|
||||
local_currency = local_currency.upper()
|
||||
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
|
||||
if foreign_currency:
|
||||
foreign_currency = foreign_currency.upper()
|
||||
|
||||
key_date = parsed_date.date()
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
|
||||
if existing:
|
||||
existing.value = value_decimal
|
||||
existing.local_currency = local_currency or None
|
||||
existing.foreign_currency = foreign_currency or None
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
date=parsed_date,
|
||||
value=value_decimal,
|
||||
local_currency=local_currency,
|
||||
foreign_currency=foreign_currency,
|
||||
)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
key_date = data["date"].date() if hasattr(data["date"], "date") else data["date"]
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
if existing:
|
||||
existing.value = data["value"]
|
||||
existing.local_currency = data.get("local_currency")
|
||||
existing.foreign_currency = data.get("foreign_currency")
|
||||
session.add(existing)
|
||||
else:
|
||||
new_er = ExchangeRate(**data)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"ER import DB error: {db_err}")
|
||||
logger.error("ER import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("ER import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"ER import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("ER import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_exchange_rate
|
||||
|
||||
__all__ = ["validate_row_exchange_rate"]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de tipos de cambio.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_date,
|
||||
check_required_value_positive,
|
||||
check_optional_max_length,
|
||||
CURRENCY_MAX,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de tipos de cambio.
|
||||
FECHA y VALOR requeridos; MONEDA_LOCAL y MONEDA_EXTRANJERA opcionales (max 7).
|
||||
"""
|
||||
err = check_required_date(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_required_value_positive(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "MONEDA_LOCAL", CURRENCY_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "MONEDA_EXTRANJERA", CURRENCY_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila tipo de cambio.
|
||||
"""
|
||||
from .common import validate_row_exchange_rate
|
||||
|
||||
__all__ = ["validate_row_exchange_rate"]
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import base64
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import csv
|
||||
@@ -13,86 +12,35 @@ from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process
|
||||
|
||||
# We'll need schemas for validation
|
||||
# from api.v1.modules.a76.invoices.schemas import InvoiceHeaderCreate
|
||||
# But for Phase 1 we use a lighter check
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys and TTL for import file/meta (shared between API and worker when no shared filesystem)
|
||||
# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py)
|
||||
JOB_TYPE = ""
|
||||
|
||||
# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="")
|
||||
IMPORT_FILE_KEY_PREFIX = "import_file:"
|
||||
IMPORT_META_KEY_PREFIX = "import_meta:"
|
||||
IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:"
|
||||
IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Redis client using same URL as Celery broker (worker and API can share data)."""
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
"""Directory on the worker for temp CSV and meta (same structure as API, but local to worker)."""
|
||||
return layout_path("imports", "temp")
|
||||
IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
"""
|
||||
Load file content from Redis and write to worker's upload dir.
|
||||
Returns local file_path if successful, None otherwise.
|
||||
"""
|
||||
redis_client = _get_redis()
|
||||
key = f"{IMPORT_FILE_KEY_PREFIX}{job_id}"
|
||||
data = redis_client.get(key)
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to decode import file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
"""Usa common storage con job_type vacío (prefijo import_)."""
|
||||
return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import")
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
"""Load meta from Redis and write to worker's meta file. Returns True if meta was found and written."""
|
||||
redis_client = _get_redis()
|
||||
key = f"{IMPORT_META_KEY_PREFIX}{job_id}"
|
||||
data = redis_client.get(key)
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to decode import meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import")
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
"""Remove file, meta and error lines from Redis after commit (cleanup)."""
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
|
||||
f"{IMPORT_META_KEY_PREFIX}{job_id}",
|
||||
f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete import keys from Redis: {e}")
|
||||
common_storage.delete_import_from_redis(JOB_TYPE, job_id)
|
||||
|
||||
class ForeignKeyValidator:
|
||||
def __init__(self, session, tenant_id, company_id):
|
||||
@@ -189,10 +137,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
# 2. Setup Error Log
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
error_count = 0
|
||||
@@ -213,22 +158,12 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
date_format = "yyyy-mm-dd" # Default to ISO format
|
||||
logger.info(f"No date_format specified in config, using default: {date_format}")
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f_meta:
|
||||
meta = json.load(f_meta) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read meta for job {job_id}: {e}")
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Missing context (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
template_id = meta.get("template_id") or (
|
||||
"imp_temp_header" if model_target == "invoice_header" else "imp_temp_details"
|
||||
)
|
||||
@@ -337,24 +272,11 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store error lines in Redis: {e}")
|
||||
|
||||
# 5. Result (incluye lista de errores para que el usuario pueda corregir el CSV)
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
def validate_row_phase_1(
|
||||
row: Dict[str, Any],
|
||||
@@ -772,13 +694,23 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
# Ensure we have the file on this worker: prefer Redis (so any worker can run commit)
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
upload_dir = _worker_upload_dir()
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
if not os.path.exists(file_path):
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
from api.v1.modules.a76.invoices.models import (
|
||||
InvoiceHeader,
|
||||
@@ -800,67 +732,16 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
|
||||
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
|
||||
from api.v1.modules.a76.items.models import LineItem
|
||||
from api.v1.modules.a76.items.models import LineItem
|
||||
from api.v1.modules.a76.items.line_financials.models import LineFinancial
|
||||
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
|
||||
from api.v1.modules.a76.items.line_customs.models import LineCustom
|
||||
from api.v1.modules.a76.items.line_descriptions.models import LineDescription
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"{job_id}.jsonl")
|
||||
footer_config = parse_footer_config(meta.get("footer_config"))
|
||||
|
||||
# 1. Load Error Line Numbers (from Redis if scan ran on another worker, else from file)
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{IMPORT_ERROR_LINES_KEY_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err['line'])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Load Metadata (Context)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
footer_config = {}
|
||||
meta = {}
|
||||
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get('tenant_id')
|
||||
company_id = meta.get('company_id')
|
||||
operation_type_raw = meta.get('operation_type', 'imp')
|
||||
footer_config = parse_footer_config(meta.get('footer_config'))
|
||||
except: pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Missing context (tenant/company)"}
|
||||
|
||||
# 2. Re-read and Map
|
||||
# Initialize counters outside the session block so they're accessible later
|
||||
headers_to_insert = []
|
||||
details_to_insert = []
|
||||
skipped_invalid = 0
|
||||
skipped_missing_invoice = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_fk_details = []
|
||||
inserted_count = 0
|
||||
response = None # Will be set inside the session block
|
||||
|
||||
date_format = footer_config.get("dateFormat")
|
||||
|
||||
# Validate and set default date_format if not provided
|
||||
if not date_format:
|
||||
date_format = "yyyy-mm-dd" # Default to ISO format
|
||||
@@ -874,6 +755,15 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
|
||||
logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}")
|
||||
|
||||
headers_to_insert = []
|
||||
details_to_insert = []
|
||||
skipped_invalid = 0
|
||||
skipped_missing_invoice = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_fk_details = []
|
||||
inserted_count = 0
|
||||
response = None
|
||||
|
||||
with CoreSessionLocal() as session:
|
||||
invoice_id_cache = {}
|
||||
cleared_invoices = set() # Track invoices where we've already cleared items in this job
|
||||
@@ -1462,11 +1352,12 @@ def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
|
||||
# 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err)
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# common validators for parts CSV import
|
||||
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (partes).
|
||||
Cada función devuelve Optional[Dict] con keys line, col, msg.
|
||||
Referencia: a76/invoices (common_validators, validators/common).
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(
|
||||
row: Dict[str, Any], col: str, line_num: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Devuelve error si el campo está vacío."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Devuelve error si el campo excede max_len. Si required=True, también exige que haya valor."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_decimal(
|
||||
row: Dict[str, Any], col: str, line_num: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Devuelve error si el valor no es un número decimal válido (solo cuando hay valor)."""
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación/mapeo de import CSV de partes.
|
||||
"""
|
||||
from typing import Set, Tuple
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_parts_fk_sets(
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Tuple[Set[str], Set[str], Set[str]]:
|
||||
"""
|
||||
Carga valid_class_codes, valid_uom_codes, valid_currency_codes desde BD.
|
||||
Devuelve (valid_class_codes, valid_uom_codes, valid_currency_codes).
|
||||
"""
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("Parts import: could not load FK sets: %s", e)
|
||||
return valid_class_codes, valid_uom_codes, valid_currency_codes
|
||||
111
backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py
Normal file
111
backend/api/v1/modules/a76/layouts_csv/parts/common/mappers.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Part. Helpers de normalización de valores.
|
||||
Referencia: a76/invoices common/mappers.py
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def clean_dict(data_dict: dict) -> dict:
|
||||
"""Convierte strings vacíos a None; mantiene 0 en campos no-_id. Igual que a76/invoices."""
|
||||
cleaned = {}
|
||||
for key, value in data_dict.items():
|
||||
if isinstance(value, str) and not value.strip():
|
||||
cleaned[key] = None
|
||||
elif value == 0 and (key.endswith("_id") or key == "remesa"):
|
||||
cleaned[key] = None
|
||||
else:
|
||||
cleaned[key] = value
|
||||
return cleaned
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _bool_from_row(val: Any) -> bool:
|
||||
if val is None or val == "":
|
||||
return True
|
||||
s = str(val).strip().upper()
|
||||
if s in ("0", "F", "FALSE", "NO", "N"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def row_to_part_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_class_codes: Set[str],
|
||||
valid_uom_codes: Set[str],
|
||||
valid_currency_codes: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para Part.
|
||||
Ajusta FKs opcionales (part_class, unit_of_measure, currency_key) a None si no están en los conjuntos.
|
||||
El caller debe añadir tenant_id, company_id, client_id al crear Part.
|
||||
"""
|
||||
part_number = _str_or_none(row_norm.get("NUMPARTE"), 70)
|
||||
commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
part_class = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if part_class and part_class not in valid_class_codes:
|
||||
part_class = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
currency_key = _str_or_none(row_norm.get("MONEDA"), 3)
|
||||
if currency_key and currency_key not in valid_currency_codes:
|
||||
currency_key = None
|
||||
|
||||
unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT"))
|
||||
currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None
|
||||
unit_weight = _decimal_or_none(row_norm.get("PESOUNIT"))
|
||||
weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6)
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 10)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
fda_key = _str_or_none(row_norm.get("FDAKEY"), 20)
|
||||
fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30)
|
||||
license_code = _str_or_none(row_norm.get("LICENCIA"), 3)
|
||||
eccn = _str_or_none(row_norm.get("ECCN"), 20)
|
||||
export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2)
|
||||
exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19)
|
||||
is_active = _bool_from_row(row_norm.get("ACTIVO"))
|
||||
|
||||
return {
|
||||
"part_number": part_number,
|
||||
"commercial_part_number": commercial,
|
||||
"description_spanish": desc_es,
|
||||
"description_english": desc_en,
|
||||
"part_class": part_class,
|
||||
"unit_of_measure": unit_of_measure,
|
||||
"unit_cost": unit_cost,
|
||||
"currency_type": currency_type,
|
||||
"currency_key": currency_key,
|
||||
"unit_weight": unit_weight,
|
||||
"weight_type": weight_type,
|
||||
"fraction": fraction,
|
||||
"us_fraction": us_fraction,
|
||||
"fda_key": fda_key,
|
||||
"fcc_key": fcc_key,
|
||||
"license_code": license_code,
|
||||
"eccn": eccn,
|
||||
"export_code": export_code,
|
||||
"exclusion_symbol": exclusion_symbol,
|
||||
"is_active": is_active,
|
||||
}
|
||||
@@ -1,270 +1,75 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Números de Parte.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Orquestación usa common (storage, normalize, csv_reader, meta, responses) y common.fk_loader.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_part
|
||||
from .common.mappers import row_to_part_data
|
||||
from .common.fk_loader import load_parts_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "part"
|
||||
|
||||
# Para routes.py (upload guarda con estos prefijos)
|
||||
PART_IMPORT_FILE_PREFIX = "part_import_file:"
|
||||
PART_IMPORT_META_PREFIX = "part_import_meta:"
|
||||
PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:"
|
||||
PART_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"part_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PART_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_part(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_class_codes: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
valid_currency_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
part_number = (row.get("NUMPARTE") or "").strip()
|
||||
if not part_number:
|
||||
return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"}
|
||||
if len(part_number) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
commercial = (row.get("NUMPARTECOM") or "").strip()
|
||||
if commercial and len(commercial) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
part_class = (row.get("CLASE") or "").strip()
|
||||
if part_class and len(part_class) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
# Si CLASE no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
# Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
currency_key = (row.get("MONEDA") or "").strip()
|
||||
if currency_key and len(currency_key) > 3:
|
||||
return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"}
|
||||
# Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
unit_cost = row.get("COSTOUNIT")
|
||||
if unit_cost is not None and unit_cost != "":
|
||||
try:
|
||||
Decimal(str(unit_cost))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"}
|
||||
|
||||
unit_weight = row.get("PESOUNIT")
|
||||
if unit_weight is not None and unit_weight != "":
|
||||
try:
|
||||
Decimal(str(unit_weight))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 10:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
fda_key = (row.get("FDAKEY") or "").strip()
|
||||
if fda_key and len(fda_key) > 20:
|
||||
return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"}
|
||||
fcc_key = (row.get("FCCKEY") or "").strip()
|
||||
if fcc_key and len(fcc_key) > 30:
|
||||
return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"}
|
||||
license_code = (row.get("LICENCIA") or "").strip()
|
||||
if license_code and len(license_code) > 3:
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"}
|
||||
eccn = (row.get("ECCN") or "").strip()
|
||||
if eccn and len(eccn) > 20:
|
||||
return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"}
|
||||
export_code = (row.get("EXPORTCODE") or "").strip()
|
||||
if export_code and len(export_code) > 2:
|
||||
return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"}
|
||||
exclusion = (row.get("EXCLUSION") or "").strip()
|
||||
if exclusion and len(exclusion) > 19:
|
||||
return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"}
|
||||
weight_type = (row.get("TIPOPESO") or "").strip()
|
||||
if weight_type and len(weight_type) > 6:
|
||||
return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"}
|
||||
|
||||
return None
|
||||
PART_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Parts import: starting scan for job {job_id}")
|
||||
logger.info("Parts import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"part_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: could not load FK sets: {e}")
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_class_codes, valid_uom_codes, valid_currency_codes = load_parts_fk_sets(tenant_id, company_id)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_part(
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
@@ -273,170 +78,51 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import scan failed: {e}")
|
||||
logger.error("Parts import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _bool_from_row(val: Any) -> bool:
|
||||
if val is None or val == "":
|
||||
return True
|
||||
s = str(val).strip().upper()
|
||||
if s in ("0", "F", "FALSE", "NO", "N"):
|
||||
return False
|
||||
return True
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Parts import: starting commit for job {job_id}")
|
||||
logger.info("Parts import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"part_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Parts import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: could not load FK sets: {e}")
|
||||
valid_class_codes, valid_uom_codes, valid_currency_codes = load_parts_fk_sets(tenant_id, company_id)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
@@ -444,196 +130,112 @@ def insert_valid_rows(self, job_id: str):
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_part_number: Dict[str, Part] = {}
|
||||
for p in (
|
||||
session.query(Part)
|
||||
.filter(
|
||||
existing_by_part_number = {
|
||||
p.part_number: p
|
||||
for p in session.query(Part).filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
).all()
|
||||
}
|
||||
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_part_number[p.part_number] = p
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
data = row_to_part_data(
|
||||
row_norm,
|
||||
valid_class_codes,
|
||||
valid_uom_codes,
|
||||
valid_currency_codes,
|
||||
)
|
||||
part_number = data.get("part_number")
|
||||
if not part_number:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
existing = existing_by_part_number.get(part_number)
|
||||
if existing:
|
||||
for key, value in data.items():
|
||||
setattr(existing, key, value)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_part = Part(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
client_id=company_id,
|
||||
**data,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
part_number = _str_or_none(row_norm.get("NUMPARTE"), 70)
|
||||
if not part_number:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
part_class = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if part_class and part_class not in valid_class_codes:
|
||||
part_class = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
currency_key = _str_or_none(row_norm.get("MONEDA"), 3)
|
||||
if currency_key and currency_key not in valid_currency_codes:
|
||||
currency_key = None
|
||||
|
||||
unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT"))
|
||||
currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None
|
||||
unit_weight = _decimal_or_none(row_norm.get("PESOUNIT"))
|
||||
weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6)
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 10)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
fda_key = _str_or_none(row_norm.get("FDAKEY"), 20)
|
||||
fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30)
|
||||
license_code = _str_or_none(row_norm.get("LICENCIA"), 3)
|
||||
eccn = _str_or_none(row_norm.get("ECCN"), 20)
|
||||
export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2)
|
||||
exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19)
|
||||
is_active = _bool_from_row(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_part_number.get(part_number)
|
||||
if existing:
|
||||
existing.commercial_part_number = commercial
|
||||
existing.description_spanish = desc_es
|
||||
existing.description_english = desc_en
|
||||
existing.part_class = part_class
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.unit_cost = unit_cost
|
||||
existing.currency_type = currency_type
|
||||
existing.currency_key = currency_key
|
||||
existing.unit_weight = unit_weight
|
||||
existing.weight_type = weight_type
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.fda_key = fda_key
|
||||
existing.fcc_key = fcc_key
|
||||
existing.license_code = license_code
|
||||
existing.eccn = eccn
|
||||
existing.export_code = export_code
|
||||
existing.exclusion_symbol = exclusion_symbol
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_part = Part(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
client_id=company_id,
|
||||
part_number=part_number,
|
||||
commercial_part_number=commercial,
|
||||
description_spanish=desc_es,
|
||||
description_english=desc_en,
|
||||
part_class=part_class,
|
||||
unit_of_measure=unit_of_measure,
|
||||
unit_cost=unit_cost,
|
||||
currency_type=currency_type,
|
||||
currency_key=currency_key,
|
||||
unit_weight=unit_weight,
|
||||
weight_type=weight_type,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
fda_key=fda_key,
|
||||
fcc_key=fcc_key,
|
||||
license_code=license_code,
|
||||
eccn=eccn,
|
||||
export_code=export_code,
|
||||
exclusion_symbol=exclusion_symbol,
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_part)
|
||||
existing_by_part_number[part_number] = new_part
|
||||
inserted_count += 1
|
||||
session.add(new_part)
|
||||
existing_by_part_number[part_number] = new_part
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Parts import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
logger.error("Parts import DB error: %s", db_err)
|
||||
return common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
||||
skipped_details, error=str(db_err),
|
||||
)
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"warning", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
||||
skipped_details,
|
||||
message=f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
)
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
||||
skipped_details,
|
||||
error="No hay registros válidos en el archivo CSV",
|
||||
)
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, skipped_missing_fk,
|
||||
skipped_duplicate, skipped_details,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
logger.exception("Parts import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
||||
skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Parts import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
||||
skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# validators for parts CSV import row validation
|
||||
from .create import validate_row_part
|
||||
|
||||
__all__ = ["validate_row_part"]
|
||||
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de partes.
|
||||
Usa helpers de common.common_validators; agrupa por tipo (requeridos, longitudes, tipos).
|
||||
Referencia: a76/invoices imports/temporary/validators/common.py
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import check_max_length, check_decimal
|
||||
|
||||
|
||||
def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida campos obligatorios de una fila de parte."""
|
||||
return check_max_length(row, "NUMPARTE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida longitudes máximas de todos los campos de texto."""
|
||||
checks = [
|
||||
("NUMPARTECOM", 70),
|
||||
("DESCRIPCIONE", 500),
|
||||
("DESCRIPCIONI", 500),
|
||||
("CLASE", 8),
|
||||
("UNIMED", 5),
|
||||
("MONEDA", 3),
|
||||
("FRACCION", 10),
|
||||
("FRACCIONAME", 16),
|
||||
("FDAKEY", 20),
|
||||
("FCCKEY", 30),
|
||||
("LICENCIA", 3),
|
||||
("ECCN", 20),
|
||||
("EXPORTCODE", 2),
|
||||
("EXCLUSION", 19),
|
||||
("TIPOPESO", 6),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida tipos numéricos (decimales) de la fila."""
|
||||
err = check_decimal(row, "COSTOUNIT", line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_decimal(row, "PESOUNIT", line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,39 @@
|
||||
"""
|
||||
Punto de entrada de validación para creación/import de una fila de parte.
|
||||
Encadena validaciones comunes (requeridos, longitudes, tipos).
|
||||
Referencia: a76/invoices imports/temporary/validators/create.py
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required,
|
||||
validate_row_lengths,
|
||||
validate_row_types,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_part(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_class_codes: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
valid_currency_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de partes para import.
|
||||
Encadena: requeridos → longitudes → tipos.
|
||||
Los conjuntos valid_* se mantienen en la firma por compatibilidad con el caller.
|
||||
"""
|
||||
err = validate_row_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
err = validate_row_types(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers, fk_loader
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de pedimentos.
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_int_in_set(
|
||||
row: Dict[str, Any], col: str, valid_ids: Set[int], line_num: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
try:
|
||||
client_id = int(val)
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número entero"}
|
||||
if client_id not in valid_ids:
|
||||
return {"line": line_num, "col": col, "msg": "Cliente no existe en catálogo"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
valid_set: Set[str],
|
||||
line_num: int,
|
||||
max_len: int = 10,
|
||||
catalog_name: str = "catálogo",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
if valid_set and val not in valid_set:
|
||||
return {"line": line_num, "col": col, "msg": f"No existe en {catalog_name}"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_decimal(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
try:
|
||||
Decimal(val.replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return {"line": line_num, "col": col, "msg": "Valor numérico inválido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_in_set(
|
||||
row: Dict[str, Any], col: str, allowed: Set[str], line_num: int, msg: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip().lower()
|
||||
if not val:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
|
||||
|
||||
PEDIMENTO_CODE_MAX = 2
|
||||
REGIMEN_MAX = 3
|
||||
@@ -0,0 +1,38 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación de import CSV de pedimentos.
|
||||
"""
|
||||
from typing import Set, Tuple
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
||||
def load_pedimentos_fk_sets(
|
||||
session: Session, tenant_id: int, company_id: int
|
||||
) -> Tuple[Set[int], Set[str], Set[str]]:
|
||||
"""
|
||||
Carga valid_client_ids (ClientProvider.id), valid_regimes (RegimenPedimento.code),
|
||||
valid_pedimento_codes (PedimentoCode.code).
|
||||
"""
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
|
||||
return valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para PedimentosCreate.
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).strip().replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def row_to_pedimento_data(row_norm: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict for PedimentosCreate from normalized CSV row."""
|
||||
year = (row_norm.get("AÑO") or "").strip()[:2]
|
||||
customs_office = (row_norm.get("ADUANA") or "").strip()[:3]
|
||||
license_val = (row_norm.get("PATENTE") or "").strip()[:4]
|
||||
pedimento_number = (row_norm.get("NUMERO") or "").strip()[:7]
|
||||
client_id_str = (row_norm.get("CLIENTE_ID") or "").strip()
|
||||
client_id = int(client_id_str) if client_id_str else None
|
||||
pedimento_code = (row_norm.get("CODIGO_PEDIMENTO") or "").strip()[:2]
|
||||
regime = (row_norm.get("REGIMEN") or "").strip()[:3]
|
||||
|
||||
data = {
|
||||
"year": year,
|
||||
"customs_office": customs_office,
|
||||
"license": license_val,
|
||||
"pedimento_number": pedimento_number,
|
||||
"client_id": client_id,
|
||||
"pedimento_code": pedimento_code,
|
||||
"regime": regime,
|
||||
}
|
||||
|
||||
op = (row_norm.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if op in ("imp", "exp"):
|
||||
data["operation_type"] = op
|
||||
|
||||
ptype = (row_norm.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if ptype in ("normal", "consolidated", "complementary", "automobile"):
|
||||
data["pedimento_type"] = ptype
|
||||
|
||||
status = (row_norm.get("ESTATUS") or "").strip()
|
||||
if status:
|
||||
data["status"] = status[:30]
|
||||
|
||||
data["usd_value"] = parse_decimal(row_norm.get("VALOR_USD"))
|
||||
data["paid_price"] = parse_decimal(row_norm.get("PRECIO_PAGADO"))
|
||||
data["gross_weight"] = parse_decimal(row_norm.get("PESO_BRUTO"))
|
||||
data["exchange_rate"] = parse_decimal(row_norm.get("TIPO_CAMBIO"))
|
||||
|
||||
obs = (row_norm.get("OBSERVACIONES") or "").strip()
|
||||
if obs:
|
||||
data["observations"] = obs
|
||||
|
||||
return data
|
||||
@@ -1,445 +1,142 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Pedimentos.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader), fk_loader, validators, mappers.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_pedimento
|
||||
from .common.fk_loader import load_pedimentos_fk_sets
|
||||
from .common.mappers import row_to_pedimento_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con otros imports)
|
||||
JOB_TYPE = "ped"
|
||||
TEMPLATE_ID = "pedimentos"
|
||||
|
||||
# Para routes.py
|
||||
PED_IMPORT_FILE_PREFIX = "ped_import_file:"
|
||||
PED_IMPORT_META_PREFIX = "ped_import_meta:"
|
||||
PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:"
|
||||
PED_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
PED_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"ped_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_pedimento(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_client_ids: Set[int],
|
||||
valid_regimes: Set[str],
|
||||
valid_pedimento_codes: Set[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Pedimento. Retorna error dict o None."""
|
||||
# Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN
|
||||
year = (row.get("AÑO") or "").strip()
|
||||
if not year:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Requerido"}
|
||||
if len(year) > 2:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"}
|
||||
|
||||
customs_office = (row.get("ADUANA") or "").strip()
|
||||
if not customs_office:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Requerido"}
|
||||
if len(customs_office) > 3:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"}
|
||||
|
||||
license_val = (row.get("PATENTE") or "").strip()
|
||||
if not license_val:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Requerido"}
|
||||
if len(license_val) > 4:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()
|
||||
if not pedimento_number:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Requerido"}
|
||||
if len(pedimento_number) > 7:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
if not client_id_str:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"}
|
||||
try:
|
||||
client_id = int(client_id_str)
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"}
|
||||
if client_id not in valid_client_ids:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"}
|
||||
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()
|
||||
if not pedimento_code:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"}
|
||||
if len(pedimento_code) > 2:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"}
|
||||
if pedimento_code not in valid_pedimento_codes:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"}
|
||||
|
||||
regime = (row.get("REGIMEN") or "").strip()
|
||||
if not regime:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"}
|
||||
if len(regime) > 3:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"}
|
||||
if regime not in valid_regimes:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"}
|
||||
|
||||
# Optional numeric/string fields - validate format if present
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status and len(status) > 30:
|
||||
return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
continue
|
||||
try:
|
||||
Decimal(val.replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return {"line": line_num, "col": col, "msg": "Valor numérico inválido"}
|
||||
|
||||
operation_type = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if operation_type and operation_type not in ("imp", "exp", ""):
|
||||
return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"}
|
||||
|
||||
pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""):
|
||||
return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).strip().replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict for PedimentosCreate from normalized CSV row (canonical names)."""
|
||||
year = (row.get("AÑO") or "").strip()[:2]
|
||||
customs_office = (row.get("ADUANA") or "").strip()[:3]
|
||||
license_val = (row.get("PATENTE") or "").strip()[:4]
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()[:7]
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
client_id = int(client_id_str) if client_id_str else None
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2]
|
||||
regime = (row.get("REGIMEN") or "").strip()[:3]
|
||||
|
||||
data = {
|
||||
"year": year,
|
||||
"customs_office": customs_office,
|
||||
"license": license_val,
|
||||
"pedimento_number": pedimento_number,
|
||||
"client_id": client_id,
|
||||
"pedimento_code": pedimento_code,
|
||||
"regime": regime,
|
||||
}
|
||||
|
||||
op = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if op in ("imp", "exp"):
|
||||
data["operation_type"] = op
|
||||
|
||||
ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if ptype in ("normal", "consolidated", "complementary", "automobile"):
|
||||
data["pedimento_type"] = ptype
|
||||
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status:
|
||||
data["status"] = status[:30]
|
||||
|
||||
data["usd_value"] = _parse_decimal(row.get("VALOR_USD"))
|
||||
data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO"))
|
||||
data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO"))
|
||||
data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO"))
|
||||
|
||||
obs = (row.get("OBSERVACIONES") or "").strip()
|
||||
if obs:
|
||||
data["observations"] = obs
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to read meta: {e}")
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Load valid FK sets for validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
valid_client_ids, valid_regimes, valid_pedimento_codes = load_pedimentos_fk_sets(
|
||||
session, tenant_id, company_id
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"}
|
||||
logger.error("Pedimentos import: failed to load FK sets: %s", e)
|
||||
return {"status": "failed", "error": "No se pudo cargar catálogos"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import scan failed: {e}")
|
||||
logger.error("Pedimentos import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting commit for job {job_id}")
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("Pedimentos import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Pedimentos import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Pedimentos import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Reload FK sets for commit-time validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
valid_client_ids, valid_regimes, valid_pedimento_codes = load_pedimentos_fk_sets(
|
||||
session, tenant_id, company_id
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
logger.error("Pedimentos import: failed to load FK sets: %s", e)
|
||||
return {"status": "failed", "error": "No se pudo cargar catálogos"}
|
||||
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate
|
||||
@@ -450,114 +147,94 @@ def insert_valid_rows(self, job_id: str):
|
||||
skipped_missing_fk = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
data = row_to_pedimento_data(row_norm)
|
||||
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
||||
data["pedimento_dates"] = PedimentoDatesCreate(
|
||||
entry_date=datetime.now(),
|
||||
end_date=datetime.now(),
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
data = _row_to_pedimentos_create(row_norm)
|
||||
# Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL
|
||||
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
||||
data["pedimento_dates"] = PedimentoDatesCreate(
|
||||
entry_date=datetime.now(),
|
||||
end_date=datetime.now(),
|
||||
)
|
||||
create_data = PedimentosCreate(**data)
|
||||
PedimentosService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except ValueError as ve:
|
||||
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
else:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import line {i}: {e}")
|
||||
create_data = PedimentosCreate(**data)
|
||||
PedimentosService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except ValueError as ve:
|
||||
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
else:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(e)})
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
except Exception as e:
|
||||
logger.warning("Pedimentos import line %s: %s", i, e)
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(e)})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("Pedimentos import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("Pedimentos import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_pedimento
|
||||
|
||||
__all__ = ["validate_row_pedimento"]
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de pedimentos (por tipo).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_max,
|
||||
check_int_in_set,
|
||||
check_in_set,
|
||||
check_optional_max_length,
|
||||
check_optional_decimal,
|
||||
check_optional_in_set,
|
||||
PEDIMENTO_CODE_MAX,
|
||||
REGIMEN_MAX,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_pedimento_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
for col, max_len in [
|
||||
("AÑO", 2),
|
||||
("ADUANA", 3),
|
||||
("PATENTE", 4),
|
||||
("NUMERO", 7),
|
||||
]:
|
||||
err = check_required_max(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_pedimento_fk(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_client_ids: Set[int],
|
||||
valid_regimes: Set[str],
|
||||
valid_pedimento_codes: Set[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
err = check_int_in_set(row, "CLIENTE_ID", valid_client_ids, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_in_set(
|
||||
row, "CODIGO_PEDIMENTO", valid_pedimento_codes, line_num,
|
||||
max_len=PEDIMENTO_CODE_MAX, catalog_name="código pedimento",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_in_set(
|
||||
row, "REGIMEN", valid_regimes, line_num,
|
||||
max_len=REGIMEN_MAX, catalog_name="régimen",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_pedimento_optionals(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_optional_max_length(row, "ESTATUS", 30, line_num)
|
||||
if err:
|
||||
return err
|
||||
for col in ("VALOR_USD", "PRECIO_PAGADO", "PESO_BRUTO", "TIPO_CAMBIO"):
|
||||
err = check_optional_decimal(row, col, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_in_set(
|
||||
row, "TIPO_OPERACION", {"imp", "exp"}, line_num,
|
||||
"Debe ser imp o exp",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_in_set(
|
||||
row, "TIPO_PEDIMENTO",
|
||||
{"normal", "consolidated", "complementary", "automobile"},
|
||||
line_num,
|
||||
"Tipo no válido (normal, consolidated, complementary, automobile)",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,37 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila pedimento.
|
||||
Recibe conjuntos FK (valid_client_ids, valid_regimes, valid_pedimento_codes) para validar referencias.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_pedimento_required,
|
||||
validate_row_pedimento_fk,
|
||||
validate_row_pedimento_optionals,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_pedimento(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_client_ids: Set[int],
|
||||
valid_regimes: Set[str],
|
||||
valid_pedimento_codes: Set[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de pedimentos.
|
||||
Requeridos: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN (y FKs en catálogos).
|
||||
Opcionales: ESTATUS, decimales, TIPO_OPERACION, TIPO_PEDIMENTO.
|
||||
"""
|
||||
err = validate_row_pedimento_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_pedimento_fk(
|
||||
row, line_num, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_pedimento_optionals(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for trailers)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de trailers (longitudes, requerido).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
# Max lengths from Trailer model (a76.trailer)
|
||||
MAX_LEN = {
|
||||
"trailer_number": 20,
|
||||
"ace_trailer_number": 10,
|
||||
"trailer_type_key": 2,
|
||||
"seal": 15,
|
||||
"entity_code": 1,
|
||||
"plate_number": 17,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"container_key": 3,
|
||||
}
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any], col: str, max_len: int, line_num: int, required: bool = False
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Trailer.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import MAX_LEN
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_trailer_data(row_norm: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row."""
|
||||
trailer_number = _str_or_none(row_norm.get("NUMERO TRAILER"), MAX_LEN["trailer_number"])
|
||||
if not trailer_number:
|
||||
return {}
|
||||
return {
|
||||
"trailer_number": trailer_number,
|
||||
"ace_trailer_number": _str_or_none(row_norm.get("CLAVE ACE"), MAX_LEN["ace_trailer_number"]),
|
||||
"trailer_type_key": _str_or_none(row_norm.get("TIPO TRAILER"), MAX_LEN["trailer_type_key"]),
|
||||
"seal": _str_or_none(row_norm.get("PRECINTO"), MAX_LEN["seal"]),
|
||||
"entity_code": _str_or_none(row_norm.get("CODIGO ENTIDAD"), MAX_LEN["entity_code"]),
|
||||
"plate_number": _str_or_none(row_norm.get("PLACAS"), MAX_LEN["plate_number"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"container_key": _str_or_none(row_norm.get("CLAVE CONTENEDOR"), MAX_LEN["container_key"]),
|
||||
}
|
||||
@@ -1,30 +1,35 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Trailers y Cajas.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Upsert por trailer_number usando TrailerService.
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_trailer
|
||||
from .common.mappers import row_to_trailer_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "trl"
|
||||
|
||||
# Para routes.py
|
||||
TRL_IMPORT_FILE_PREFIX = "trl_import_file:"
|
||||
TRL_IMPORT_META_PREFIX = "trl_import_meta:"
|
||||
TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:"
|
||||
TRL_IMPORT_STATUS_PREFIX = "trl_import_status:"
|
||||
TRL_IMPORT_REDIS_TTL = 3600
|
||||
TRL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
@@ -33,80 +38,6 @@ def _get_redis():
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"trl_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Trailer model (a76.trailer)
|
||||
_MAX = {
|
||||
"trailer_number": 20,
|
||||
"ace_trailer_number": 10,
|
||||
"trailer_type_key": 2,
|
||||
"seal": 15,
|
||||
"entity_code": 1,
|
||||
"plate_number": 17,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"container_key": 3,
|
||||
}
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
@@ -114,101 +45,33 @@ def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
unique.append(name if count == 1 else f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Trailer. Retorna error dict o None."""
|
||||
trailer_number = (row.get("NUMERO TRAILER") or "").strip()
|
||||
if not trailer_number:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"}
|
||||
if len(trailer_number) > _MAX["trailer_number"]:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_trailer_number"]),
|
||||
("TIPO TRAILER", _MAX["trailer_type_key"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("CODIGO ENTIDAD", _MAX["entity_code"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("CLAVE CONTENEDOR", _MAX["container_key"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
"""Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row."""
|
||||
trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"])
|
||||
if not trailer_number:
|
||||
return {}
|
||||
return {
|
||||
"trailer_number": trailer_number,
|
||||
"ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]),
|
||||
"trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]),
|
||||
}
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Trailers import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Trailers import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
@@ -232,51 +95,27 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import scan failed: {e}")
|
||||
logger.error("Trailers import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
@@ -289,19 +128,16 @@ def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
logger.warning("Trailers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Trailers import: starting scan for job {job_id}")
|
||||
logger.info("Trailers import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
@@ -312,58 +148,27 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
logger.warning("Trailers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Trailers import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Trailers import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.transportation.trailers.services import TrailerService
|
||||
from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO
|
||||
@@ -374,6 +179,7 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
@@ -396,21 +202,20 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
tn = (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-"
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"trailer_number": tn,
|
||||
"invoice": tn,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = _row_to_trailer_dto(row_norm, tenant_id, company_id)
|
||||
data = row_to_trailer_data(row_norm)
|
||||
if not data or not data.get("trailer_number"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
@@ -418,14 +223,12 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
tn = data["trailer_number"]
|
||||
if tn in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": tn,
|
||||
"invoice": tn,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"trailer_number": tn,
|
||||
"invoice": tn,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
})
|
||||
continue
|
||||
seen_keys_in_file[tn] = i
|
||||
|
||||
@@ -442,34 +245,33 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)}
|
||||
)
|
||||
skipped_details.append({
|
||||
"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err),
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Trailers import DB error: {db_err}")
|
||||
logger.error("Trailers import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("Trailers import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Trailers import cleanup failed: {cleanup_err}")
|
||||
r = _get_redis()
|
||||
r.delete(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
except Exception as e:
|
||||
logger.warning("Trailers import: failed to delete status key: %s", e)
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
@@ -515,13 +317,13 @@ def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
logger.warning("Trailers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Trailers import: starting commit for job {job_id}")
|
||||
logger.info("Trailers import: starting commit for job %s", job_id)
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
@@ -531,5 +333,5 @@ def insert_valid_rows(self, job_id: str):
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
logger.warning("Trailers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_trailer
|
||||
|
||||
__all__ = ["validate_row_trailer"]
|
||||
@@ -0,0 +1,32 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de trailers.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
MAX_LEN,
|
||||
check_required,
|
||||
check_max_length,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_trailer_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_required(row, "NUMERO TRAILER", MAX_LEN["trailer_number"], line_num)
|
||||
|
||||
|
||||
def validate_row_trailer_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("CLAVE ACE", MAX_LEN["ace_trailer_number"]),
|
||||
("TIPO TRAILER", MAX_LEN["trailer_type_key"]),
|
||||
("PRECINTO", MAX_LEN["seal"]),
|
||||
("CODIGO ENTIDAD", MAX_LEN["entity_code"]),
|
||||
("PLACAS", MAX_LEN["plate_number"]),
|
||||
("ESTADO", MAX_LEN["state"]),
|
||||
("PAIS", MAX_LEN["country"]),
|
||||
("CLAVE CONTENEDOR", MAX_LEN["container_key"]),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,22 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila trailer.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import (
|
||||
validate_row_trailer_required,
|
||||
validate_row_trailer_lengths,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de trailers.
|
||||
"""
|
||||
err = validate_row_trailer_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_trailer_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for us_tariff_fractions)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de fracciones arancelarias americanas.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
CODE_MAX = 16
|
||||
PREFIX_MAX = 10
|
||||
UNIT_MAX = 10
|
||||
TYPE_MAX = 10
|
||||
|
||||
|
||||
def normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:CODE_MAX] if len(s) > CODE_MAX else s
|
||||
|
||||
|
||||
def check_required_code(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
code_raw = (row.get(col) or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
code_norm = normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(code_norm) > CODE_MAX:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {CODE_MAX} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_float_min_zero(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
v = float(str(val).strip().replace(",", "."))
|
||||
return v if v >= 0 else None
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_decimal_min_zero(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or not str(val).strip():
|
||||
return None
|
||||
try:
|
||||
v = float(str(val).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser un número"}
|
||||
return None
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para USTariffFraction.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
normalize_code,
|
||||
parse_float_min_zero,
|
||||
)
|
||||
|
||||
MAX_LEN = {"prefix": 10, "unit_of_measure": 10, "type_code": 10}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_us_tariff_fraction_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for USTariffFraction model."""
|
||||
code = normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
return {}
|
||||
fixed_cost_raw = parse_float_min_zero(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"code": code,
|
||||
"prefix": _str_or_none(row_norm.get("PREFIJO"), MAX_LEN["prefix"]),
|
||||
"type_code": _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), MAX_LEN["type_code"]),
|
||||
"ad_valorem": parse_float_min_zero(row_norm.get("ADVALOREM_PCT")),
|
||||
"fixed_cost": fixed_cost,
|
||||
"unit_of_measure": _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), MAX_LEN["unit_of_measure"]),
|
||||
"description": _str_or_none(row_norm.get("DESCRIPCION")),
|
||||
}
|
||||
@@ -1,474 +1,224 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_us_tariff_fraction
|
||||
from .common.mappers import row_to_us_tariff_fraction_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "fa"
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
|
||||
# Para routes.py
|
||||
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
|
||||
FA_IMPORT_META_PREFIX = "fa_import_meta:"
|
||||
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
|
||||
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
FA_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:16] if len(s) > 16 else s
|
||||
|
||||
|
||||
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
|
||||
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
code_norm = _normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
if len(code_norm) > 16:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
|
||||
|
||||
prefix_raw = (row.get("PREFIJO") or "").strip()
|
||||
if prefix_raw and len(prefix_raw) > 10:
|
||||
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
|
||||
if um_raw and len(um_raw) > 10:
|
||||
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
|
||||
if tipo_raw and len(tipo_raw) > 10:
|
||||
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
adv_pct = row.get("ADVALOREM_PCT")
|
||||
if adv_pct is not None and str(adv_pct).strip():
|
||||
try:
|
||||
v = float(str(adv_pct).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
|
||||
|
||||
adv_dlls = row.get("ADVALOREM_DLLS")
|
||||
if adv_dlls is not None and str(adv_dlls).strip():
|
||||
try:
|
||||
v = float(str(adv_dlls).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"FA import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "FA import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "FA import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import scan failed: {e}")
|
||||
logger.error("FA import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(str(val).strip().replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
|
||||
"""
|
||||
logger.info(f"FA import: starting commit for job {job_id}")
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("FA import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "FA import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "FA import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"FA import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
data = row_to_us_tariff_fraction_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("code"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA vacío"})
|
||||
continue
|
||||
|
||||
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
|
||||
continue
|
||||
|
||||
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
|
||||
description = _str_or_none(row_norm.get("DESCRIPCION"))
|
||||
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
|
||||
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
|
||||
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == code,
|
||||
)
|
||||
.first()
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == data["code"],
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.prefix = prefix
|
||||
existing.type_code = type_code
|
||||
existing.ad_valorem = ad_valorem
|
||||
existing.fixed_cost = fixed_cost
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.description = description
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_row = USTariffFraction(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
code=code,
|
||||
prefix=prefix,
|
||||
type_code=type_code,
|
||||
ad_valorem=ad_valorem,
|
||||
fixed_cost=fixed_cost,
|
||||
unit_of_measure=unit_of_measure,
|
||||
description=description,
|
||||
)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
existing.prefix = data.get("prefix")
|
||||
existing.type_code = data.get("type_code")
|
||||
existing.ad_valorem = data.get("ad_valorem")
|
||||
existing.fixed_cost = data.get("fixed_cost")
|
||||
existing.unit_of_measure = data.get("unit_of_measure")
|
||||
existing.description = data.get("description")
|
||||
session.add(existing)
|
||||
else:
|
||||
new_row = USTariffFraction(**data)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"FA import DB error: {db_err}")
|
||||
logger.error("FA import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("FA import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path_clean = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path_clean):
|
||||
os.remove(meta_path_clean)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"FA import cleanup failed: {cleanup_err}")
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("FA import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_us_tariff_fraction
|
||||
|
||||
__all__ = ["validate_row_us_tariff_fraction"]
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de fracciones arancelarias americanas.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_code,
|
||||
check_optional_max_length,
|
||||
check_optional_decimal_min_zero,
|
||||
PREFIX_MAX,
|
||||
UNIT_MAX,
|
||||
TYPE_MAX,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de fracción arancelaria americana.
|
||||
FRACCION_ARANCELARIA requerida (max 16); PREFIJO, UNIDAD_DE_MEDIDA, TIPO_DE_ADVALOREM opcionales (max 10);
|
||||
ADVALOREM_PCT y ADVALOREM_DLLS opcionales numéricos >= 0.
|
||||
"""
|
||||
err = check_required_code(row, "FRACCION_ARANCELARIA", line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "PREFIJO", PREFIX_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "UNIDAD_DE_MEDIDA", UNIT_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "TIPO_DE_ADVALOREM", TYPE_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_decimal_min_zero(row, "ADVALOREM_PCT", line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_decimal_min_zero(row, "ADVALOREM_DLLS", line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila fracción arancelaria americana.
|
||||
"""
|
||||
from .common import validate_row_us_tariff_fraction
|
||||
|
||||
__all__ = ["validate_row_us_tariff_fraction"]
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for vehicles)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de vehículos (longitudes, decimal, fecha aseguradora).
|
||||
"""
|
||||
import re
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
# Max lengths from Vehicle model (a76.vehicle)
|
||||
MAX_LEN = {
|
||||
"vehicle_key": 14,
|
||||
"ace_vehicle_key": 10,
|
||||
"transporter_key": 23,
|
||||
"transport_identifier": 30,
|
||||
"transport_type": 2,
|
||||
"entity_code": 1,
|
||||
"transponder_number": 16,
|
||||
"dot_number": 8,
|
||||
"plate_number": 17,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"seal": 49,
|
||||
"insurance_company_name": 30,
|
||||
"insurance_number": 20,
|
||||
"series": 30,
|
||||
}
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any], col: str, max_len: int, line_num: int, required: bool = False
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
s = str(val).strip().replace(",", "")
|
||||
return Decimal(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def parse_insurance_date(val: Any) -> Optional[int]:
|
||||
"""Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD."""
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_decimal(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
if parse_decimal(val) is None:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser numérico"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_insurance_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
if parse_insurance_date(val) is None:
|
||||
return {"line": line_num, "col": col, "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
|
||||
return None
|
||||
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Vehicle.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
MAX_LEN,
|
||||
parse_decimal,
|
||||
parse_insurance_date,
|
||||
)
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_vehicle_data(row_norm: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row."""
|
||||
vehicle_key = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["vehicle_key"])
|
||||
if not vehicle_key:
|
||||
return {}
|
||||
return {
|
||||
"vehicle_key": vehicle_key,
|
||||
"ace_vehicle_key": _str_or_none(row_norm.get("CLAVE ACE"), MAX_LEN["ace_vehicle_key"]),
|
||||
"transporter_key": _str_or_none(row_norm.get("CLAVE TRANSPORTE"), MAX_LEN["transporter_key"]),
|
||||
"series": _str_or_none(row_norm.get("VIN"), MAX_LEN["series"]),
|
||||
"transport_type": _str_or_none(row_norm.get("TIPO TRANSPORTE"), MAX_LEN["transport_type"]),
|
||||
"entity_code": _str_or_none(row_norm.get("CODIGO DE ENTIDAD"), MAX_LEN["entity_code"]),
|
||||
"transponder_number": _str_or_none(row_norm.get("TRANSPONDEDOR"), MAX_LEN["transponder_number"]),
|
||||
"dot_number": _str_or_none(row_norm.get("NUMERO DOT"), MAX_LEN["dot_number"]),
|
||||
"plate_number": _str_or_none(row_norm.get("PLACAS"), MAX_LEN["plate_number"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"seal": _str_or_none(row_norm.get("PRECINTO"), MAX_LEN["seal"]),
|
||||
"insurance_company_name": _str_or_none(row_norm.get("EMPRESA ASEGURADORA"), MAX_LEN["insurance_company_name"]),
|
||||
"insurance_number": _str_or_none(row_norm.get("NUM. ASEGURADORA"), MAX_LEN["insurance_number"]),
|
||||
"insurance_amount": parse_decimal(row_norm.get("MONTO ASEGURADO") or row_norm.get("MONTO")),
|
||||
"insurance_date": parse_insurance_date(row_norm.get("FECHA DE ASEGURADORA") or row_norm.get("FECHA ASEGURADORA")),
|
||||
}
|
||||
@@ -1,31 +1,35 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Vehículos (Transportes).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key.
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from core.paths import layout_path
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_vehicle
|
||||
from .common.mappers import row_to_vehicle_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "veh"
|
||||
|
||||
# Para routes.py
|
||||
VEHL_IMPORT_FILE_PREFIX = "veh_import_file:"
|
||||
VEHL_IMPORT_META_PREFIX = "veh_import_meta:"
|
||||
VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:"
|
||||
VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:"
|
||||
VEHL_IMPORT_REDIS_TTL = 3600
|
||||
VEHL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _get_redis():
|
||||
@@ -34,283 +38,59 @@ def _get_redis():
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"veh_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Vehicle model (a76.vehicle)
|
||||
_MAX = {
|
||||
"vehicle_key": 14,
|
||||
"ace_vehicle_key": 10,
|
||||
"transporter_key": 23,
|
||||
"transport_identifier": 30,
|
||||
"transport_type": 2,
|
||||
"entity_code": 1,
|
||||
"transponder_number": 16,
|
||||
"dot_number": 8,
|
||||
"plate_number": 17,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"seal": 49,
|
||||
"insurance_company_name": 30,
|
||||
"insurance_number": 20,
|
||||
"series": 30,
|
||||
}
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
s = str(val).strip().replace(",", "")
|
||||
return Decimal(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_insurance_date(val: Any) -> Optional[int]:
|
||||
"""Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD."""
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
# Already integer-like
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
# Try DD/MM/YYYY or similar
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Vehículo. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > _MAX["vehicle_key"]:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"}
|
||||
|
||||
# Optional fields: max lengths only
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_vehicle_key"]),
|
||||
("CLAVE TRANSPORTE", _MAX["transporter_key"]),
|
||||
("VIN", _MAX["series"]),
|
||||
("TIPO TRANSPORTE", _MAX["transport_type"]),
|
||||
("CODIGO DE ENTIDAD", _MAX["entity_code"]),
|
||||
("TRANSPONDEDOR", _MAX["transponder_number"]),
|
||||
("NUMERO DOT", _MAX["dot_number"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("CIUDAD", _MAX["city"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]),
|
||||
("NUM. ASEGURADORA", _MAX["insurance_number"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
# MONTO ASEGURADO: must be numeric if present
|
||||
monto = row.get("MONTO ASEGURADO") or row.get("MONTO")
|
||||
if monto is not None and str(monto).strip():
|
||||
if _parse_decimal(monto) is None:
|
||||
return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"}
|
||||
|
||||
# FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None)
|
||||
# Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either
|
||||
# reject or set None. We reject invalid date to keep data quality.
|
||||
fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_insurance_date(fecha) is None:
|
||||
return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _do_scan(
|
||||
job_id: str,
|
||||
progress_callback: Optional[Any] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync.
|
||||
progress_callback(current, total, error_count) opcional.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import scan failed: {e}")
|
||||
logger.error("Vehicles import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
@@ -320,23 +100,16 @@ def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
logger.warning("Vehicles import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting scan for job {job_id}")
|
||||
logger.info("Vehicles import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
@@ -347,99 +120,27 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
logger.warning("Vehicles import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row."""
|
||||
vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"])
|
||||
if not vehicle_key:
|
||||
return {}
|
||||
data = {
|
||||
"vehicle_key": vehicle_key,
|
||||
"ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]),
|
||||
"transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]),
|
||||
"series": _str_or_none(row.get("VIN"), _MAX["series"]),
|
||||
"transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]),
|
||||
"transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]),
|
||||
"dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"city": _str_or_none(row.get("CIUDAD"), _MAX["city"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]),
|
||||
"insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]),
|
||||
"insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")),
|
||||
"insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import")
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv")
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import")
|
||||
|
||||
error_dir = layout_path("imports", "errors")
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.transportation.vehicles.services import VehicleService
|
||||
from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO
|
||||
@@ -450,91 +151,84 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
vk = (row_norm.get("CLAVE") or "").strip()[:14] or "-"
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"vehicle_key": vk,
|
||||
"invoice": vk,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_vehicle_data(row_norm)
|
||||
if not data or not data.get("vehicle_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
vk = data["vehicle_key"]
|
||||
if vk in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"vehicle_key": vk,
|
||||
"invoice": vk,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
})
|
||||
continue
|
||||
seen_keys_in_file[vk] = i
|
||||
|
||||
existing = VehicleService.get_by_id(session, vk, tenant_id, company_id)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_vehicle_dto(row_norm)
|
||||
if not data or not data.get("vehicle_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
vk = data["vehicle_key"]
|
||||
if vk in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[vk] = i
|
||||
|
||||
existing = VehicleService.get_by_id(session, vk, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"})
|
||||
VehicleService.update(session, vk, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = VehicleCreateDTO(**data)
|
||||
VehicleService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
if existing:
|
||||
update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"})
|
||||
VehicleService.update(session, vk, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = VehicleCreateDTO(**data)
|
||||
VehicleService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err),
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Vehicles import DB error: {db_err}")
|
||||
logger.error("Vehicles import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
logger.exception("Vehicles import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Vehicles import cleanup failed: {cleanup_err}")
|
||||
r = _get_redis()
|
||||
r.delete(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
except Exception as e:
|
||||
logger.warning("Vehicles import: failed to delete status key: %s", e)
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
@@ -571,10 +265,6 @@ def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el commit en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
@@ -584,16 +274,13 @@ def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
logger.warning("Vehicles import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting commit for job {job_id}")
|
||||
logger.info("Vehicles import: starting commit for job %s", job_id)
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
@@ -603,5 +290,5 @@ def insert_valid_rows(self, job_id: str):
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
logger.warning("Vehicles import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_vehicle
|
||||
|
||||
__all__ = ["validate_row_vehicle"]
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de vehículos.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
MAX_LEN,
|
||||
check_required,
|
||||
check_max_length,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_vehicle_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_required(row, "CLAVE", MAX_LEN["vehicle_key"], line_num)
|
||||
|
||||
|
||||
def validate_row_vehicle_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("CLAVE ACE", MAX_LEN["ace_vehicle_key"]),
|
||||
("CLAVE TRANSPORTE", MAX_LEN["transporter_key"]),
|
||||
("VIN", MAX_LEN["series"]),
|
||||
("TIPO TRANSPORTE", MAX_LEN["transport_type"]),
|
||||
("CODIGO DE ENTIDAD", MAX_LEN["entity_code"]),
|
||||
("TRANSPONDEDOR", MAX_LEN["transponder_number"]),
|
||||
("NUMERO DOT", MAX_LEN["dot_number"]),
|
||||
("PLACAS", MAX_LEN["plate_number"]),
|
||||
("CIUDAD", MAX_LEN["city"]),
|
||||
("ESTADO", MAX_LEN["state"]),
|
||||
("PAIS", MAX_LEN["country"]),
|
||||
("PRECINTO", MAX_LEN["seal"]),
|
||||
("EMPRESA ASEGURADORA", MAX_LEN["insurance_company_name"]),
|
||||
("NUM. ASEGURADORA", MAX_LEN["insurance_number"]),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_vehicle_amount(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
from ..common.common_validators import parse_decimal
|
||||
monto = row.get("MONTO ASEGURADO") or row.get("MONTO")
|
||||
if monto is None or not str(monto).strip():
|
||||
return None
|
||||
if parse_decimal(monto) is None:
|
||||
return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_vehicle_insurance_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
from ..common.common_validators import parse_insurance_date
|
||||
fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")
|
||||
if fecha is None or not str(fecha).strip():
|
||||
return None
|
||||
if parse_insurance_date(fecha) is None:
|
||||
return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
|
||||
return None
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila vehículo.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import (
|
||||
validate_row_vehicle_required,
|
||||
validate_row_vehicle_lengths,
|
||||
validate_row_vehicle_amount,
|
||||
validate_row_vehicle_insurance_date,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de vehículos.
|
||||
Encadena: requerido (CLAVE) → longitudes → monto opcional → fecha aseguradora opcional.
|
||||
"""
|
||||
err = validate_row_vehicle_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_vehicle_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_vehicle_amount(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_vehicle_insurance_date(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
Reference in New Issue
Block a user