Merge branch 'development' into feature/data-entry
This commit is contained in:
0
backend/api/__init__.py
Normal file
0
backend/api/__init__.py
Normal file
0
backend/api/v1/__init__.py
Normal file
0
backend/api/v1/__init__.py
Normal file
0
backend/api/v1/modules/__init__.py
Normal file
0
backend/api/v1/modules/__init__.py
Normal file
0
backend/api/v1/modules/a76/__init__.py
Normal file
0
backend/api/v1/modules/a76/__init__.py
Normal file
7
backend/api/v1/modules/a76/boms/__init__.py
Normal file
7
backend/api/v1/modules/a76/boms/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
Módulo de importación CSV para BOMs (Bills of Materials).
|
||||
"""
|
||||
|
||||
from .routes import router
|
||||
|
||||
__all__ = ["router"]
|
||||
1
backend/api/v1/modules/a76/boms/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/boms/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# boms.imports
|
||||
151
backend/api/v1/modules/a76/boms/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/boms/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para BOMs.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
BOM_IMPORT_FILE_PREFIX,
|
||||
BOM_IMPORT_META_PREFIX,
|
||||
BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "boms",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
22
backend/api/v1/modules/a76/boms/imports/schemas.py
Normal file
22
backend/api/v1/modules/a76/boms/imports/schemas.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
430
backend/api/v1/modules/a76/boms/imports/tasks.py
Normal file
430
backend/api/v1/modules/a76/boms/imports/tasks.py
Normal file
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"bom_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila BOM según columnas del template. FK opcional a a76.parts."""
|
||||
parent = (row.get("NUMPARTE_PADRE") or "").strip()
|
||||
if not parent:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"}
|
||||
if len(parent) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
component = (row.get("NUMPARTE_COMPONENTE") or "").strip()
|
||||
if not component:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"}
|
||||
if len(component) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
qty = row.get("CANTIDAD")
|
||||
if qty is None or qty == "":
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"}
|
||||
try:
|
||||
val = Decimal(str(qty))
|
||||
if val < 0:
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 10:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
version_bom = row.get("VERSION_BOM")
|
||||
if not _optional_number(version_bom):
|
||||
return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"}
|
||||
|
||||
version_bill = row.get("VERSION_BILL")
|
||||
if not _optional_number(version_bill):
|
||||
return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"}
|
||||
|
||||
# Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas)
|
||||
if valid_part_numbers is not None and len(valid_part_numbers) > 0:
|
||||
if parent not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"}
|
||||
if component not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"BOMs import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts for FK validation: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"BOMs import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"BOMs import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
valid_count = 0
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
# Cuando exista la tabla de destino, aquí se hará insert/update.
|
||||
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"BOMs import cleanup failed: {cleanup_err}")
|
||||
|
||||
return response
|
||||
42
backend/api/v1/modules/a76/boms/imports/template_config.py
Normal file
42
backend/api/v1/modules/a76/boms/imports/template_config.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para BOMs (EstructuraBOMS.xlsx).
|
||||
Placeholders hasta tener el XLS definitivo; ajustar canónicos y aliases según el archivo.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"boms": [
|
||||
{"canonical": "NUMPARTE_PADRE", "aliases": ["PARTE PADRE", "PART NUMBER", "PARENT PART", "NUM PARTE PADRE"]},
|
||||
{"canonical": "NUMPARTE_COMPONENTE", "aliases": ["PARTE COMPONENTE", "COMPONENT PART", "NUM PARTE COMPONENTE"]},
|
||||
{"canonical": "CANTIDAD", "aliases": ["QTY", "QUANTITY", "CANT"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "VERSION_BOM", "aliases": ["VERSION BOM", "BOM VERSION", "VERSIONBOM"]},
|
||||
{"canonical": "VERSION_BILL", "aliases": ["VERSION BILL", "BILL VERSION", "VERSIONBILL"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("boms")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
13
backend/api/v1/modules/a76/boms/routes.py
Normal file
13
backend/api/v1/modules/a76/boms/routes.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Endpoints para importación CSV de BOMs.
|
||||
Mismo patrón que parts y classes: upload → scan → status → commit.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / boms / csv_import"])
|
||||
@@ -2,6 +2,20 @@
|
||||
Módulo de Class
|
||||
"""
|
||||
|
||||
from .routes import router
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["router"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""
|
||||
Lazy export to avoid circular imports during Celery init.
|
||||
|
||||
This package is imported when models are loaded (e.g. a76.items.models),
|
||||
so importing FastAPI routes at import-time can break Celery startup.
|
||||
"""
|
||||
if name == "router":
|
||||
from .routes import router
|
||||
|
||||
return router
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
1
backend/api/v1/modules/a76/classes/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/classes/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV import for Clases de Materiales (upload → scan → status → commit).
|
||||
151
backend/api/v1/modules/a76/classes/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/classes/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para Clases de Materiales.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CLS_IMPORT_FILE_PREFIX,
|
||||
CLS_IMPORT_META_PREFIX,
|
||||
CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "material_classes",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
21
backend/api/v1/modules/a76/classes/imports/schemas.py
Normal file
21
backend/api/v1/modules/a76/classes/imports/schemas.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
528
backend/api/v1/modules/a76/classes/imports/tasks.py
Normal file
528
backend/api/v1/modules/a76/classes/imports/tasks.py
Normal file
@@ -0,0 +1,528 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
if not class_code:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
|
||||
if len(class_code) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
material_key = (row.get("CLAVEMAT") or "").strip()
|
||||
if material_key:
|
||||
if len(material_key) > 10:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
|
||||
if valid_material_keys is not None and material_key not in valid_material_keys:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom:
|
||||
if len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
if valid_uom_codes is not None and uom not in valid_uom_codes:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 20:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
sub_key = (row.get("CLAVESUB") or "").strip()
|
||||
if sub_key and len(sub_key) > 5:
|
||||
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
|
||||
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if iva_exempt and len(iva_exempt) > 4:
|
||||
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
rev_fisica = row.get("REVFISICA")
|
||||
if rev_fisica is not None and rev_fisica != "":
|
||||
try:
|
||||
v = int(rev_fisica)
|
||||
if v < -32768 or v > 32767:
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Classes import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Classes import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code: Dict[str, Class] = {}
|
||||
for c in (
|
||||
session.query(Class)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_code[c.class_code] = c
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
if existing:
|
||||
existing.description_es = desc_es
|
||||
existing.description_en = desc_en
|
||||
existing.material_key = material_key
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.sub_key = sub_key
|
||||
existing.physical_review = physical_review
|
||||
existing.iva_exempt_fraction = iva_exempt_fraction
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
class_code=class_code,
|
||||
description_es=desc_es,
|
||||
description_en=desc_en,
|
||||
material_key=material_key,
|
||||
unit_of_measure=unit_of_measure,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
sub_key=sub_key,
|
||||
physical_review=physical_review,
|
||||
iva_exempt_fraction=iva_exempt_fraction,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Classes import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t
|
||||
|
||||
from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse
|
||||
from .service import ClassService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create a new router for custom endpoints
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"])
|
||||
|
||||
# Add consolidated catalog endpoints FIRST (before generic CRUD routes)
|
||||
# This ensures they have priority over the generic /{id} route
|
||||
@router.get(
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# CSV import flow for Clientes y Proveedores (Client Providers).
|
||||
# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit.
|
||||
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Rutas de importación CSV para Clientes y Proveedores.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CP_IMPORT_FILE_PREFIX,
|
||||
CP_IMPORT_META_PREFIX,
|
||||
CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "client_providers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("CP import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientOrProviderEnum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active(val: Optional[str]) -> bool:
|
||||
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
|
||||
rfc = (row.get("RFC") or "").strip()
|
||||
if not rfc:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
|
||||
if len(rfc) > 30:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
|
||||
name = (row.get("NOMBRE") or "").strip()
|
||||
if len(name) > 256:
|
||||
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
if short_name and len(short_name) > 10:
|
||||
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
curp = (row.get("CURP") or "").strip()
|
||||
if curp and len(curp) > 19:
|
||||
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CP import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
|
||||
Upsert por (tenant_id, company_id, rfc).
|
||||
"""
|
||||
logger.info(f"CP import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CP import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
rfc = _str_or_none(row_norm.get("RFC"), 30)
|
||||
if not rfc:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
|
||||
if client_or_provider is None:
|
||||
client_or_provider = ClientOrProviderEnum.BOTH
|
||||
|
||||
is_active = _parse_active(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
|
||||
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
|
||||
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
|
||||
existing.client_or_provider = client_or_provider
|
||||
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
rfc=rfc,
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 256),
|
||||
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
|
||||
curp=_str_or_none(row_norm.get("CURP"), 19),
|
||||
client_or_provider=client_or_provider,
|
||||
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
|
||||
# Opcional: crear dirección si hay email/teléfono/dirección
|
||||
email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
|
||||
if email or phone or address_str:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_str,
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
phone=phone,
|
||||
email=email,
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
|
||||
)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CP import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CP import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"client_providers": [
|
||||
{"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]},
|
||||
{"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
|
||||
{"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]},
|
||||
{"canonical": "CURP", "aliases": []},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]},
|
||||
{"canonical": "ESTADO", "aliases": []},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]},
|
||||
{"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "INCOTERM", "aliases": []},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla client_providers."""
|
||||
cols = TEMPLATE_COLUMNS.get("client_providers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -20,10 +20,14 @@ from .dto import (
|
||||
)
|
||||
from .service import ClientProviderService
|
||||
from .models import ClientProvider
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create main router to add custom endpoints
|
||||
router = APIRouter(prefix="/clients-providers")
|
||||
|
||||
# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"])
|
||||
|
||||
|
||||
@router.get("/", response_model=ClientProviderPaginatedResponseDTO)
|
||||
async def get_clients_and_providers(
|
||||
|
||||
1
backend/api/v1/modules/a76/csv_templates/__init__.py
Normal file
1
backend/api/v1/modules/a76/csv_templates/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV templates: generate CSV from code (no physical XLS/XLSX files)
|
||||
147
backend/api/v1/modules/a76/csv_templates/registry.py
Normal file
147
backend/api/v1/modules/a76/csv_templates/registry.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Registro central de plantillas CSV: template_id -> lista de cabeceras canónicas.
|
||||
Construido a partir de los TEMPLATE_COLUMNS de cada módulo de imports.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# Importar configs de cada módulo
|
||||
from api.v1.modules.a76.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as IMPORTS_TEMPLATE_COLUMNS,
|
||||
_resolve_template_columns as resolve_imports_template,
|
||||
)
|
||||
from api.v1.modules.a76.parts.imports.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.boms.imports.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.classes.imports.template_config import TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.customs_brokers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as CUSTOMS_BROKERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.clients_and_providers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as CLIENTS_PROVIDERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as EXCHANGE_RATE_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.pedmientos.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as PEDIMENTOS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.vehicles.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as VEHICLES_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.drivers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as DRIVERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.trailers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as TRAILERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
|
||||
|
||||
def _canonicals_from_columns(cols: Optional[List[Dict]]) -> List[str]:
|
||||
"""Extrae la lista de nombres canónicos en orden a partir de una lista de columnas."""
|
||||
if not cols:
|
||||
return []
|
||||
return [item["canonical"] for item in cols]
|
||||
|
||||
|
||||
def _build_registry() -> Dict[str, List[str]]:
|
||||
registry: Dict[str, List[str]] = {}
|
||||
|
||||
# a76/imports (facturas): imp_temp_header, imp_temp_details, imp_def_*, exp_def_*
|
||||
for tid in ("imp_temp_header", "imp_temp_details", "imp_def_header", "imp_def_details", "exp_def_header", "exp_def_details"):
|
||||
cols = resolve_imports_template(tid)
|
||||
registry[tid] = _canonicals_from_columns(cols)
|
||||
|
||||
# part_numbers (parts); "items" usa la misma plantilla
|
||||
part_cols = PARTS_TEMPLATE_COLUMNS.get("part_numbers")
|
||||
registry["part_numbers"] = _canonicals_from_columns(part_cols)
|
||||
registry["items"] = _canonicals_from_columns(part_cols)
|
||||
|
||||
# boms
|
||||
registry["boms"] = _canonicals_from_columns(BOMS_TEMPLATE_COLUMNS.get("boms"))
|
||||
|
||||
# material_classes
|
||||
registry["material_classes"] = _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes"))
|
||||
|
||||
# customs_brokers
|
||||
registry["customs_brokers"] = _canonicals_from_columns(CUSTOMS_BROKERS_TEMPLATE_COLUMNS.get("customs_brokers"))
|
||||
|
||||
# clients_providers
|
||||
registry["clients_providers"] = _canonicals_from_columns(CLIENTS_PROVIDERS_TEMPLATE_COLUMNS.get("client_providers"))
|
||||
|
||||
# exchange_rates
|
||||
registry["exchange_rates"] = _canonicals_from_columns(EXCHANGE_RATE_TEMPLATE_COLUMNS.get("exchange_rates"))
|
||||
|
||||
# american_fractions
|
||||
registry["american_fractions"] = _canonicals_from_columns(
|
||||
US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS.get("us_tariff_fractions")
|
||||
)
|
||||
|
||||
# pedimentos
|
||||
registry["pedimentos"] = _canonicals_from_columns(PEDIMENTOS_TEMPLATE_COLUMNS.get("pedimentos"))
|
||||
|
||||
# transports (vehicles)
|
||||
registry["transports"] = _canonicals_from_columns(VEHICLES_TEMPLATE_COLUMNS.get("vehicles"))
|
||||
|
||||
# drivers
|
||||
registry["drivers"] = _canonicals_from_columns(DRIVERS_TEMPLATE_COLUMNS.get("drivers"))
|
||||
|
||||
# trailers
|
||||
registry["trailers"] = _canonicals_from_columns(TRAILERS_TEMPLATE_COLUMNS.get("trailers"))
|
||||
|
||||
return registry
|
||||
|
||||
|
||||
_TEMPLATE_HEADERS: Dict[str, List[str]] = _build_registry()
|
||||
|
||||
# Nombre de archivo sugerido para descarga (sin path)
|
||||
TEMPLATE_FILENAMES: Dict[str, str] = {
|
||||
"customs_brokers": "EstructuraCatAgenteAduanal.csv",
|
||||
"clients_providers": "EstructuraCatClienteProv.csv",
|
||||
"exchange_rates": "EstructuraCatTiposCambio.csv",
|
||||
"american_fractions": "EstructuraCatFraccAme.csv",
|
||||
"material_classes": "EstructuraCatClasesAF.csv",
|
||||
"part_numbers": "EstructuraCatPartesAF.csv",
|
||||
"items": "EstructuraCatPartesAF.csv",
|
||||
"boms": "EstructuraBOMS.csv",
|
||||
"pedimentos": "EstructuraCatPedimentos.csv",
|
||||
"transports": "EstructuraCatTransportes.csv",
|
||||
"drivers": "EstructuraCatConductor.csv",
|
||||
"trailers": "EstructuraCatTrailers.csv",
|
||||
"imp_temp_header": "EstructuraEncFacImpoTemp.csv",
|
||||
"imp_temp_details": "EstructuraParFacImpoTempAF.csv",
|
||||
"imp_def_header": "EstructuraEncFacImpoDef.csv",
|
||||
"imp_def_details": "EstructuraParFacImpoDefAF.csv",
|
||||
"exp_def_header": "EstructuraEncFacExpoCamReg.csv",
|
||||
"exp_def_details": "EstructuraParExpoCamReg.csv",
|
||||
}
|
||||
|
||||
|
||||
def get_template_headers(template_id: str) -> Optional[List[str]]:
|
||||
"""Devuelve la lista de cabeceras canónicas para el template_id, o None si no existe."""
|
||||
return _TEMPLATE_HEADERS.get(template_id)
|
||||
|
||||
|
||||
def get_template_filename(template_id: str) -> str:
|
||||
"""Nombre de archivo sugerido para la descarga."""
|
||||
return TEMPLATE_FILENAMES.get(template_id, f"plantilla_{template_id}.csv")
|
||||
|
||||
|
||||
def generate_csv_content(template_id: str, include_bom: bool = True) -> Optional[bytes]:
|
||||
"""
|
||||
Genera el contenido CSV (solo fila de cabeceras) para el template_id.
|
||||
UTF-8, opcionalmente con BOM para Excel.
|
||||
"""
|
||||
headers = get_template_headers(template_id)
|
||||
if not headers:
|
||||
return None
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf, lineterminator="\n")
|
||||
writer.writerow(headers)
|
||||
content = buf.getvalue().encode("utf-8")
|
||||
if include_bom:
|
||||
content = b"\xef\xbb\xbf" + content
|
||||
return content
|
||||
35
backend/api/v1/modules/a76/csv_templates/routes.py
Normal file
35
backend/api/v1/modules/a76/csv_templates/routes.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Rutas para descargar plantillas CSV generadas desde código (sin archivos XLS/XLSX).
|
||||
"""
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import Response
|
||||
|
||||
from core.security import get_current_user
|
||||
|
||||
from .registry import generate_csv_content, get_template_filename
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/{template_id}", response_class=Response)
|
||||
async def download_csv_template(
|
||||
template_id: str,
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Devuelve un CSV con solo la fila de cabeceras para la plantilla indicada.
|
||||
Las cabeceras son los nombres canónicos definidos en cada template_config.
|
||||
"""
|
||||
content = generate_csv_content(template_id, include_bom=True)
|
||||
if content is None:
|
||||
raise HTTPException(status_code=404, detail=f"Plantilla desconocida: {template_id}")
|
||||
filename = get_template_filename(template_id)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# CSV import flow for Agentes Aduanales (Customs Brokers).
|
||||
# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit.
|
||||
161
backend/api/v1/modules/a76/customs_brokers/imports/routes.py
Normal file
161
backend/api/v1/modules/a76/customs_brokers/imports/routes.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Rutas de importación CSV para Agentes Aduanales.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CB_IMPORT_FILE_PREFIX,
|
||||
CB_IMPORT_META_PREFIX,
|
||||
CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "customs_brokers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("CB import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
447
backend/api/v1/modules/a76/customs_brokers/imports/tasks.py
Normal file
447
backend/api/v1/modules/a76/customs_brokers/imports/tasks.py
Normal file
@@ -0,0 +1,447 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con a76.imports)
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > 5:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CB import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Guardar números de línea con error en Redis para insert_valid_rows
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
|
||||
"""
|
||||
logger.info(f"CB import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CB import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
clave = (row_norm.get("CLAVE") or "").strip()[:5]
|
||||
if not clave:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
|
||||
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
|
||||
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
|
||||
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
|
||||
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
|
||||
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
|
||||
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
|
||||
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
|
||||
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
existing.license = lic[:4] if lic and lic.isdigit() else None
|
||||
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
|
||||
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
license_val = lic[:4] if lic and lic.isdigit() else None
|
||||
new_broker = CustomsBroker(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
broker_key=clave,
|
||||
type=_str_or_none(row_norm.get("TIPO"), 9),
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 80),
|
||||
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
|
||||
fax=_str_or_none(row_norm.get("FAX"), 30),
|
||||
email=_str_or_none(row_norm.get("EMAIL"), 100),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
tax_id=_str_or_none(row_norm.get("RFC"), 30),
|
||||
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
license=license_val,
|
||||
company=_str_or_none(row_norm.get("EMPRESA"), 200),
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
|
||||
)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CB import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Limpieza
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CB import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"customs_brokers": [
|
||||
{"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]},
|
||||
{"canonical": "TIPO"},
|
||||
{"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD"},
|
||||
{"canonical": "ESTADO"},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]},
|
||||
{"canonical": "FAX"},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]},
|
||||
{"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]},
|
||||
{"canonical": "EMPRESA", "aliases": ["COMPANY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla customs_brokers."""
|
||||
cols = TEMPLATE_COLUMNS.get("customs_brokers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from . import dto, services
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (mismo flujo que a76.imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"])
|
||||
|
||||
customs_broker_crud = TenantCRUDRoutes(
|
||||
service=services.CustomsBrokerService,
|
||||
create_schema=dto.CustomsBrokerCreateDTO,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Exchange rate CSV import module
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Tipos de Cambio.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
ER_IMPORT_FILE_PREFIX,
|
||||
ER_IMPORT_META_PREFIX,
|
||||
ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "exchange_rates",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("ER import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,465 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if _parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
|
||||
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
|
||||
if local_raw and len(local_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
|
||||
if foreign_raw and len(foreign_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"ER import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
|
||||
"""
|
||||
logger.info(f"ER import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"ER import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
parsed_date = _parse_date(row_norm.get("FECHA"))
|
||||
if not parsed_date:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
|
||||
continue
|
||||
|
||||
try:
|
||||
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
|
||||
value_decimal = Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
|
||||
continue
|
||||
|
||||
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
|
||||
if local_currency:
|
||||
local_currency = local_currency.upper()
|
||||
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
|
||||
if foreign_currency:
|
||||
foreign_currency = foreign_currency.upper()
|
||||
|
||||
key_date = parsed_date.date()
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
|
||||
if existing:
|
||||
existing.value = value_decimal
|
||||
existing.local_currency = local_currency or None
|
||||
existing.foreign_currency = foreign_currency or None
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
date=parsed_date,
|
||||
value=value_decimal,
|
||||
local_currency=local_currency,
|
||||
foreign_currency=foreign_currency,
|
||||
)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"ER import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"ER import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"exchange_rates": [
|
||||
{"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]},
|
||||
{"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]},
|
||||
{"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]},
|
||||
{"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla exchange_rates."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -34,6 +34,10 @@ crud_router = route_handler.router
|
||||
from fastapi import APIRouter
|
||||
custom_router = APIRouter(prefix="/exchange-rate", tags=[])
|
||||
|
||||
# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered
|
||||
from .imports.routes import router as imports_router
|
||||
custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"])
|
||||
|
||||
@custom_router.get("/test-ping")
|
||||
async def test_ping():
|
||||
return {"message": "pong"}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Fracción Americana (US Tariff Fractions).
|
||||
Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
FA_IMPORT_FILE_PREFIX,
|
||||
FA_IMPORT_META_PREFIX,
|
||||
FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "us_tariff_fractions",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("FA import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,474 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
|
||||
FA_IMPORT_META_PREFIX = "fa_import_meta:"
|
||||
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
|
||||
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:16] if len(s) > 16 else s
|
||||
|
||||
|
||||
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
|
||||
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
code_norm = _normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
if len(code_norm) > 16:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
|
||||
|
||||
prefix_raw = (row.get("PREFIJO") or "").strip()
|
||||
if prefix_raw and len(prefix_raw) > 10:
|
||||
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
|
||||
if um_raw and len(um_raw) > 10:
|
||||
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
|
||||
if tipo_raw and len(tipo_raw) > 10:
|
||||
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
adv_pct = row.get("ADVALOREM_PCT")
|
||||
if adv_pct is not None and str(adv_pct).strip():
|
||||
try:
|
||||
v = float(str(adv_pct).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
|
||||
|
||||
adv_dlls = row.get("ADVALOREM_DLLS")
|
||||
if adv_dlls is not None and str(adv_dlls).strip():
|
||||
try:
|
||||
v = float(str(adv_dlls).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"FA import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(str(val).strip().replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
|
||||
"""
|
||||
logger.info(f"FA import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"FA import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
|
||||
continue
|
||||
|
||||
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
|
||||
description = _str_or_none(row_norm.get("DESCRIPCION"))
|
||||
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
|
||||
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
|
||||
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.prefix = prefix
|
||||
existing.type_code = type_code
|
||||
existing.ad_valorem = ad_valorem
|
||||
existing.fixed_cost = fixed_cost
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.description = description
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_row = USTariffFraction(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
code=code,
|
||||
prefix=prefix,
|
||||
type_code=type_code,
|
||||
ad_valorem=ad_valorem,
|
||||
fixed_cost=fixed_cost,
|
||||
unit_of_measure=unit_of_measure,
|
||||
description=description,
|
||||
)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"FA import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path_clean = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path_clean):
|
||||
os.remove(meta_path_clean)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"FA import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"us_tariff_fractions": [
|
||||
{"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]},
|
||||
{"canonical": "PREFIJO"},
|
||||
{"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]},
|
||||
{"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]},
|
||||
{"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla us_tariff_fractions."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(
|
||||
row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions"
|
||||
) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -17,23 +17,28 @@ from .dto import (
|
||||
)
|
||||
from .service import USTariffFractionService
|
||||
|
||||
# Create router using TenantCRUDRoutes factory for basic CRUD operations
|
||||
# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router)
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=USTariffFractionService,
|
||||
create_schema=USTariffFractionCreateDTO,
|
||||
update_schema=USTariffFractionUpdateDTO,
|
||||
response_schema=USTariffFractionResponseDTO,
|
||||
prefix="/us-tariff-fractions",
|
||||
prefix="",
|
||||
tags=["a76 / general catalogs / us tariff fractions"],
|
||||
resource_name="US Tariff Fraction",
|
||||
id_name="id",
|
||||
enable_list=False, # We implement our custom list endpoint
|
||||
)
|
||||
|
||||
router = crud_router.router
|
||||
# Master router with prefix so all routes live under /us-tariff-fractions
|
||||
from .imports.routes import router as imports_router
|
||||
main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"])
|
||||
main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"])
|
||||
main_router.include_router(crud_router.router)
|
||||
|
||||
# Custom list endpoint with search filter
|
||||
@router.get(
|
||||
|
||||
# Custom list endpoint with search filter (under /us-tariff-fractions/)
|
||||
@main_router.get(
|
||||
"/",
|
||||
response_model=Dict[str, Any],
|
||||
summary="List US Tariff Fractions",
|
||||
@@ -65,3 +70,6 @@ async def list_us_tariff_fractions(
|
||||
"page_size": page_size,
|
||||
"pages": (total + page_size - 1) // page_size,
|
||||
}
|
||||
|
||||
|
||||
router = main_router
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
import base64
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
@@ -12,24 +13,39 @@ from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .tasks import scan_file, insert_valid_rows
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
IMPORT_FILE_KEY_PREFIX,
|
||||
IMPORT_META_KEY_PREFIX,
|
||||
IMPORT_REDIS_TTL,
|
||||
)
|
||||
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Redis client (same broker as Celery so worker can read)."""
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
@router.post("/upload/{model_target}", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
model_target: Literal["invoice_header", "invoice_details"],
|
||||
file: UploadFile = File(...),
|
||||
footer_config: Optional[str] = Form(None), # JSON string with settings
|
||||
company_id: int = Query(..., description="Company ID"), # Required for context
|
||||
template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas
|
||||
company_id: int = Query(..., description="Company ID"), # Required for context
|
||||
operation_type: Optional[str] = Query("imp"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Step 1: Upload CSV, save to temp, trigger scan task.
|
||||
Si se envía template_id, solo se leen las columnas de esa plantilla.
|
||||
"""
|
||||
# 1. Validate Access & Get Tenant
|
||||
try:
|
||||
@@ -40,40 +56,51 @@ async def upload_import_file(
|
||||
|
||||
if not file.filename.endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Only .csv files allowed")
|
||||
|
||||
|
||||
job_id = str(uuid4())
|
||||
|
||||
# Ensure directory exists (Safety check)
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"operation_type": operation_type,
|
||||
"template_id": template_id,
|
||||
}
|
||||
|
||||
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
|
||||
try:
|
||||
# Save CSV
|
||||
contents = await file.read()
|
||||
redis_client = _get_redis()
|
||||
redis_client.set(
|
||||
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
redis_client.set(
|
||||
f"{IMPORT_META_KEY_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
|
||||
|
||||
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(contents)
|
||||
|
||||
# Save Metadata (Context)
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"operation_type": operation_type,
|
||||
}
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"File save error: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
logger.warning(f"Local file save failed (worker will use Redis): {e}")
|
||||
|
||||
# Trigger Celery Task (Async)
|
||||
# Use our job_id as the Celery task_id for easier tracking
|
||||
scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id)
|
||||
# Trigger Celery Task (Async). Worker loads file from Redis.
|
||||
scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
@@ -84,24 +111,56 @@ async def upload_import_file(
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Poll this endpoint to get % progress or final report.
|
||||
Poll to get progress or final report. Always returns an object with "status".
|
||||
"""
|
||||
# In a real app, query Redis or DB.
|
||||
# For MVP, we might mock or use Celery AsyncResult if backend shares Redis.
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == 'PENDING':
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
elif task_result.state == 'PROGRESS':
|
||||
if task_result.state == "PROGRESS":
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": task_result.info.get('current', 0),
|
||||
"total": task_result.info.get('total', 0)
|
||||
"status": "processing",
|
||||
"progress": (task_result.info or {}).get("current", 0),
|
||||
"total": (task_result.info or {}).get("total", 0),
|
||||
}
|
||||
elif task_result.state == 'SUCCESS':
|
||||
return task_result.result # Should return the report
|
||||
else:
|
||||
return {"status": task_result.state, "error": str(task_result.info)}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
|
||||
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb:
|
||||
logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg and len(lines) > 1:
|
||||
err_msg = lines[-2] + " " + (lines[-1] or "")
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg:
|
||||
result = getattr(task_result, "result", None)
|
||||
info = getattr(task_result, "info", None)
|
||||
if result is not None and not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif isinstance(result, dict) and (result.get("error") or result.get("message")):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
if not err_msg and isinstance(info, str):
|
||||
err_msg = info
|
||||
elif not err_msg and isinstance(info, dict) and "error" in info:
|
||||
err_msg = str(info["error"])
|
||||
if not err_msg:
|
||||
err_msg = "Task failed"
|
||||
return {"status": "failed", "error": err_msg}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
118
backend/api/v1/modules/a76/imports/template_config.py
Normal file
118
backend/api/v1/modules/a76/imports/template_config.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean.
|
||||
La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV).
|
||||
# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que
|
||||
# espera la lógica de validación e insert (tasks.py).
|
||||
# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA).
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
# --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) ---
|
||||
"imp_temp_header": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
|
||||
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
|
||||
{"canonical": "FECHA EMISION"},
|
||||
{"canonical": "CLAVE PROVEEDOR"},
|
||||
{"canonical": "CLAVE VENDIDO A"},
|
||||
{"canonical": "CLAVE ENVIADO A"},
|
||||
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
|
||||
{"canonical": "ADUANA DE CRUCE"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
{"canonical": "CLAVE INCOTERM"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "TIPO DE CAMBIO"},
|
||||
{"canonical": "TIPO PESO"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "REMESA"},
|
||||
{"canonical": "AGENTE ADUANAL"},
|
||||
{"canonical": "FLETES"},
|
||||
{"canonical": "VALOR SEGUROS"},
|
||||
{"canonical": "SEGUROS"},
|
||||
{"canonical": "EMBALAJES"},
|
||||
{"canonical": "OTROS INCREMENTABLES"},
|
||||
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
|
||||
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
{"canonical": "FACTURA ALTERNA"},
|
||||
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
|
||||
{"canonical": "OBSERVACIONES E"},
|
||||
{"canonical": "OBSERVACIONES I"},
|
||||
{"canonical": "E DOCUMENT"},
|
||||
{"canonical": "NUM OPERACION"},
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "PRECINTO"},
|
||||
],
|
||||
# --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura ---
|
||||
"imp_def_header": None, # se resuelve igual que imp_temp_header
|
||||
# --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura ---
|
||||
"exp_def_header": None,
|
||||
# --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) ---
|
||||
"imp_temp_details": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
|
||||
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
|
||||
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
|
||||
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
|
||||
{"canonical": "CANTIDAD"},
|
||||
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
|
||||
{"canonical": "FRACCION"},
|
||||
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
],
|
||||
# --- Partidas: Impo Def y Expo - misma estructura ---
|
||||
"imp_def_details": None,
|
||||
"exp_def_details": None,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]:
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if cols is not None:
|
||||
return cols
|
||||
if template_id in ("imp_def_header", "exp_def_header"):
|
||||
return TEMPLATE_COLUMNS.get("imp_temp_header")
|
||||
if template_id in ("imp_def_details", "exp_def_details"):
|
||||
return TEMPLATE_COLUMNS.get("imp_temp_details")
|
||||
return None
|
||||
|
||||
|
||||
def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]:
|
||||
"""
|
||||
Construye un diccionario: normalized_header -> canonical_name.
|
||||
normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos).
|
||||
"""
|
||||
cols = _resolve_template_columns(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]:
|
||||
"""
|
||||
A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict
|
||||
solo con las columnas de la plantilla, usando nombres canónicos.
|
||||
Así la plantilla se respeta: solo entran columnas definidas en la plantilla.
|
||||
"""
|
||||
lookup = build_normalized_lookup(template_id, normalize_header_fn)
|
||||
if not lookup:
|
||||
# Sin template definido: comportamiento legacy (normalizar todo)
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
from api.v1.modules.public.reference_data.transport_types.models import TransportType
|
||||
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen
|
||||
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode
|
||||
|
||||
@@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel):
|
||||
operation_type: Optional[OperationType] = Field(
|
||||
..., description="Operation type: imp/exp/sm/ctm"
|
||||
)
|
||||
|
||||
@field_validator("operation_type", mode="before")
|
||||
@classmethod
|
||||
def normalize_operation_type(cls, v):
|
||||
"""Accept DB string (e.g. 'IMP') and coerce to enum value ('imp')."""
|
||||
if isinstance(v, str):
|
||||
return v.lower() if v else v
|
||||
return v
|
||||
invoice_type: Optional[str] = Field(
|
||||
None, max_length=5, description="Invoice type key"
|
||||
)
|
||||
@@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel):
|
||||
class InvoiceFinancialsBase(BaseModel):
|
||||
"""Base fields for Financials"""
|
||||
|
||||
currency: Currency = Field(None, max_length=7, description="Currency code")
|
||||
currency: Optional[Currency] = Field(None, max_length=7, description="Currency code")
|
||||
currency_type: Optional[str] = Field("USD", description="Currency type")
|
||||
exchange_rate: Decimal = Field(0.00, description="Exchange rate")
|
||||
exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate")
|
||||
exchange_rate_mm: Optional[Decimal] = Field(
|
||||
None, description="Exchange rate currency to currency"
|
||||
)
|
||||
|
||||
@@ -67,11 +67,13 @@ class InvoiceService:
|
||||
|
||||
# Apply filters if provided
|
||||
if filters:
|
||||
if filters.get("status"):
|
||||
if filters.get("status") is not None:
|
||||
query = query.filter(models.InvoiceHeader.is_updated == filters["status"])
|
||||
if filters.get("operation_type"):
|
||||
ot = filters["operation_type"]
|
||||
ot_val = ot.value if hasattr(ot, "value") else ot
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.operation_type == filters["operation_type"]
|
||||
models.InvoiceHeader.operation_type == ot_val
|
||||
)
|
||||
if filters.get("invoice_type"):
|
||||
query = query.filter(
|
||||
@@ -89,10 +91,9 @@ class InvoiceService:
|
||||
f"%{filters['pedimento']}%"
|
||||
)
|
||||
)
|
||||
if (
|
||||
not filters.get("invoice_type")
|
||||
and filters.get("operation_type") == "exp"
|
||||
):
|
||||
ot_exp = filters.get("operation_type")
|
||||
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
|
||||
if not filters.get("invoice_type") and ot_exp_val == "exp":
|
||||
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
|
||||
|
||||
if filters.get("manifest_number"):
|
||||
|
||||
1
backend/api/v1/modules/a76/parts/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/parts/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV import for Parts (Números de parte)
|
||||
151
backend/api/v1/modules/a76/parts/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/parts/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para Números de Parte.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
PART_IMPORT_FILE_PREFIX,
|
||||
PART_IMPORT_META_PREFIX,
|
||||
PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "part_numbers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{PART_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Parts import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
22
backend/api/v1/modules/a76/parts/imports/schemas.py
Normal file
22
backend/api/v1/modules/a76/parts/imports/schemas.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
639
backend/api/v1/modules/a76/parts/imports/tasks.py
Normal file
639
backend/api/v1/modules/a76/parts/imports/tasks.py
Normal file
@@ -0,0 +1,639 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Números de Parte.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PART_IMPORT_FILE_PREFIX = "part_import_file:"
|
||||
PART_IMPORT_META_PREFIX = "part_import_meta:"
|
||||
PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:"
|
||||
PART_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"part_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PART_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_part(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_class_codes: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
valid_currency_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
part_number = (row.get("NUMPARTE") or "").strip()
|
||||
if not part_number:
|
||||
return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"}
|
||||
if len(part_number) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
commercial = (row.get("NUMPARTECOM") or "").strip()
|
||||
if commercial and len(commercial) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
part_class = (row.get("CLASE") or "").strip()
|
||||
if part_class and len(part_class) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
# Si CLASE no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
# Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
currency_key = (row.get("MONEDA") or "").strip()
|
||||
if currency_key and len(currency_key) > 3:
|
||||
return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"}
|
||||
# Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
unit_cost = row.get("COSTOUNIT")
|
||||
if unit_cost is not None and unit_cost != "":
|
||||
try:
|
||||
Decimal(str(unit_cost))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"}
|
||||
|
||||
unit_weight = row.get("PESOUNIT")
|
||||
if unit_weight is not None and unit_weight != "":
|
||||
try:
|
||||
Decimal(str(unit_weight))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 10:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
fda_key = (row.get("FDAKEY") or "").strip()
|
||||
if fda_key and len(fda_key) > 20:
|
||||
return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"}
|
||||
fcc_key = (row.get("FCCKEY") or "").strip()
|
||||
if fcc_key and len(fcc_key) > 30:
|
||||
return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"}
|
||||
license_code = (row.get("LICENCIA") or "").strip()
|
||||
if license_code and len(license_code) > 3:
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"}
|
||||
eccn = (row.get("ECCN") or "").strip()
|
||||
if eccn and len(eccn) > 20:
|
||||
return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"}
|
||||
export_code = (row.get("EXPORTCODE") or "").strip()
|
||||
if export_code and len(export_code) > 2:
|
||||
return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"}
|
||||
exclusion = (row.get("EXCLUSION") or "").strip()
|
||||
if exclusion and len(exclusion) > 19:
|
||||
return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"}
|
||||
weight_type = (row.get("TIPOPESO") or "").strip()
|
||||
if weight_type and len(weight_type) > 6:
|
||||
return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Parts import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"part_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _bool_from_row(val: Any) -> bool:
|
||||
if val is None or val == "":
|
||||
return True
|
||||
s = str(val).strip().upper()
|
||||
if s in ("0", "F", "FALSE", "NO", "N"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Parts import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"part_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Parts import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_part_number: Dict[str, Part] = {}
|
||||
for p in (
|
||||
session.query(Part)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_part_number[p.part_number] = p
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
part_number = _str_or_none(row_norm.get("NUMPARTE"), 70)
|
||||
if not part_number:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
part_class = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if part_class and part_class not in valid_class_codes:
|
||||
part_class = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
currency_key = _str_or_none(row_norm.get("MONEDA"), 3)
|
||||
if currency_key and currency_key not in valid_currency_codes:
|
||||
currency_key = None
|
||||
|
||||
unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT"))
|
||||
currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None
|
||||
unit_weight = _decimal_or_none(row_norm.get("PESOUNIT"))
|
||||
weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6)
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 10)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
fda_key = _str_or_none(row_norm.get("FDAKEY"), 20)
|
||||
fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30)
|
||||
license_code = _str_or_none(row_norm.get("LICENCIA"), 3)
|
||||
eccn = _str_or_none(row_norm.get("ECCN"), 20)
|
||||
export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2)
|
||||
exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19)
|
||||
is_active = _bool_from_row(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_part_number.get(part_number)
|
||||
if existing:
|
||||
existing.commercial_part_number = commercial
|
||||
existing.description_spanish = desc_es
|
||||
existing.description_english = desc_en
|
||||
existing.part_class = part_class
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.unit_cost = unit_cost
|
||||
existing.currency_type = currency_type
|
||||
existing.currency_key = currency_key
|
||||
existing.unit_weight = unit_weight
|
||||
existing.weight_type = weight_type
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.fda_key = fda_key
|
||||
existing.fcc_key = fcc_key
|
||||
existing.license_code = license_code
|
||||
existing.eccn = eccn
|
||||
existing.export_code = export_code
|
||||
existing.exclusion_symbol = exclusion_symbol
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_part = Part(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
client_id=company_id,
|
||||
part_number=part_number,
|
||||
commercial_part_number=commercial,
|
||||
description_spanish=desc_es,
|
||||
description_english=desc_en,
|
||||
part_class=part_class,
|
||||
unit_of_measure=unit_of_measure,
|
||||
unit_cost=unit_cost,
|
||||
currency_type=currency_type,
|
||||
currency_key=currency_key,
|
||||
unit_weight=unit_weight,
|
||||
weight_type=weight_type,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
fda_key=fda_key,
|
||||
fcc_key=fcc_key,
|
||||
license_code=license_code,
|
||||
eccn=eccn,
|
||||
export_code=export_code,
|
||||
exclusion_symbol=exclusion_symbol,
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_part)
|
||||
existing_by_part_number[part_number] = new_part
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Parts import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Parts import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
54
backend/api/v1/modules/a76/parts/imports/template_config.py
Normal file
54
backend/api/v1/modules/a76/parts/imports/template_config.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Números de Parte (EstructuraCatPartesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"part_numbers": [
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE", "PART NUMBER", "NUM PARTE"]},
|
||||
{"canonical": "NUMPARTECOM", "aliases": ["NUMERO PARTE COMERCIAL", "COMMERCIAL PART", "PARTE COMERCIAL"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES", "DESC ESPANOL"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION", "DESC INGLES"]},
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CLASE MATERIAL", "PART CLASS"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM", "UNIT OF MEASURE"]},
|
||||
{"canonical": "COSTOUNIT", "aliases": ["COSTO UNITARIO", "UNIT COST", "COSTO"]},
|
||||
{"canonical": "MONEDA", "aliases": ["CURRENCY", "MONEDA CLAVE", "CURRENCY KEY"]},
|
||||
{"canonical": "PESOUNIT", "aliases": ["PESO UNITARIO", "UNIT WEIGHT", "PESO"]},
|
||||
{"canonical": "TIPOPESO", "aliases": ["WEIGHT TYPE", "TIPO PESO"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "FDAKEY", "aliases": ["FDA", "FDA KEY"]},
|
||||
{"canonical": "FCCKEY", "aliases": ["FCC", "FCC KEY"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["LICENSE CODE", "LICENSE"]},
|
||||
{"canonical": "ECCN", "aliases": ["ECCN CODE"]},
|
||||
{"canonical": "EXPORTCODE", "aliases": ["EXPORT CODE", "CODIGO EXPORT"]},
|
||||
{"canonical": "EXCLUSION", "aliases": ["EXCLUSION SYMBOL", "SIMBOLO EXCLUSION"]},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS ACTIVE", "ACTIVE", "ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("part_numbers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -2,21 +2,31 @@
|
||||
Endpoints API para gestión de partes (SCAII)
|
||||
"""
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import PartCreateDTO, PartResponseDTO, PartUpdateDTO
|
||||
from .service import PartService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
router = TenantCRUDRoutes(
|
||||
service=PartService,
|
||||
create_schema=PartCreateDTO,
|
||||
update_schema=PartUpdateDTO,
|
||||
response_schema=PartResponseDTO,
|
||||
prefix="/parts",
|
||||
tags=["a76 / parts"],
|
||||
resource_name="Part",
|
||||
id_name="part_id",
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
).router
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/parts/imports", tags=["a76 / parts / csv_import"])
|
||||
|
||||
# CRUD
|
||||
router.include_router(
|
||||
TenantCRUDRoutes(
|
||||
service=PartService,
|
||||
create_schema=PartCreateDTO,
|
||||
update_schema=PartUpdateDTO,
|
||||
response_schema=PartResponseDTO,
|
||||
prefix="/parts",
|
||||
tags=["a76 / parts"],
|
||||
resource_name="Part",
|
||||
id_name="part_id",
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
).router
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Pedimentos (upload → scan → commit)
|
||||
160
backend/api/v1/modules/a76/pedmientos/imports/routes.py
Normal file
160
backend/api/v1/modules/a76/pedmientos/imports/routes.py
Normal file
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Pedimentos.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
PED_IMPORT_FILE_PREFIX,
|
||||
PED_IMPORT_META_PREFIX,
|
||||
PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "pedimentos",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
25
backend/api/v1/modules/a76/pedmientos/imports/schemas.py
Normal file
25
backend/api/v1/modules/a76/pedmientos/imports/schemas.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: Optional[str] = None
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
563
backend/api/v1/modules/a76/pedmientos/imports/tasks.py
Normal file
563
backend/api/v1/modules/a76/pedmientos/imports/tasks.py
Normal file
@@ -0,0 +1,563 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Pedimentos.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con otros imports)
|
||||
PED_IMPORT_FILE_PREFIX = "ped_import_file:"
|
||||
PED_IMPORT_META_PREFIX = "ped_import_meta:"
|
||||
PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:"
|
||||
PED_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"ped_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_pedimento(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_client_ids: Set[int],
|
||||
valid_regimes: Set[str],
|
||||
valid_pedimento_codes: Set[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Pedimento. Retorna error dict o None."""
|
||||
# Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN
|
||||
year = (row.get("AÑO") or "").strip()
|
||||
if not year:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Requerido"}
|
||||
if len(year) > 2:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"}
|
||||
|
||||
customs_office = (row.get("ADUANA") or "").strip()
|
||||
if not customs_office:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Requerido"}
|
||||
if len(customs_office) > 3:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"}
|
||||
|
||||
license_val = (row.get("PATENTE") or "").strip()
|
||||
if not license_val:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Requerido"}
|
||||
if len(license_val) > 4:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()
|
||||
if not pedimento_number:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Requerido"}
|
||||
if len(pedimento_number) > 7:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
if not client_id_str:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"}
|
||||
try:
|
||||
client_id = int(client_id_str)
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"}
|
||||
if client_id not in valid_client_ids:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"}
|
||||
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()
|
||||
if not pedimento_code:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"}
|
||||
if len(pedimento_code) > 2:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"}
|
||||
if pedimento_code not in valid_pedimento_codes:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"}
|
||||
|
||||
regime = (row.get("REGIMEN") or "").strip()
|
||||
if not regime:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"}
|
||||
if len(regime) > 3:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"}
|
||||
if regime not in valid_regimes:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"}
|
||||
|
||||
# Optional numeric/string fields - validate format if present
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status and len(status) > 30:
|
||||
return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
continue
|
||||
try:
|
||||
Decimal(val.replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return {"line": line_num, "col": col, "msg": "Valor numérico inválido"}
|
||||
|
||||
operation_type = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if operation_type and operation_type not in ("imp", "exp", ""):
|
||||
return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"}
|
||||
|
||||
pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""):
|
||||
return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).strip().replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict for PedimentosCreate from normalized CSV row (canonical names)."""
|
||||
year = (row.get("AÑO") or "").strip()[:2]
|
||||
customs_office = (row.get("ADUANA") or "").strip()[:3]
|
||||
license_val = (row.get("PATENTE") or "").strip()[:4]
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()[:7]
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
client_id = int(client_id_str) if client_id_str else None
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2]
|
||||
regime = (row.get("REGIMEN") or "").strip()[:3]
|
||||
|
||||
data = {
|
||||
"year": year,
|
||||
"customs_office": customs_office,
|
||||
"license": license_val,
|
||||
"pedimento_number": pedimento_number,
|
||||
"client_id": client_id,
|
||||
"pedimento_code": pedimento_code,
|
||||
"regime": regime,
|
||||
}
|
||||
|
||||
op = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if op in ("imp", "exp"):
|
||||
data["operation_type"] = op
|
||||
|
||||
ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if ptype in ("normal", "consolidated", "complementary", "automobile"):
|
||||
data["pedimento_type"] = ptype
|
||||
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status:
|
||||
data["status"] = status[:30]
|
||||
|
||||
data["usd_value"] = _parse_decimal(row.get("VALOR_USD"))
|
||||
data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO"))
|
||||
data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO"))
|
||||
data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO"))
|
||||
|
||||
obs = (row.get("OBSERVACIONES") or "").strip()
|
||||
if obs:
|
||||
data["observations"] = obs
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Load valid FK sets for validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Reload FK sets for commit-time validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate
|
||||
from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
data = _row_to_pedimentos_create(row_norm)
|
||||
# Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL
|
||||
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
||||
data["pedimento_dates"] = PedimentoDatesCreate(
|
||||
entry_date=datetime.now(),
|
||||
end_date=datetime.now(),
|
||||
)
|
||||
create_data = PedimentosCreate(**data)
|
||||
PedimentosService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except ValueError as ve:
|
||||
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
else:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import line {i}: {e}")
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(e)})
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"pedimentos": [
|
||||
{"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]},
|
||||
{"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]},
|
||||
{"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]},
|
||||
{"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]},
|
||||
{"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]},
|
||||
{"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]},
|
||||
{"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]},
|
||||
{"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]},
|
||||
{"canonical": "REGIMEN", "aliases": ["REGIME"]},
|
||||
{"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]},
|
||||
{"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]},
|
||||
{"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]},
|
||||
{"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]},
|
||||
{"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]},
|
||||
{"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla pedimentos."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import (
|
||||
from .routes.pedimento_transport_means import router as pedimento_transport_means_router
|
||||
from .routes.pedimento_validation import router as pedimento_validation_router
|
||||
from .routes.pedimentos import router as pedimentos_router
|
||||
from .imports.routes import router as pedimentos_imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -117,3 +118,8 @@ router.include_router(
|
||||
router.include_router(
|
||||
pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"]
|
||||
)
|
||||
router.include_router(
|
||||
pedimentos_imports_router,
|
||||
prefix="/pedimentos/imports",
|
||||
tags=["a76 / pedimentos / csv_import"],
|
||||
)
|
||||
|
||||
@@ -12,9 +12,10 @@ from .general_catalogs.router import router as general_catalogs_router
|
||||
from .invoices.routes import router as invoices_router
|
||||
from .items.routes import router as items_router
|
||||
from .classes import router as classes_router
|
||||
from .classes import router as classes_router
|
||||
|
||||
from .clients_and_providers import router as client_and_provider_router
|
||||
from .imports.routes import router as imports_router
|
||||
from .csv_templates.routes import router as csv_templates_router
|
||||
from .invoice_settings.routes import router as invoice_settings_router
|
||||
from .item_presets.routes import router as item_presets_router
|
||||
from .general_catalogs.company import router as company_router
|
||||
@@ -23,6 +24,7 @@ from .transportation.drivers.routes import router as drivers_router
|
||||
from .doc_types_dig.routes import router as doc_types_dig_router
|
||||
from .fraction_rule_octave.routes import router as fraction_rule_octave_router
|
||||
from .parts import router as parts_router
|
||||
from .boms import router as boms_router
|
||||
from .pedmientos.router import router as pedimentos_router
|
||||
from .permission_rule_oct.routes import router as permission_rule_oct_router
|
||||
from .transportation.trailers.routes import router as trailers_router
|
||||
@@ -53,12 +55,14 @@ router.include_router(general_catalogs_router, prefix="/a76", tags=["a76 / gener
|
||||
router.include_router(invoices_router, prefix="/a76", tags=["a76 / invoices"])
|
||||
router.include_router(items_router, prefix="/a76", tags=["a76 / items"])
|
||||
router.include_router(imports_router, prefix="/a76/imports", tags=["a76 / imports"])
|
||||
router.include_router(csv_templates_router, prefix="/a76/csv-templates", tags=["a76 / csv_templates"])
|
||||
router.include_router(invoice_settings_router)
|
||||
router.include_router(item_presets_router, prefix="/a76/item-presets", tags=["a76 / item_presets"])
|
||||
router.include_router(pedimentos_router, prefix="/a76")
|
||||
router.include_router(client_and_provider_router, prefix="/a76", tags=["a76 / clients_and_providers"])
|
||||
router.include_router(classes_router, prefix="/a76/classes", tags=["a76 / classes"])
|
||||
router.include_router(parts_router, prefix="/a76", tags=["a76 / parts"])
|
||||
router.include_router(boms_router, prefix="/a76/boms", tags=["a76 / boms"])
|
||||
router.include_router(permission_rule_oct_router, prefix="/a76", tags=["a76 / permission_rule_oct"])
|
||||
router.include_router(fraction_rule_octave_router, prefix="/a76", tags=["a76 / fraction_rule_octave"])
|
||||
router.include_router(country_rule_oct_router, prefix="/a76", tags=["a76 / country_rule_oct"])
|
||||
|
||||
@@ -6,28 +6,28 @@ from pydantic import BaseModel
|
||||
class DriverBaseDTO(BaseModel):
|
||||
transporter_key: str
|
||||
line: int
|
||||
driver_name: Optional[str]
|
||||
license_number: Optional[str]
|
||||
express_line_id: Optional[str]
|
||||
ace_id: Optional[str]
|
||||
birth_date: Optional[int]
|
||||
gender: Optional[str]
|
||||
birth_country: Optional[str]
|
||||
hazardous_material_auth: Optional[str]
|
||||
hazardous_material_state: Optional[str]
|
||||
first_name: Optional[str]
|
||||
last_name: Optional[str]
|
||||
id_key1: Optional[str]
|
||||
id_number1: Optional[str]
|
||||
id_state1: Optional[str]
|
||||
id_country1: Optional[str]
|
||||
id_key2: Optional[str]
|
||||
id_number2: Optional[str]
|
||||
id_state2: Optional[str]
|
||||
id_country2: Optional[str]
|
||||
badge_number: Optional[str]
|
||||
class_type: Optional[str]
|
||||
unique_badge_number: Optional[str]
|
||||
driver_name: Optional[str] = None
|
||||
license_number: Optional[str] = None
|
||||
express_line_id: Optional[str] = None
|
||||
ace_id: Optional[str] = None
|
||||
birth_date: Optional[int] = None
|
||||
gender: Optional[str] = None
|
||||
birth_country: Optional[str] = None
|
||||
hazardous_material_auth: Optional[str] = None
|
||||
hazardous_material_state: Optional[str] = None
|
||||
first_name: Optional[str] = None
|
||||
last_name: Optional[str] = None
|
||||
id_key1: Optional[str] = None
|
||||
id_number1: Optional[str] = None
|
||||
id_state1: Optional[str] = None
|
||||
id_country1: Optional[str] = None
|
||||
id_key2: Optional[str] = None
|
||||
id_number2: Optional[str] = None
|
||||
id_state2: Optional[str] = None
|
||||
id_country2: Optional[str] = None
|
||||
badge_number: Optional[str] = None
|
||||
class_type: Optional[str] = None
|
||||
unique_badge_number: Optional[str] = None
|
||||
company_id: int
|
||||
tenant_id: int
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Drivers / Conductores (upload -> scan -> status -> commit).
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Rutas de importacion CSV para Conductores.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
DRV_IMPORT_FILE_PREFIX,
|
||||
DRV_IMPORT_META_PREFIX,
|
||||
DRV_IMPORT_STATUS_PREFIX,
|
||||
DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "drivers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Drivers import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not write processing status: {e}")
|
||||
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Drivers import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Insercion iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
updated: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,640 @@
|
||||
"""
|
||||
Tareas Celery para importacion CSV de Conductores.
|
||||
Flujo: scan_file (validacion) -> insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
|
||||
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
|
||||
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
|
||||
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
|
||||
DRV_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"drv_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
_MAX = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
"badge_number": 20,
|
||||
"class_type": 1,
|
||||
"unique_badge_number": 100,
|
||||
}
|
||||
|
||||
|
||||
def _parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
transporter_key = (row.get("TRANSPORTISTA") or "").strip()
|
||||
if not transporter_key:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"}
|
||||
if len(transporter_key) > _MAX["transporter_key"]:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"}
|
||||
|
||||
line_val = _parse_int(row.get("LINEA"))
|
||||
if line_val is None:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"}
|
||||
if line_val <= 0:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"}
|
||||
|
||||
driver_name = (row.get("CLAVE CONDUCTOR") or "").strip()
|
||||
if driver_name and len(driver_name) > _MAX["driver_name"]:
|
||||
return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("LICENCIA", _MAX["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", _MAX["express_line_id"]),
|
||||
("IDENTIFICACION ACE", _MAX["ace_id"]),
|
||||
("SEXO", _MAX["gender"]),
|
||||
("PAIS NACIMIENTO", _MAX["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]),
|
||||
("NOMBRE(S)", _MAX["first_name"]),
|
||||
("APELLIDO PATERNO", _MAX["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", _MAX["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", _MAX["id_number1"]),
|
||||
("ESTADO", _MAX["id_state1"]),
|
||||
("PAIS", _MAX["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", _MAX["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", _MAX["id_number2"]),
|
||||
("ESTADO 2", _MAX["id_state2"]),
|
||||
("PAIS 2", _MAX["id_country2"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
|
||||
fecha = row.get("FECHA NACIMIENTO")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_birth_date(fecha) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA NACIMIENTO",
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"])
|
||||
line = _parse_int(row.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
data = {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]),
|
||||
"license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]),
|
||||
"express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]),
|
||||
"ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]),
|
||||
"birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row.get("SEXO"), _MAX["gender"]),
|
||||
"birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]),
|
||||
"last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]),
|
||||
"id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]),
|
||||
"id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]),
|
||||
"id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]),
|
||||
"id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]),
|
||||
"id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]),
|
||||
"id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]),
|
||||
"id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]),
|
||||
"id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Drivers import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.drivers.services import DriverService
|
||||
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_driver_dto(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("transporter_key") or data.get("line") is None:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
key = f"{data['transporter_key']}:{data['line']}"
|
||||
if key in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[key] = i
|
||||
|
||||
existing = DriverService.get_driver_by_key_and_line(
|
||||
session, data["transporter_key"], data["line"], str(company_id), tenant_id
|
||||
)
|
||||
try:
|
||||
if existing:
|
||||
update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")}
|
||||
for field, value in update_fields.items():
|
||||
setattr(existing, field, value)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = DriverCreateDTO(**data)
|
||||
DriverService.create_driver(session, create_data)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Drivers import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Drivers import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros validos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Drivers import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Configuracion de plantilla CSV para Conductores (EstructuraCatConductor.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"drivers": [
|
||||
{"canonical": "TRANSPORTISTA", "aliases": ["TRANSPORTISTA CLAVE", "CLAVE TRANSPORTISTA", "TRANSPORTER"]},
|
||||
{"canonical": "LINEA", "aliases": ["LINE", "LINEA CONDUCTOR"]},
|
||||
{"canonical": "CLAVE CONDUCTOR", "aliases": ["CONDUCTOR", "DRIVER", "NOMBRE CONDUCTOR"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["LICENSE", "LICENCE"]},
|
||||
{"canonical": "PERMISO LINEA EXPRESS", "aliases": ["LINEA EXPRESS", "LINEA EXPRESS ID", "EXPRESS LINE"]},
|
||||
{"canonical": "IDENTIFICACION ACE", "aliases": ["ACE", "ACE ID", "IDENTIFICACION ACE ID"]},
|
||||
{"canonical": "FECHA NACIMIENTO", "aliases": ["FECHA NAC", "BIRTH DATE", "BIRTHDATE"]},
|
||||
{"canonical": "SEXO", "aliases": ["GENERO", "GENDER"]},
|
||||
{"canonical": "PAIS NACIMIENTO", "aliases": ["PAIS Nacimiento", "BIRTH COUNTRY"]},
|
||||
{"canonical": "TRANSPORTA MAT. PELIGROSO?", "aliases": ["MATERIAL PELIGROSO", "HAZMAT", "HAZARDOUS MATERIAL"]},
|
||||
{"canonical": "PERMISO MAT. PELIGROSO", "aliases": ["PERMISO MAT PELIGROSO", "HAZMAT PERMIT"]},
|
||||
{"canonical": "NOMBRE(S)", "aliases": ["NOMBRE", "NOMBRES", "FIRST NAME"]},
|
||||
{"canonical": "APELLIDO PATERNO", "aliases": ["APELLIDO", "LAST NAME", "APELLIDO P"]},
|
||||
{"canonical": "FORMA IDENTIFICACION 1", "aliases": ["FORMA IDENTIFICACION", "ID TIPO 1", "ID KEY 1"]},
|
||||
{"canonical": "NUM. IDENTIFICACION 1", "aliases": ["NUM IDENTIFICACION 1", "ID NUMERO 1", "ID NUMBER 1"]},
|
||||
{"canonical": "ESTADO", "aliases": ["ESTADO 1", "STATE 1"]},
|
||||
{"canonical": "PAIS", "aliases": ["PAIS 1", "COUNTRY 1"]},
|
||||
{"canonical": "FORMA IDENTIFICACION 2", "aliases": ["ID TIPO 2", "ID KEY 2"]},
|
||||
{"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]},
|
||||
{"canonical": "ESTADO 2", "aliases": ["STATE 2"]},
|
||||
{"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("drivers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -8,9 +8,13 @@ from sqlalchemy.orm import Session
|
||||
from .dto import DriverCreateDTO, DriverResponseDTO
|
||||
from .models import Driver
|
||||
from .services import DriverService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter(prefix="/drivers")
|
||||
|
||||
# CSV import (upload -> scan -> status -> commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / drivers / csv_import"])
|
||||
|
||||
|
||||
@router.get("/", response_model=Dict[str, Any])
|
||||
async def list_drivers(
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Trailers CSV import: upload -> scan -> status -> commit
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Rutas de importación CSV para Trailers y Cajas.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
TRL_IMPORT_FILE_PREFIX,
|
||||
TRL_IMPORT_META_PREFIX,
|
||||
TRL_IMPORT_STATUS_PREFIX,
|
||||
TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "trailers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Trailers import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not write processing status: {e}")
|
||||
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Trailers import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
updated: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,535 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Trailers y Cajas.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Upsert por trailer_number usando TrailerService.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRL_IMPORT_FILE_PREFIX = "trl_import_file:"
|
||||
TRL_IMPORT_META_PREFIX = "trl_import_meta:"
|
||||
TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:"
|
||||
TRL_IMPORT_STATUS_PREFIX = "trl_import_status:"
|
||||
TRL_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"trl_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Trailer model (a76.trailer)
|
||||
_MAX = {
|
||||
"trailer_number": 20,
|
||||
"ace_trailer_number": 10,
|
||||
"trailer_type_key": 2,
|
||||
"seal": 15,
|
||||
"entity_code": 1,
|
||||
"plate_number": 17,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"container_key": 3,
|
||||
}
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Trailer. Retorna error dict o None."""
|
||||
trailer_number = (row.get("NUMERO TRAILER") or "").strip()
|
||||
if not trailer_number:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"}
|
||||
if len(trailer_number) > _MAX["trailer_number"]:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_trailer_number"]),
|
||||
("TIPO TRAILER", _MAX["trailer_type_key"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("CODIGO ENTIDAD", _MAX["entity_code"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("CLAVE CONTENEDOR", _MAX["container_key"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
"""Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row."""
|
||||
trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"])
|
||||
if not trailer_number:
|
||||
return {}
|
||||
return {
|
||||
"trailer_number": trailer_number,
|
||||
"ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]),
|
||||
"trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]),
|
||||
}
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Trailers import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.trailers.services import TrailerService
|
||||
from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_trailer_dto(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("trailer_number"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
tn = data["trailer_number"]
|
||||
if tn in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": tn,
|
||||
"invoice": tn,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[tn] = i
|
||||
|
||||
existing = TrailerService.get_by_id(session, tn, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"})
|
||||
TrailerService.update(session, tn, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = TrailerCreateDTO(**data)
|
||||
TrailerService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Trailers import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Trailers import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Trailers import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Trailers / Cajas (EstructuraCatTrailers.xls).
|
||||
Mapeo: NUMERO TRAILER → trailer_number, CLAVE ACE → ace_trailer_number, etc.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"trailers": [
|
||||
{"canonical": "NUMERO TRAILER", "aliases": ["CLAVE TRAILER", "TRAILER NUMBER", "TRAILER", "NUMERO"]},
|
||||
{"canonical": "CLAVE ACE", "aliases": ["ACE", "NUMERO ACE", "ACE TRAILER"]},
|
||||
{"canonical": "TIPO TRAILER", "aliases": ["TIPO", "TRAILER TYPE", "TIPO CAJA"]},
|
||||
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
|
||||
{"canonical": "CODIGO ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE", "CODIGO DE ENTIDAD"]},
|
||||
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE NUMBER", "PLATE"]},
|
||||
{"canonical": "ESTADO", "aliases": ["STATE"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
|
||||
{"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("trailers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -1,11 +1,19 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO
|
||||
from .services import TrailerService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create router using TenantCRUDRoutes factory
|
||||
# Note: trailer_number is a string (not int) and is used as the primary key
|
||||
router = TenantCRUDRoutes(
|
||||
# Main router: trailers CRUD + CSV imports
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/trailers/imports", tags=["a76 / trailers / csv_import"])
|
||||
|
||||
# CRUD routes
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=TrailerService,
|
||||
create_schema=TrailerCreateDTO,
|
||||
update_schema=TrailerUpdateDTO,
|
||||
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
|
||||
prefix="/trailers",
|
||||
tags=[],
|
||||
resource_name="Trailer",
|
||||
id_name="trailer_number", # Using trailer_number instead of numeric ID
|
||||
id_type=str, # Specify that the ID is a string
|
||||
enable_list=True, # Enable GET /trailers with pagination
|
||||
enable_filters=True, # Enable filtering by plate_number and trailer_type_key
|
||||
id_name="trailer_number",
|
||||
id_type=str,
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
default_page_size=50,
|
||||
max_page_size=100,
|
||||
).router
|
||||
router.include_router(crud_router)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Vehicles / Transportes (upload → scan → status → commit).
|
||||
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
Rutas de importación CSV para Vehículos (Transportes).
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
VEHL_IMPORT_FILE_PREFIX,
|
||||
VEHL_IMPORT_META_PREFIX,
|
||||
VEHL_IMPORT_STATUS_PREFIX,
|
||||
VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "vehicles",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
# Fallback sin worker: ejecutar scan en un hilo y guardar resultado en Redis
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Vehicles import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
# Si el resultado del scan está en Redis (worker o hilo de fallback), usarlo
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
# Escribir "processing" en Redis para que el polling muestre estado hasta que termine el commit
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not write processing status: {e}")
|
||||
|
||||
# Ejecutar commit en hilo (mismo fallback que el scan: no depende del worker de Celery)
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Vehicles import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
# Mismo job_id para que el front siga haciendo polling y reciba el resultado de Redis
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,607 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Vehículos (Transportes).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VEHL_IMPORT_FILE_PREFIX = "veh_import_file:"
|
||||
VEHL_IMPORT_META_PREFIX = "veh_import_meta:"
|
||||
VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:"
|
||||
VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:"
|
||||
VEHL_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"veh_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Vehicle model (a76.vehicle)
|
||||
_MAX = {
|
||||
"vehicle_key": 14,
|
||||
"ace_vehicle_key": 10,
|
||||
"transporter_key": 23,
|
||||
"transport_identifier": 30,
|
||||
"transport_type": 2,
|
||||
"entity_code": 1,
|
||||
"transponder_number": 16,
|
||||
"dot_number": 8,
|
||||
"plate_number": 17,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"seal": 49,
|
||||
"insurance_company_name": 30,
|
||||
"insurance_number": 20,
|
||||
"series": 30,
|
||||
}
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
s = str(val).strip().replace(",", "")
|
||||
return Decimal(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_insurance_date(val: Any) -> Optional[int]:
|
||||
"""Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD."""
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
# Already integer-like
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
# Try DD/MM/YYYY or similar
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Vehículo. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > _MAX["vehicle_key"]:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"}
|
||||
|
||||
# Optional fields: max lengths only
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_vehicle_key"]),
|
||||
("CLAVE TRANSPORTE", _MAX["transporter_key"]),
|
||||
("VIN", _MAX["series"]),
|
||||
("TIPO TRANSPORTE", _MAX["transport_type"]),
|
||||
("CODIGO DE ENTIDAD", _MAX["entity_code"]),
|
||||
("TRANSPONDEDOR", _MAX["transponder_number"]),
|
||||
("NUMERO DOT", _MAX["dot_number"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("CIUDAD", _MAX["city"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]),
|
||||
("NUM. ASEGURADORA", _MAX["insurance_number"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
# MONTO ASEGURADO: must be numeric if present
|
||||
monto = row.get("MONTO ASEGURADO") or row.get("MONTO")
|
||||
if monto is not None and str(monto).strip():
|
||||
if _parse_decimal(monto) is None:
|
||||
return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"}
|
||||
|
||||
# FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None)
|
||||
# Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either
|
||||
# reject or set None. We reject invalid date to keep data quality.
|
||||
fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_insurance_date(fecha) is None:
|
||||
return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _do_scan(
|
||||
job_id: str,
|
||||
progress_callback: Optional[Any] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync.
|
||||
progress_callback(current, total, error_count) opcional.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row."""
|
||||
vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"])
|
||||
if not vehicle_key:
|
||||
return {}
|
||||
data = {
|
||||
"vehicle_key": vehicle_key,
|
||||
"ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]),
|
||||
"transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]),
|
||||
"series": _str_or_none(row.get("VIN"), _MAX["series"]),
|
||||
"transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]),
|
||||
"transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]),
|
||||
"dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"city": _str_or_none(row.get("CIUDAD"), _MAX["city"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]),
|
||||
"insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]),
|
||||
"insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")),
|
||||
"insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.vehicles.services import VehicleService
|
||||
from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_vehicle_dto(row_norm)
|
||||
if not data or not data.get("vehicle_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
vk = data["vehicle_key"]
|
||||
if vk in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[vk] = i
|
||||
|
||||
existing = VehicleService.get_by_id(session, vk, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"})
|
||||
VehicleService.update(session, vk, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = VehicleCreateDTO(**data)
|
||||
VehicleService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Vehicles import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Vehicles import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el commit en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Vehículos / Transportes (EstructuraCatTransportes.xls).
|
||||
Mapeo: CLAVE → vehicle_key, CLAVE ACE → ace_vehicle_key, etc.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"vehicles": [
|
||||
{"canonical": "CLAVE", "aliases": ["CLAVE VEHICULO", "VEHICLE KEY", "KEY"]},
|
||||
{"canonical": "CLAVE ACE", "aliases": ["ACE", "CLAVEACE"]},
|
||||
{"canonical": "CLAVE TRANSPORTE", "aliases": ["TRANSPORTE", "TRANSPORTISTA"]},
|
||||
{"canonical": "VIN", "aliases": ["SERIE", "SERIES", "NUMERO SERIE"]},
|
||||
{"canonical": "TIPO TRANSPORTE", "aliases": ["TIPO", "TRANSPORT TYPE"]},
|
||||
{"canonical": "CODIGO DE ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE"]},
|
||||
{"canonical": "TRANSPONDEDOR", "aliases": ["TRANSPONDER", "TRANSPONDER NUMBER"]},
|
||||
{"canonical": "NUMERO DOT", "aliases": ["DOT", "NUM DOT"]},
|
||||
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE", "PLATE NUMBER"]},
|
||||
{"canonical": "CIUDAD", "aliases": ["CITY"]},
|
||||
{"canonical": "ESTADO", "aliases": ["STATE"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
|
||||
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
|
||||
{"canonical": "EMPRESA ASEGURADORA", "aliases": ["ASEGURADORA", "INSURANCE COMPANY"]},
|
||||
{"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]},
|
||||
{"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]},
|
||||
{"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("vehicles")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -1,11 +1,19 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO
|
||||
from .services import VehicleService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create router using TenantCRUDRoutes factory
|
||||
# Note: vehicle_key is a string (not int) and is used as the primary key
|
||||
router = TenantCRUDRoutes(
|
||||
# Main router: vehicles CRUD + CSV imports
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/vehicles/imports", tags=["a76 / vehicles / csv_import"])
|
||||
|
||||
# CRUD routes
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=VehicleService,
|
||||
create_schema=VehicleCreateDTO,
|
||||
update_schema=VehicleUpdateDTO,
|
||||
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
|
||||
prefix="/vehicles",
|
||||
tags=[],
|
||||
resource_name="Vehicle",
|
||||
id_name="vehicle_key", # Using vehicle_key instead of numeric ID
|
||||
id_type=str, # Specify that the ID is a string
|
||||
enable_list=True, # Enable GET /vehicles with pagination
|
||||
enable_filters=True, # Enable filtering by plate_number and transport_type
|
||||
id_name="vehicle_key",
|
||||
id_type=str,
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
default_page_size=50,
|
||||
max_page_size=100,
|
||||
).router
|
||||
router.include_router(crud_router)
|
||||
|
||||
Reference in New Issue
Block a user