Merge pull request 'feature/exchange-type' (#181) from feature/exchange-type into development
Reviewed-on: ADUANASOFT/anexo76#181
This commit is contained in:
8
.gitignore
vendored
8
.gitignore
vendored
@@ -24,9 +24,11 @@ wheels/
|
||||
*.egg
|
||||
.pnpm-store/
|
||||
|
||||
# Environment
|
||||
# Environment (no subir: cada quien puede usar puertos distintos vía .env)
|
||||
.env
|
||||
.env.local
|
||||
backend/.env
|
||||
frontend/.env
|
||||
backend/SCRIPTS/
|
||||
# IDEs
|
||||
.vscode/
|
||||
@@ -68,6 +70,6 @@ backend/uploads/
|
||||
docker-compose.yml
|
||||
.mypy_cache/
|
||||
|
||||
|
||||
# Celery
|
||||
celerybeat-schedule
|
||||
backend/celerybeat-schedule
|
||||
celerybeat-schedule
|
||||
|
||||
0
backend/api/__init__.py
Normal file
0
backend/api/__init__.py
Normal file
0
backend/api/v1/__init__.py
Normal file
0
backend/api/v1/__init__.py
Normal file
0
backend/api/v1/modules/__init__.py
Normal file
0
backend/api/v1/modules/__init__.py
Normal file
0
backend/api/v1/modules/a76/__init__.py
Normal file
0
backend/api/v1/modules/a76/__init__.py
Normal file
7
backend/api/v1/modules/a76/boms/__init__.py
Normal file
7
backend/api/v1/modules/a76/boms/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
Módulo de importación CSV para BOMs (Bills of Materials).
|
||||
"""
|
||||
|
||||
from .routes import router
|
||||
|
||||
__all__ = ["router"]
|
||||
1
backend/api/v1/modules/a76/boms/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/boms/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# boms.imports
|
||||
151
backend/api/v1/modules/a76/boms/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/boms/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para BOMs.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
BOM_IMPORT_FILE_PREFIX,
|
||||
BOM_IMPORT_META_PREFIX,
|
||||
BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "boms",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
22
backend/api/v1/modules/a76/boms/imports/schemas.py
Normal file
22
backend/api/v1/modules/a76/boms/imports/schemas.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
430
backend/api/v1/modules/a76/boms/imports/tasks.py
Normal file
430
backend/api/v1/modules/a76/boms/imports/tasks.py
Normal file
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"bom_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila BOM según columnas del template. FK opcional a a76.parts."""
|
||||
parent = (row.get("NUMPARTE_PADRE") or "").strip()
|
||||
if not parent:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"}
|
||||
if len(parent) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
component = (row.get("NUMPARTE_COMPONENTE") or "").strip()
|
||||
if not component:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"}
|
||||
if len(component) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
qty = row.get("CANTIDAD")
|
||||
if qty is None or qty == "":
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"}
|
||||
try:
|
||||
val = Decimal(str(qty))
|
||||
if val < 0:
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 10:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
version_bom = row.get("VERSION_BOM")
|
||||
if not _optional_number(version_bom):
|
||||
return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"}
|
||||
|
||||
version_bill = row.get("VERSION_BILL")
|
||||
if not _optional_number(version_bill):
|
||||
return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"}
|
||||
|
||||
# Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas)
|
||||
if valid_part_numbers is not None and len(valid_part_numbers) > 0:
|
||||
if parent not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"}
|
||||
if component not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"BOMs import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts for FK validation: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"BOMs import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"BOMs import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
valid_count = 0
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
# Cuando exista la tabla de destino, aquí se hará insert/update.
|
||||
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"BOMs import cleanup failed: {cleanup_err}")
|
||||
|
||||
return response
|
||||
42
backend/api/v1/modules/a76/boms/imports/template_config.py
Normal file
42
backend/api/v1/modules/a76/boms/imports/template_config.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para BOMs (EstructuraBOMS.xlsx).
|
||||
Placeholders hasta tener el XLS definitivo; ajustar canónicos y aliases según el archivo.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"boms": [
|
||||
{"canonical": "NUMPARTE_PADRE", "aliases": ["PARTE PADRE", "PART NUMBER", "PARENT PART", "NUM PARTE PADRE"]},
|
||||
{"canonical": "NUMPARTE_COMPONENTE", "aliases": ["PARTE COMPONENTE", "COMPONENT PART", "NUM PARTE COMPONENTE"]},
|
||||
{"canonical": "CANTIDAD", "aliases": ["QTY", "QUANTITY", "CANT"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "VERSION_BOM", "aliases": ["VERSION BOM", "BOM VERSION", "VERSIONBOM"]},
|
||||
{"canonical": "VERSION_BILL", "aliases": ["VERSION BILL", "BILL VERSION", "VERSIONBILL"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("boms")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
13
backend/api/v1/modules/a76/boms/routes.py
Normal file
13
backend/api/v1/modules/a76/boms/routes.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Endpoints para importación CSV de BOMs.
|
||||
Mismo patrón que parts y classes: upload → scan → status → commit.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / boms / csv_import"])
|
||||
@@ -2,6 +2,20 @@
|
||||
Módulo de Class
|
||||
"""
|
||||
|
||||
from .routes import router
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["router"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""
|
||||
Lazy export to avoid circular imports during Celery init.
|
||||
|
||||
This package is imported when models are loaded (e.g. a76.items.models),
|
||||
so importing FastAPI routes at import-time can break Celery startup.
|
||||
"""
|
||||
if name == "router":
|
||||
from .routes import router
|
||||
|
||||
return router
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
1
backend/api/v1/modules/a76/classes/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/classes/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV import for Clases de Materiales (upload → scan → status → commit).
|
||||
151
backend/api/v1/modules/a76/classes/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/classes/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para Clases de Materiales.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CLS_IMPORT_FILE_PREFIX,
|
||||
CLS_IMPORT_META_PREFIX,
|
||||
CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "material_classes",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
21
backend/api/v1/modules/a76/classes/imports/schemas.py
Normal file
21
backend/api/v1/modules/a76/classes/imports/schemas.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
528
backend/api/v1/modules/a76/classes/imports/tasks.py
Normal file
528
backend/api/v1/modules/a76/classes/imports/tasks.py
Normal file
@@ -0,0 +1,528 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
if not class_code:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
|
||||
if len(class_code) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
material_key = (row.get("CLAVEMAT") or "").strip()
|
||||
if material_key:
|
||||
if len(material_key) > 10:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
|
||||
if valid_material_keys is not None and material_key not in valid_material_keys:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom:
|
||||
if len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
if valid_uom_codes is not None and uom not in valid_uom_codes:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 20:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
sub_key = (row.get("CLAVESUB") or "").strip()
|
||||
if sub_key and len(sub_key) > 5:
|
||||
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
|
||||
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if iva_exempt and len(iva_exempt) > 4:
|
||||
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
rev_fisica = row.get("REVFISICA")
|
||||
if rev_fisica is not None and rev_fisica != "":
|
||||
try:
|
||||
v = int(rev_fisica)
|
||||
if v < -32768 or v > 32767:
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Classes import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Classes import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code: Dict[str, Class] = {}
|
||||
for c in (
|
||||
session.query(Class)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_code[c.class_code] = c
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
if existing:
|
||||
existing.description_es = desc_es
|
||||
existing.description_en = desc_en
|
||||
existing.material_key = material_key
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.sub_key = sub_key
|
||||
existing.physical_review = physical_review
|
||||
existing.iva_exempt_fraction = iva_exempt_fraction
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
class_code=class_code,
|
||||
description_es=desc_es,
|
||||
description_en=desc_en,
|
||||
material_key=material_key,
|
||||
unit_of_measure=unit_of_measure,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
sub_key=sub_key,
|
||||
physical_review=physical_review,
|
||||
iva_exempt_fraction=iva_exempt_fraction,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Classes import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t
|
||||
|
||||
from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse
|
||||
from .service import ClassService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create a new router for custom endpoints
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"])
|
||||
|
||||
# Add consolidated catalog endpoints FIRST (before generic CRUD routes)
|
||||
# This ensures they have priority over the generic /{id} route
|
||||
@router.get(
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# CSV import flow for Clientes y Proveedores (Client Providers).
|
||||
# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit.
|
||||
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Rutas de importación CSV para Clientes y Proveedores.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CP_IMPORT_FILE_PREFIX,
|
||||
CP_IMPORT_META_PREFIX,
|
||||
CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "client_providers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("CP import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientOrProviderEnum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active(val: Optional[str]) -> bool:
|
||||
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
|
||||
rfc = (row.get("RFC") or "").strip()
|
||||
if not rfc:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
|
||||
if len(rfc) > 30:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
|
||||
name = (row.get("NOMBRE") or "").strip()
|
||||
if len(name) > 256:
|
||||
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
if short_name and len(short_name) > 10:
|
||||
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
curp = (row.get("CURP") or "").strip()
|
||||
if curp and len(curp) > 19:
|
||||
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CP import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
|
||||
Upsert por (tenant_id, company_id, rfc).
|
||||
"""
|
||||
logger.info(f"CP import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CP import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
rfc = _str_or_none(row_norm.get("RFC"), 30)
|
||||
if not rfc:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
|
||||
if client_or_provider is None:
|
||||
client_or_provider = ClientOrProviderEnum.BOTH
|
||||
|
||||
is_active = _parse_active(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
|
||||
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
|
||||
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
|
||||
existing.client_or_provider = client_or_provider
|
||||
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
rfc=rfc,
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 256),
|
||||
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
|
||||
curp=_str_or_none(row_norm.get("CURP"), 19),
|
||||
client_or_provider=client_or_provider,
|
||||
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
|
||||
# Opcional: crear dirección si hay email/teléfono/dirección
|
||||
email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
|
||||
if email or phone or address_str:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_str,
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
phone=phone,
|
||||
email=email,
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
|
||||
)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CP import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CP import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"client_providers": [
|
||||
{"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]},
|
||||
{"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
|
||||
{"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]},
|
||||
{"canonical": "CURP", "aliases": []},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]},
|
||||
{"canonical": "ESTADO", "aliases": []},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]},
|
||||
{"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "INCOTERM", "aliases": []},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla client_providers."""
|
||||
cols = TEMPLATE_COLUMNS.get("client_providers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -20,10 +20,14 @@ from .dto import (
|
||||
)
|
||||
from .service import ClientProviderService
|
||||
from .models import ClientProvider
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create main router to add custom endpoints
|
||||
router = APIRouter(prefix="/clients-providers")
|
||||
|
||||
# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"])
|
||||
|
||||
|
||||
@router.get("/", response_model=ClientProviderPaginatedResponseDTO)
|
||||
async def get_clients_and_providers(
|
||||
|
||||
1
backend/api/v1/modules/a76/csv_templates/__init__.py
Normal file
1
backend/api/v1/modules/a76/csv_templates/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV templates: generate CSV from code (no physical XLS/XLSX files)
|
||||
147
backend/api/v1/modules/a76/csv_templates/registry.py
Normal file
147
backend/api/v1/modules/a76/csv_templates/registry.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Registro central de plantillas CSV: template_id -> lista de cabeceras canónicas.
|
||||
Construido a partir de los TEMPLATE_COLUMNS de cada módulo de imports.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# Importar configs de cada módulo
|
||||
from api.v1.modules.a76.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as IMPORTS_TEMPLATE_COLUMNS,
|
||||
_resolve_template_columns as resolve_imports_template,
|
||||
)
|
||||
from api.v1.modules.a76.parts.imports.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.boms.imports.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.classes.imports.template_config import TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.customs_brokers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as CUSTOMS_BROKERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.clients_and_providers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as CLIENTS_PROVIDERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as EXCHANGE_RATE_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.pedmientos.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as PEDIMENTOS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.vehicles.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as VEHICLES_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.drivers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as DRIVERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.trailers.imports.template_config import (
|
||||
TEMPLATE_COLUMNS as TRAILERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
|
||||
|
||||
def _canonicals_from_columns(cols: Optional[List[Dict]]) -> List[str]:
|
||||
"""Extrae la lista de nombres canónicos en orden a partir de una lista de columnas."""
|
||||
if not cols:
|
||||
return []
|
||||
return [item["canonical"] for item in cols]
|
||||
|
||||
|
||||
def _build_registry() -> Dict[str, List[str]]:
|
||||
registry: Dict[str, List[str]] = {}
|
||||
|
||||
# a76/imports (facturas): imp_temp_header, imp_temp_details, imp_def_*, exp_def_*
|
||||
for tid in ("imp_temp_header", "imp_temp_details", "imp_def_header", "imp_def_details", "exp_def_header", "exp_def_details"):
|
||||
cols = resolve_imports_template(tid)
|
||||
registry[tid] = _canonicals_from_columns(cols)
|
||||
|
||||
# part_numbers (parts); "items" usa la misma plantilla
|
||||
part_cols = PARTS_TEMPLATE_COLUMNS.get("part_numbers")
|
||||
registry["part_numbers"] = _canonicals_from_columns(part_cols)
|
||||
registry["items"] = _canonicals_from_columns(part_cols)
|
||||
|
||||
# boms
|
||||
registry["boms"] = _canonicals_from_columns(BOMS_TEMPLATE_COLUMNS.get("boms"))
|
||||
|
||||
# material_classes
|
||||
registry["material_classes"] = _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes"))
|
||||
|
||||
# customs_brokers
|
||||
registry["customs_brokers"] = _canonicals_from_columns(CUSTOMS_BROKERS_TEMPLATE_COLUMNS.get("customs_brokers"))
|
||||
|
||||
# clients_providers
|
||||
registry["clients_providers"] = _canonicals_from_columns(CLIENTS_PROVIDERS_TEMPLATE_COLUMNS.get("client_providers"))
|
||||
|
||||
# exchange_rates
|
||||
registry["exchange_rates"] = _canonicals_from_columns(EXCHANGE_RATE_TEMPLATE_COLUMNS.get("exchange_rates"))
|
||||
|
||||
# american_fractions
|
||||
registry["american_fractions"] = _canonicals_from_columns(
|
||||
US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS.get("us_tariff_fractions")
|
||||
)
|
||||
|
||||
# pedimentos
|
||||
registry["pedimentos"] = _canonicals_from_columns(PEDIMENTOS_TEMPLATE_COLUMNS.get("pedimentos"))
|
||||
|
||||
# transports (vehicles)
|
||||
registry["transports"] = _canonicals_from_columns(VEHICLES_TEMPLATE_COLUMNS.get("vehicles"))
|
||||
|
||||
# drivers
|
||||
registry["drivers"] = _canonicals_from_columns(DRIVERS_TEMPLATE_COLUMNS.get("drivers"))
|
||||
|
||||
# trailers
|
||||
registry["trailers"] = _canonicals_from_columns(TRAILERS_TEMPLATE_COLUMNS.get("trailers"))
|
||||
|
||||
return registry
|
||||
|
||||
|
||||
_TEMPLATE_HEADERS: Dict[str, List[str]] = _build_registry()
|
||||
|
||||
# Nombre de archivo sugerido para descarga (sin path)
|
||||
TEMPLATE_FILENAMES: Dict[str, str] = {
|
||||
"customs_brokers": "EstructuraCatAgenteAduanal.csv",
|
||||
"clients_providers": "EstructuraCatClienteProv.csv",
|
||||
"exchange_rates": "EstructuraCatTiposCambio.csv",
|
||||
"american_fractions": "EstructuraCatFraccAme.csv",
|
||||
"material_classes": "EstructuraCatClasesAF.csv",
|
||||
"part_numbers": "EstructuraCatPartesAF.csv",
|
||||
"items": "EstructuraCatPartesAF.csv",
|
||||
"boms": "EstructuraBOMS.csv",
|
||||
"pedimentos": "EstructuraCatPedimentos.csv",
|
||||
"transports": "EstructuraCatTransportes.csv",
|
||||
"drivers": "EstructuraCatConductor.csv",
|
||||
"trailers": "EstructuraCatTrailers.csv",
|
||||
"imp_temp_header": "EstructuraEncFacImpoTemp.csv",
|
||||
"imp_temp_details": "EstructuraParFacImpoTempAF.csv",
|
||||
"imp_def_header": "EstructuraEncFacImpoDef.csv",
|
||||
"imp_def_details": "EstructuraParFacImpoDefAF.csv",
|
||||
"exp_def_header": "EstructuraEncFacExpoCamReg.csv",
|
||||
"exp_def_details": "EstructuraParExpoCamReg.csv",
|
||||
}
|
||||
|
||||
|
||||
def get_template_headers(template_id: str) -> Optional[List[str]]:
|
||||
"""Devuelve la lista de cabeceras canónicas para el template_id, o None si no existe."""
|
||||
return _TEMPLATE_HEADERS.get(template_id)
|
||||
|
||||
|
||||
def get_template_filename(template_id: str) -> str:
|
||||
"""Nombre de archivo sugerido para la descarga."""
|
||||
return TEMPLATE_FILENAMES.get(template_id, f"plantilla_{template_id}.csv")
|
||||
|
||||
|
||||
def generate_csv_content(template_id: str, include_bom: bool = True) -> Optional[bytes]:
|
||||
"""
|
||||
Genera el contenido CSV (solo fila de cabeceras) para el template_id.
|
||||
UTF-8, opcionalmente con BOM para Excel.
|
||||
"""
|
||||
headers = get_template_headers(template_id)
|
||||
if not headers:
|
||||
return None
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf, lineterminator="\n")
|
||||
writer.writerow(headers)
|
||||
content = buf.getvalue().encode("utf-8")
|
||||
if include_bom:
|
||||
content = b"\xef\xbb\xbf" + content
|
||||
return content
|
||||
35
backend/api/v1/modules/a76/csv_templates/routes.py
Normal file
35
backend/api/v1/modules/a76/csv_templates/routes.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Rutas para descargar plantillas CSV generadas desde código (sin archivos XLS/XLSX).
|
||||
"""
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import Response
|
||||
|
||||
from core.security import get_current_user
|
||||
|
||||
from .registry import generate_csv_content, get_template_filename
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/{template_id}", response_class=Response)
|
||||
async def download_csv_template(
|
||||
template_id: str,
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Devuelve un CSV con solo la fila de cabeceras para la plantilla indicada.
|
||||
Las cabeceras son los nombres canónicos definidos en cada template_config.
|
||||
"""
|
||||
content = generate_csv_content(template_id, include_bom=True)
|
||||
if content is None:
|
||||
raise HTTPException(status_code=404, detail=f"Plantilla desconocida: {template_id}")
|
||||
filename = get_template_filename(template_id)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# CSV import flow for Agentes Aduanales (Customs Brokers).
|
||||
# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit.
|
||||
161
backend/api/v1/modules/a76/customs_brokers/imports/routes.py
Normal file
161
backend/api/v1/modules/a76/customs_brokers/imports/routes.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Rutas de importación CSV para Agentes Aduanales.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CB_IMPORT_FILE_PREFIX,
|
||||
CB_IMPORT_META_PREFIX,
|
||||
CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "customs_brokers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("CB import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
447
backend/api/v1/modules/a76/customs_brokers/imports/tasks.py
Normal file
447
backend/api/v1/modules/a76/customs_brokers/imports/tasks.py
Normal file
@@ -0,0 +1,447 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con a76.imports)
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > 5:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CB import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Guardar números de línea con error en Redis para insert_valid_rows
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
|
||||
"""
|
||||
logger.info(f"CB import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CB import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
clave = (row_norm.get("CLAVE") or "").strip()[:5]
|
||||
if not clave:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
|
||||
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
|
||||
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
|
||||
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
|
||||
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
|
||||
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
|
||||
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
|
||||
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
|
||||
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
existing.license = lic[:4] if lic and lic.isdigit() else None
|
||||
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
|
||||
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
license_val = lic[:4] if lic and lic.isdigit() else None
|
||||
new_broker = CustomsBroker(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
broker_key=clave,
|
||||
type=_str_or_none(row_norm.get("TIPO"), 9),
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 80),
|
||||
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
|
||||
fax=_str_or_none(row_norm.get("FAX"), 30),
|
||||
email=_str_or_none(row_norm.get("EMAIL"), 100),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
tax_id=_str_or_none(row_norm.get("RFC"), 30),
|
||||
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
license=license_val,
|
||||
company=_str_or_none(row_norm.get("EMPRESA"), 200),
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
|
||||
)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CB import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Limpieza
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CB import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"customs_brokers": [
|
||||
{"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]},
|
||||
{"canonical": "TIPO"},
|
||||
{"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD"},
|
||||
{"canonical": "ESTADO"},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]},
|
||||
{"canonical": "FAX"},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]},
|
||||
{"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]},
|
||||
{"canonical": "EMPRESA", "aliases": ["COMPANY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla customs_brokers."""
|
||||
cols = TEMPLATE_COLUMNS.get("customs_brokers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from . import dto, services
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (mismo flujo que a76.imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"])
|
||||
|
||||
customs_broker_crud = TenantCRUDRoutes(
|
||||
service=services.CustomsBrokerService,
|
||||
create_schema=dto.CustomsBrokerCreateDTO,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Exchange rate CSV import module
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Tipos de Cambio.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
ER_IMPORT_FILE_PREFIX,
|
||||
ER_IMPORT_META_PREFIX,
|
||||
ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "exchange_rates",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("ER import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,465 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if _parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
|
||||
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
|
||||
if local_raw and len(local_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
|
||||
if foreign_raw and len(foreign_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"ER import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
|
||||
"""
|
||||
logger.info(f"ER import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"ER import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
parsed_date = _parse_date(row_norm.get("FECHA"))
|
||||
if not parsed_date:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
|
||||
continue
|
||||
|
||||
try:
|
||||
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
|
||||
value_decimal = Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
|
||||
continue
|
||||
|
||||
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
|
||||
if local_currency:
|
||||
local_currency = local_currency.upper()
|
||||
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
|
||||
if foreign_currency:
|
||||
foreign_currency = foreign_currency.upper()
|
||||
|
||||
key_date = parsed_date.date()
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
|
||||
if existing:
|
||||
existing.value = value_decimal
|
||||
existing.local_currency = local_currency or None
|
||||
existing.foreign_currency = foreign_currency or None
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
date=parsed_date,
|
||||
value=value_decimal,
|
||||
local_currency=local_currency,
|
||||
foreign_currency=foreign_currency,
|
||||
)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"ER import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"ER import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"exchange_rates": [
|
||||
{"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]},
|
||||
{"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]},
|
||||
{"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]},
|
||||
{"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla exchange_rates."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -34,6 +34,10 @@ crud_router = route_handler.router
|
||||
from fastapi import APIRouter
|
||||
custom_router = APIRouter(prefix="/exchange-rate", tags=[])
|
||||
|
||||
# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered
|
||||
from .imports.routes import router as imports_router
|
||||
custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"])
|
||||
|
||||
@custom_router.get("/test-ping")
|
||||
async def test_ping():
|
||||
return {"message": "pong"}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Fracción Americana (US Tariff Fractions).
|
||||
Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
FA_IMPORT_FILE_PREFIX,
|
||||
FA_IMPORT_META_PREFIX,
|
||||
FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "us_tariff_fractions",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("FA import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,474 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
|
||||
FA_IMPORT_META_PREFIX = "fa_import_meta:"
|
||||
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
|
||||
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:16] if len(s) > 16 else s
|
||||
|
||||
|
||||
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
|
||||
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
code_norm = _normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
if len(code_norm) > 16:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
|
||||
|
||||
prefix_raw = (row.get("PREFIJO") or "").strip()
|
||||
if prefix_raw and len(prefix_raw) > 10:
|
||||
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
|
||||
if um_raw and len(um_raw) > 10:
|
||||
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
|
||||
if tipo_raw and len(tipo_raw) > 10:
|
||||
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
adv_pct = row.get("ADVALOREM_PCT")
|
||||
if adv_pct is not None and str(adv_pct).strip():
|
||||
try:
|
||||
v = float(str(adv_pct).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
|
||||
|
||||
adv_dlls = row.get("ADVALOREM_DLLS")
|
||||
if adv_dlls is not None and str(adv_dlls).strip():
|
||||
try:
|
||||
v = float(str(adv_dlls).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"FA import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(str(val).strip().replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
|
||||
"""
|
||||
logger.info(f"FA import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"FA import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
|
||||
continue
|
||||
|
||||
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
|
||||
description = _str_or_none(row_norm.get("DESCRIPCION"))
|
||||
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
|
||||
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
|
||||
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.prefix = prefix
|
||||
existing.type_code = type_code
|
||||
existing.ad_valorem = ad_valorem
|
||||
existing.fixed_cost = fixed_cost
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.description = description
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_row = USTariffFraction(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
code=code,
|
||||
prefix=prefix,
|
||||
type_code=type_code,
|
||||
ad_valorem=ad_valorem,
|
||||
fixed_cost=fixed_cost,
|
||||
unit_of_measure=unit_of_measure,
|
||||
description=description,
|
||||
)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"FA import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path_clean = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path_clean):
|
||||
os.remove(meta_path_clean)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"FA import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"us_tariff_fractions": [
|
||||
{"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]},
|
||||
{"canonical": "PREFIJO"},
|
||||
{"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]},
|
||||
{"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]},
|
||||
{"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla us_tariff_fractions."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(
|
||||
row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions"
|
||||
) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -17,23 +17,28 @@ from .dto import (
|
||||
)
|
||||
from .service import USTariffFractionService
|
||||
|
||||
# Create router using TenantCRUDRoutes factory for basic CRUD operations
|
||||
# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router)
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=USTariffFractionService,
|
||||
create_schema=USTariffFractionCreateDTO,
|
||||
update_schema=USTariffFractionUpdateDTO,
|
||||
response_schema=USTariffFractionResponseDTO,
|
||||
prefix="/us-tariff-fractions",
|
||||
prefix="",
|
||||
tags=["a76 / general catalogs / us tariff fractions"],
|
||||
resource_name="US Tariff Fraction",
|
||||
id_name="id",
|
||||
enable_list=False, # We implement our custom list endpoint
|
||||
)
|
||||
|
||||
router = crud_router.router
|
||||
# Master router with prefix so all routes live under /us-tariff-fractions
|
||||
from .imports.routes import router as imports_router
|
||||
main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"])
|
||||
main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"])
|
||||
main_router.include_router(crud_router.router)
|
||||
|
||||
# Custom list endpoint with search filter
|
||||
@router.get(
|
||||
|
||||
# Custom list endpoint with search filter (under /us-tariff-fractions/)
|
||||
@main_router.get(
|
||||
"/",
|
||||
response_model=Dict[str, Any],
|
||||
summary="List US Tariff Fractions",
|
||||
@@ -65,3 +70,6 @@ async def list_us_tariff_fractions(
|
||||
"page_size": page_size,
|
||||
"pages": (total + page_size - 1) // page_size,
|
||||
}
|
||||
|
||||
|
||||
router = main_router
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
import base64
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
@@ -12,24 +13,39 @@ from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .tasks import scan_file, insert_valid_rows
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
IMPORT_FILE_KEY_PREFIX,
|
||||
IMPORT_META_KEY_PREFIX,
|
||||
IMPORT_REDIS_TTL,
|
||||
)
|
||||
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Redis client (same broker as Celery so worker can read)."""
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
@router.post("/upload/{model_target}", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
model_target: Literal["invoice_header", "invoice_details"],
|
||||
file: UploadFile = File(...),
|
||||
footer_config: Optional[str] = Form(None), # JSON string with settings
|
||||
company_id: int = Query(..., description="Company ID"), # Required for context
|
||||
template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas
|
||||
company_id: int = Query(..., description="Company ID"), # Required for context
|
||||
operation_type: Optional[str] = Query("imp"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Step 1: Upload CSV, save to temp, trigger scan task.
|
||||
Si se envía template_id, solo se leen las columnas de esa plantilla.
|
||||
"""
|
||||
# 1. Validate Access & Get Tenant
|
||||
try:
|
||||
@@ -40,40 +56,51 @@ async def upload_import_file(
|
||||
|
||||
if not file.filename.endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Only .csv files allowed")
|
||||
|
||||
|
||||
job_id = str(uuid4())
|
||||
|
||||
# Ensure directory exists (Safety check)
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"operation_type": operation_type,
|
||||
"template_id": template_id,
|
||||
}
|
||||
|
||||
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
|
||||
try:
|
||||
# Save CSV
|
||||
contents = await file.read()
|
||||
redis_client = _get_redis()
|
||||
redis_client.set(
|
||||
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
redis_client.set(
|
||||
f"{IMPORT_META_KEY_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
|
||||
|
||||
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(contents)
|
||||
|
||||
# Save Metadata (Context)
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"operation_type": operation_type,
|
||||
}
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"File save error: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
logger.warning(f"Local file save failed (worker will use Redis): {e}")
|
||||
|
||||
# Trigger Celery Task (Async)
|
||||
# Use our job_id as the Celery task_id for easier tracking
|
||||
scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id)
|
||||
# Trigger Celery Task (Async). Worker loads file from Redis.
|
||||
scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
@@ -84,24 +111,56 @@ async def upload_import_file(
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Poll this endpoint to get % progress or final report.
|
||||
Poll to get progress or final report. Always returns an object with "status".
|
||||
"""
|
||||
# In a real app, query Redis or DB.
|
||||
# For MVP, we might mock or use Celery AsyncResult if backend shares Redis.
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == 'PENDING':
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
elif task_result.state == 'PROGRESS':
|
||||
if task_result.state == "PROGRESS":
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": task_result.info.get('current', 0),
|
||||
"total": task_result.info.get('total', 0)
|
||||
"status": "processing",
|
||||
"progress": (task_result.info or {}).get("current", 0),
|
||||
"total": (task_result.info or {}).get("total", 0),
|
||||
}
|
||||
elif task_result.state == 'SUCCESS':
|
||||
return task_result.result # Should return the report
|
||||
else:
|
||||
return {"status": task_result.state, "error": str(task_result.info)}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
|
||||
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb:
|
||||
logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg and len(lines) > 1:
|
||||
err_msg = lines[-2] + " " + (lines[-1] or "")
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg:
|
||||
result = getattr(task_result, "result", None)
|
||||
info = getattr(task_result, "info", None)
|
||||
if result is not None and not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif isinstance(result, dict) and (result.get("error") or result.get("message")):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
if not err_msg and isinstance(info, str):
|
||||
err_msg = info
|
||||
elif not err_msg and isinstance(info, dict) and "error" in info:
|
||||
err_msg = str(info["error"])
|
||||
if not err_msg:
|
||||
err_msg = "Task failed"
|
||||
return {"status": "failed", "error": err_msg}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
118
backend/api/v1/modules/a76/imports/template_config.py
Normal file
118
backend/api/v1/modules/a76/imports/template_config.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean.
|
||||
La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV).
|
||||
# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que
|
||||
# espera la lógica de validación e insert (tasks.py).
|
||||
# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA).
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
# --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) ---
|
||||
"imp_temp_header": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
|
||||
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
|
||||
{"canonical": "FECHA EMISION"},
|
||||
{"canonical": "CLAVE PROVEEDOR"},
|
||||
{"canonical": "CLAVE VENDIDO A"},
|
||||
{"canonical": "CLAVE ENVIADO A"},
|
||||
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
|
||||
{"canonical": "ADUANA DE CRUCE"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
{"canonical": "CLAVE INCOTERM"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "TIPO DE CAMBIO"},
|
||||
{"canonical": "TIPO PESO"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "REMESA"},
|
||||
{"canonical": "AGENTE ADUANAL"},
|
||||
{"canonical": "FLETES"},
|
||||
{"canonical": "VALOR SEGUROS"},
|
||||
{"canonical": "SEGUROS"},
|
||||
{"canonical": "EMBALAJES"},
|
||||
{"canonical": "OTROS INCREMENTABLES"},
|
||||
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
|
||||
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
{"canonical": "FACTURA ALTERNA"},
|
||||
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
|
||||
{"canonical": "OBSERVACIONES E"},
|
||||
{"canonical": "OBSERVACIONES I"},
|
||||
{"canonical": "E DOCUMENT"},
|
||||
{"canonical": "NUM OPERACION"},
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "PRECINTO"},
|
||||
],
|
||||
# --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura ---
|
||||
"imp_def_header": None, # se resuelve igual que imp_temp_header
|
||||
# --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura ---
|
||||
"exp_def_header": None,
|
||||
# --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) ---
|
||||
"imp_temp_details": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
|
||||
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
|
||||
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
|
||||
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
|
||||
{"canonical": "CANTIDAD"},
|
||||
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
|
||||
{"canonical": "FRACCION"},
|
||||
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
],
|
||||
# --- Partidas: Impo Def y Expo - misma estructura ---
|
||||
"imp_def_details": None,
|
||||
"exp_def_details": None,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]:
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if cols is not None:
|
||||
return cols
|
||||
if template_id in ("imp_def_header", "exp_def_header"):
|
||||
return TEMPLATE_COLUMNS.get("imp_temp_header")
|
||||
if template_id in ("imp_def_details", "exp_def_details"):
|
||||
return TEMPLATE_COLUMNS.get("imp_temp_details")
|
||||
return None
|
||||
|
||||
|
||||
def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]:
|
||||
"""
|
||||
Construye un diccionario: normalized_header -> canonical_name.
|
||||
normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos).
|
||||
"""
|
||||
cols = _resolve_template_columns(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]:
|
||||
"""
|
||||
A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict
|
||||
solo con las columnas de la plantilla, usando nombres canónicos.
|
||||
Así la plantilla se respeta: solo entran columnas definidas en la plantilla.
|
||||
"""
|
||||
lookup = build_normalized_lookup(template_id, normalize_header_fn)
|
||||
if not lookup:
|
||||
# Sin template definido: comportamiento legacy (normalizar todo)
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
from api.v1.modules.public.reference_data.transport_types.models import TransportType
|
||||
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen
|
||||
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode
|
||||
|
||||
@@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel):
|
||||
operation_type: Optional[OperationType] = Field(
|
||||
..., description="Operation type: imp/exp/sm/ctm"
|
||||
)
|
||||
|
||||
@field_validator("operation_type", mode="before")
|
||||
@classmethod
|
||||
def normalize_operation_type(cls, v):
|
||||
"""Accept DB string (e.g. 'IMP') and coerce to enum value ('imp')."""
|
||||
if isinstance(v, str):
|
||||
return v.lower() if v else v
|
||||
return v
|
||||
invoice_type: Optional[str] = Field(
|
||||
None, max_length=5, description="Invoice type key"
|
||||
)
|
||||
@@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel):
|
||||
class InvoiceFinancialsBase(BaseModel):
|
||||
"""Base fields for Financials"""
|
||||
|
||||
currency: Currency = Field(None, max_length=7, description="Currency code")
|
||||
currency: Optional[Currency] = Field(None, max_length=7, description="Currency code")
|
||||
currency_type: Optional[str] = Field("USD", description="Currency type")
|
||||
exchange_rate: Decimal = Field(0.00, description="Exchange rate")
|
||||
exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate")
|
||||
exchange_rate_mm: Optional[Decimal] = Field(
|
||||
None, description="Exchange rate currency to currency"
|
||||
)
|
||||
|
||||
@@ -67,11 +67,13 @@ class InvoiceService:
|
||||
|
||||
# Apply filters if provided
|
||||
if filters:
|
||||
if filters.get("status"):
|
||||
if filters.get("status") is not None:
|
||||
query = query.filter(models.InvoiceHeader.is_updated == filters["status"])
|
||||
if filters.get("operation_type"):
|
||||
ot = filters["operation_type"]
|
||||
ot_val = ot.value if hasattr(ot, "value") else ot
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.operation_type == filters["operation_type"]
|
||||
models.InvoiceHeader.operation_type == ot_val
|
||||
)
|
||||
if filters.get("invoice_type"):
|
||||
query = query.filter(
|
||||
@@ -89,10 +91,9 @@ class InvoiceService:
|
||||
f"%{filters['pedimento']}%"
|
||||
)
|
||||
)
|
||||
if (
|
||||
not filters.get("invoice_type")
|
||||
and filters.get("operation_type") == "exp"
|
||||
):
|
||||
ot_exp = filters.get("operation_type")
|
||||
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
|
||||
if not filters.get("invoice_type") and ot_exp_val == "exp":
|
||||
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
|
||||
|
||||
if filters.get("manifest_number"):
|
||||
|
||||
1
backend/api/v1/modules/a76/parts/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/parts/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV import for Parts (Números de parte)
|
||||
151
backend/api/v1/modules/a76/parts/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/parts/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para Números de Parte.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
PART_IMPORT_FILE_PREFIX,
|
||||
PART_IMPORT_META_PREFIX,
|
||||
PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "part_numbers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{PART_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Parts import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
22
backend/api/v1/modules/a76/parts/imports/schemas.py
Normal file
22
backend/api/v1/modules/a76/parts/imports/schemas.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
639
backend/api/v1/modules/a76/parts/imports/tasks.py
Normal file
639
backend/api/v1/modules/a76/parts/imports/tasks.py
Normal file
@@ -0,0 +1,639 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Números de Parte.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PART_IMPORT_FILE_PREFIX = "part_import_file:"
|
||||
PART_IMPORT_META_PREFIX = "part_import_meta:"
|
||||
PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:"
|
||||
PART_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PART_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"part_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PART_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PART_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_part(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_class_codes: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
valid_currency_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
part_number = (row.get("NUMPARTE") or "").strip()
|
||||
if not part_number:
|
||||
return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"}
|
||||
if len(part_number) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
commercial = (row.get("NUMPARTECOM") or "").strip()
|
||||
if commercial and len(commercial) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTECOM", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
part_class = (row.get("CLASE") or "").strip()
|
||||
if part_class and len(part_class) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
# Si CLASE no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
# Si UNIMED no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
currency_key = (row.get("MONEDA") or "").strip()
|
||||
if currency_key and len(currency_key) > 3:
|
||||
return {"line": line_num, "col": "MONEDA", "msg": "Máximo 3 caracteres"}
|
||||
# Si MONEDA no existe en catálogo se guardará null (no se rechaza la fila)
|
||||
|
||||
unit_cost = row.get("COSTOUNIT")
|
||||
if unit_cost is not None and unit_cost != "":
|
||||
try:
|
||||
Decimal(str(unit_cost))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "COSTOUNIT", "msg": "Debe ser número"}
|
||||
|
||||
unit_weight = row.get("PESOUNIT")
|
||||
if unit_weight is not None and unit_weight != "":
|
||||
try:
|
||||
Decimal(str(unit_weight))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "PESOUNIT", "msg": "Debe ser número"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 10:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 10 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
fda_key = (row.get("FDAKEY") or "").strip()
|
||||
if fda_key and len(fda_key) > 20:
|
||||
return {"line": line_num, "col": "FDAKEY", "msg": "Máximo 20 caracteres"}
|
||||
fcc_key = (row.get("FCCKEY") or "").strip()
|
||||
if fcc_key and len(fcc_key) > 30:
|
||||
return {"line": line_num, "col": "FCCKEY", "msg": "Máximo 30 caracteres"}
|
||||
license_code = (row.get("LICENCIA") or "").strip()
|
||||
if license_code and len(license_code) > 3:
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 3 caracteres"}
|
||||
eccn = (row.get("ECCN") or "").strip()
|
||||
if eccn and len(eccn) > 20:
|
||||
return {"line": line_num, "col": "ECCN", "msg": "Máximo 20 caracteres"}
|
||||
export_code = (row.get("EXPORTCODE") or "").strip()
|
||||
if export_code and len(export_code) > 2:
|
||||
return {"line": line_num, "col": "EXPORTCODE", "msg": "Máximo 2 caracteres"}
|
||||
exclusion = (row.get("EXCLUSION") or "").strip()
|
||||
if exclusion and len(exclusion) > 19:
|
||||
return {"line": line_num, "col": "EXCLUSION", "msg": "Máximo 19 caracteres"}
|
||||
weight_type = (row.get("TIPOPESO") or "").strip()
|
||||
if weight_type and len(weight_type) > 6:
|
||||
return {"line": line_num, "col": "TIPOPESO", "msg": "Máximo 6 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Parts import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"part_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _bool_from_row(val: Any) -> bool:
|
||||
if val is None or val == "":
|
||||
return True
|
||||
s = str(val).strip().upper()
|
||||
if s in ("0", "F", "FALSE", "NO", "N"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Parts import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"part_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"part_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PART_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Parts import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
|
||||
valid_class_codes: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_currency_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
for c in (
|
||||
session.query(Class.class_code)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_class_codes.add(c[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
for cur in session.query(CurrencyType.code).all():
|
||||
valid_currency_codes.add(cur[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_part_number: Dict[str, Part] = {}
|
||||
for p in (
|
||||
session.query(Part)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_part_number[p.part_number] = p
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_part(
|
||||
row_norm,
|
||||
i,
|
||||
valid_class_codes=valid_class_codes,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
part_number = _str_or_none(row_norm.get("NUMPARTE"), 70)
|
||||
if not part_number:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
commercial = _str_or_none(row_norm.get("NUMPARTECOM"), 70)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
part_class = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if part_class and part_class not in valid_class_codes:
|
||||
part_class = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
currency_key = _str_or_none(row_norm.get("MONEDA"), 3)
|
||||
if currency_key and currency_key not in valid_currency_codes:
|
||||
currency_key = None
|
||||
|
||||
unit_cost = _decimal_or_none(row_norm.get("COSTOUNIT"))
|
||||
currency_type = _str_or_none(row_norm.get("MONEDA"), 2) if currency_key else None
|
||||
unit_weight = _decimal_or_none(row_norm.get("PESOUNIT"))
|
||||
weight_type = _str_or_none(row_norm.get("TIPOPESO"), 6)
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 10)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
fda_key = _str_or_none(row_norm.get("FDAKEY"), 20)
|
||||
fcc_key = _str_or_none(row_norm.get("FCCKEY"), 30)
|
||||
license_code = _str_or_none(row_norm.get("LICENCIA"), 3)
|
||||
eccn = _str_or_none(row_norm.get("ECCN"), 20)
|
||||
export_code = _str_or_none(row_norm.get("EXPORTCODE"), 2)
|
||||
exclusion_symbol = _str_or_none(row_norm.get("EXCLUSION"), 19)
|
||||
is_active = _bool_from_row(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_part_number.get(part_number)
|
||||
if existing:
|
||||
existing.commercial_part_number = commercial
|
||||
existing.description_spanish = desc_es
|
||||
existing.description_english = desc_en
|
||||
existing.part_class = part_class
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.unit_cost = unit_cost
|
||||
existing.currency_type = currency_type
|
||||
existing.currency_key = currency_key
|
||||
existing.unit_weight = unit_weight
|
||||
existing.weight_type = weight_type
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.fda_key = fda_key
|
||||
existing.fcc_key = fcc_key
|
||||
existing.license_code = license_code
|
||||
existing.eccn = eccn
|
||||
existing.export_code = export_code
|
||||
existing.exclusion_symbol = exclusion_symbol
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_part = Part(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
client_id=company_id,
|
||||
part_number=part_number,
|
||||
commercial_part_number=commercial,
|
||||
description_spanish=desc_es,
|
||||
description_english=desc_en,
|
||||
part_class=part_class,
|
||||
unit_of_measure=unit_of_measure,
|
||||
unit_cost=unit_cost,
|
||||
currency_type=currency_type,
|
||||
currency_key=currency_key,
|
||||
unit_weight=unit_weight,
|
||||
weight_type=weight_type,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
fda_key=fda_key,
|
||||
fcc_key=fcc_key,
|
||||
license_code=license_code,
|
||||
eccn=eccn,
|
||||
export_code=export_code,
|
||||
exclusion_symbol=exclusion_symbol,
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_part)
|
||||
existing_by_part_number[part_number] = new_part
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Parts import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Parts import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
54
backend/api/v1/modules/a76/parts/imports/template_config.py
Normal file
54
backend/api/v1/modules/a76/parts/imports/template_config.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Números de Parte (EstructuraCatPartesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"part_numbers": [
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE", "PART NUMBER", "NUM PARTE"]},
|
||||
{"canonical": "NUMPARTECOM", "aliases": ["NUMERO PARTE COMERCIAL", "COMMERCIAL PART", "PARTE COMERCIAL"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES", "DESC ESPANOL"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION", "DESC INGLES"]},
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CLASE MATERIAL", "PART CLASS"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM", "UNIT OF MEASURE"]},
|
||||
{"canonical": "COSTOUNIT", "aliases": ["COSTO UNITARIO", "UNIT COST", "COSTO"]},
|
||||
{"canonical": "MONEDA", "aliases": ["CURRENCY", "MONEDA CLAVE", "CURRENCY KEY"]},
|
||||
{"canonical": "PESOUNIT", "aliases": ["PESO UNITARIO", "UNIT WEIGHT", "PESO"]},
|
||||
{"canonical": "TIPOPESO", "aliases": ["WEIGHT TYPE", "TIPO PESO"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "FDAKEY", "aliases": ["FDA", "FDA KEY"]},
|
||||
{"canonical": "FCCKEY", "aliases": ["FCC", "FCC KEY"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["LICENSE CODE", "LICENSE"]},
|
||||
{"canonical": "ECCN", "aliases": ["ECCN CODE"]},
|
||||
{"canonical": "EXPORTCODE", "aliases": ["EXPORT CODE", "CODIGO EXPORT"]},
|
||||
{"canonical": "EXCLUSION", "aliases": ["EXCLUSION SYMBOL", "SIMBOLO EXCLUSION"]},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS ACTIVE", "ACTIVE", "ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("part_numbers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -2,21 +2,31 @@
|
||||
Endpoints API para gestión de partes (SCAII)
|
||||
"""
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import PartCreateDTO, PartResponseDTO, PartUpdateDTO
|
||||
from .service import PartService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
router = TenantCRUDRoutes(
|
||||
service=PartService,
|
||||
create_schema=PartCreateDTO,
|
||||
update_schema=PartUpdateDTO,
|
||||
response_schema=PartResponseDTO,
|
||||
prefix="/parts",
|
||||
tags=["a76 / parts"],
|
||||
resource_name="Part",
|
||||
id_name="part_id",
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
).router
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/parts/imports", tags=["a76 / parts / csv_import"])
|
||||
|
||||
# CRUD
|
||||
router.include_router(
|
||||
TenantCRUDRoutes(
|
||||
service=PartService,
|
||||
create_schema=PartCreateDTO,
|
||||
update_schema=PartUpdateDTO,
|
||||
response_schema=PartResponseDTO,
|
||||
prefix="/parts",
|
||||
tags=["a76 / parts"],
|
||||
resource_name="Part",
|
||||
id_name="part_id",
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
).router
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Pedimentos (upload → scan → commit)
|
||||
160
backend/api/v1/modules/a76/pedmientos/imports/routes.py
Normal file
160
backend/api/v1/modules/a76/pedmientos/imports/routes.py
Normal file
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Pedimentos.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
PED_IMPORT_FILE_PREFIX,
|
||||
PED_IMPORT_META_PREFIX,
|
||||
PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "pedimentos",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
25
backend/api/v1/modules/a76/pedmientos/imports/schemas.py
Normal file
25
backend/api/v1/modules/a76/pedmientos/imports/schemas.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: Optional[str] = None
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
563
backend/api/v1/modules/a76/pedmientos/imports/tasks.py
Normal file
563
backend/api/v1/modules/a76/pedmientos/imports/tasks.py
Normal file
@@ -0,0 +1,563 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Pedimentos.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con otros imports)
|
||||
PED_IMPORT_FILE_PREFIX = "ped_import_file:"
|
||||
PED_IMPORT_META_PREFIX = "ped_import_meta:"
|
||||
PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:"
|
||||
PED_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"ped_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_pedimento(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_client_ids: Set[int],
|
||||
valid_regimes: Set[str],
|
||||
valid_pedimento_codes: Set[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Pedimento. Retorna error dict o None."""
|
||||
# Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN
|
||||
year = (row.get("AÑO") or "").strip()
|
||||
if not year:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Requerido"}
|
||||
if len(year) > 2:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"}
|
||||
|
||||
customs_office = (row.get("ADUANA") or "").strip()
|
||||
if not customs_office:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Requerido"}
|
||||
if len(customs_office) > 3:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"}
|
||||
|
||||
license_val = (row.get("PATENTE") or "").strip()
|
||||
if not license_val:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Requerido"}
|
||||
if len(license_val) > 4:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()
|
||||
if not pedimento_number:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Requerido"}
|
||||
if len(pedimento_number) > 7:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
if not client_id_str:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"}
|
||||
try:
|
||||
client_id = int(client_id_str)
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"}
|
||||
if client_id not in valid_client_ids:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"}
|
||||
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()
|
||||
if not pedimento_code:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"}
|
||||
if len(pedimento_code) > 2:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"}
|
||||
if pedimento_code not in valid_pedimento_codes:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"}
|
||||
|
||||
regime = (row.get("REGIMEN") or "").strip()
|
||||
if not regime:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"}
|
||||
if len(regime) > 3:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"}
|
||||
if regime not in valid_regimes:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"}
|
||||
|
||||
# Optional numeric/string fields - validate format if present
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status and len(status) > 30:
|
||||
return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
continue
|
||||
try:
|
||||
Decimal(val.replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return {"line": line_num, "col": col, "msg": "Valor numérico inválido"}
|
||||
|
||||
operation_type = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if operation_type and operation_type not in ("imp", "exp", ""):
|
||||
return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"}
|
||||
|
||||
pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""):
|
||||
return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).strip().replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict for PedimentosCreate from normalized CSV row (canonical names)."""
|
||||
year = (row.get("AÑO") or "").strip()[:2]
|
||||
customs_office = (row.get("ADUANA") or "").strip()[:3]
|
||||
license_val = (row.get("PATENTE") or "").strip()[:4]
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()[:7]
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
client_id = int(client_id_str) if client_id_str else None
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2]
|
||||
regime = (row.get("REGIMEN") or "").strip()[:3]
|
||||
|
||||
data = {
|
||||
"year": year,
|
||||
"customs_office": customs_office,
|
||||
"license": license_val,
|
||||
"pedimento_number": pedimento_number,
|
||||
"client_id": client_id,
|
||||
"pedimento_code": pedimento_code,
|
||||
"regime": regime,
|
||||
}
|
||||
|
||||
op = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if op in ("imp", "exp"):
|
||||
data["operation_type"] = op
|
||||
|
||||
ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if ptype in ("normal", "consolidated", "complementary", "automobile"):
|
||||
data["pedimento_type"] = ptype
|
||||
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status:
|
||||
data["status"] = status[:30]
|
||||
|
||||
data["usd_value"] = _parse_decimal(row.get("VALOR_USD"))
|
||||
data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO"))
|
||||
data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO"))
|
||||
data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO"))
|
||||
|
||||
obs = (row.get("OBSERVACIONES") or "").strip()
|
||||
if obs:
|
||||
data["observations"] = obs
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Load valid FK sets for validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Reload FK sets for commit-time validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate
|
||||
from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
data = _row_to_pedimentos_create(row_norm)
|
||||
# Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL
|
||||
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
||||
data["pedimento_dates"] = PedimentoDatesCreate(
|
||||
entry_date=datetime.now(),
|
||||
end_date=datetime.now(),
|
||||
)
|
||||
create_data = PedimentosCreate(**data)
|
||||
PedimentosService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except ValueError as ve:
|
||||
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
else:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import line {i}: {e}")
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(e)})
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"pedimentos": [
|
||||
{"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]},
|
||||
{"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]},
|
||||
{"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]},
|
||||
{"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]},
|
||||
{"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]},
|
||||
{"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]},
|
||||
{"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]},
|
||||
{"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]},
|
||||
{"canonical": "REGIMEN", "aliases": ["REGIME"]},
|
||||
{"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]},
|
||||
{"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]},
|
||||
{"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]},
|
||||
{"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]},
|
||||
{"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]},
|
||||
{"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla pedimentos."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import (
|
||||
from .routes.pedimento_transport_means import router as pedimento_transport_means_router
|
||||
from .routes.pedimento_validation import router as pedimento_validation_router
|
||||
from .routes.pedimentos import router as pedimentos_router
|
||||
from .imports.routes import router as pedimentos_imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -117,3 +118,8 @@ router.include_router(
|
||||
router.include_router(
|
||||
pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"]
|
||||
)
|
||||
router.include_router(
|
||||
pedimentos_imports_router,
|
||||
prefix="/pedimentos/imports",
|
||||
tags=["a76 / pedimentos / csv_import"],
|
||||
)
|
||||
|
||||
@@ -12,9 +12,10 @@ from .general_catalogs.router import router as general_catalogs_router
|
||||
from .invoices.routes import router as invoices_router
|
||||
from .items.routes import router as items_router
|
||||
from .classes import router as classes_router
|
||||
from .classes import router as classes_router
|
||||
|
||||
from .clients_and_providers import router as client_and_provider_router
|
||||
from .imports.routes import router as imports_router
|
||||
from .csv_templates.routes import router as csv_templates_router
|
||||
from .invoice_settings.routes import router as invoice_settings_router
|
||||
from .item_presets.routes import router as item_presets_router
|
||||
from .general_catalogs.company import router as company_router
|
||||
@@ -23,6 +24,7 @@ from .transportation.drivers.routes import router as drivers_router
|
||||
from .doc_types_dig.routes import router as doc_types_dig_router
|
||||
from .fraction_rule_octave.routes import router as fraction_rule_octave_router
|
||||
from .parts import router as parts_router
|
||||
from .boms import router as boms_router
|
||||
from .pedmientos.router import router as pedimentos_router
|
||||
from .permission_rule_oct.routes import router as permission_rule_oct_router
|
||||
from .transportation.trailers.routes import router as trailers_router
|
||||
@@ -53,12 +55,14 @@ router.include_router(general_catalogs_router, prefix="/a76", tags=["a76 / gener
|
||||
router.include_router(invoices_router, prefix="/a76", tags=["a76 / invoices"])
|
||||
router.include_router(items_router, prefix="/a76", tags=["a76 / items"])
|
||||
router.include_router(imports_router, prefix="/a76/imports", tags=["a76 / imports"])
|
||||
router.include_router(csv_templates_router, prefix="/a76/csv-templates", tags=["a76 / csv_templates"])
|
||||
router.include_router(invoice_settings_router)
|
||||
router.include_router(item_presets_router, prefix="/a76/item-presets", tags=["a76 / item_presets"])
|
||||
router.include_router(pedimentos_router, prefix="/a76")
|
||||
router.include_router(client_and_provider_router, prefix="/a76", tags=["a76 / clients_and_providers"])
|
||||
router.include_router(classes_router, prefix="/a76/classes", tags=["a76 / classes"])
|
||||
router.include_router(parts_router, prefix="/a76", tags=["a76 / parts"])
|
||||
router.include_router(boms_router, prefix="/a76/boms", tags=["a76 / boms"])
|
||||
router.include_router(permission_rule_oct_router, prefix="/a76", tags=["a76 / permission_rule_oct"])
|
||||
router.include_router(fraction_rule_octave_router, prefix="/a76", tags=["a76 / fraction_rule_octave"])
|
||||
router.include_router(country_rule_oct_router, prefix="/a76", tags=["a76 / country_rule_oct"])
|
||||
|
||||
@@ -6,28 +6,28 @@ from pydantic import BaseModel
|
||||
class DriverBaseDTO(BaseModel):
|
||||
transporter_key: str
|
||||
line: int
|
||||
driver_name: Optional[str]
|
||||
license_number: Optional[str]
|
||||
express_line_id: Optional[str]
|
||||
ace_id: Optional[str]
|
||||
birth_date: Optional[int]
|
||||
gender: Optional[str]
|
||||
birth_country: Optional[str]
|
||||
hazardous_material_auth: Optional[str]
|
||||
hazardous_material_state: Optional[str]
|
||||
first_name: Optional[str]
|
||||
last_name: Optional[str]
|
||||
id_key1: Optional[str]
|
||||
id_number1: Optional[str]
|
||||
id_state1: Optional[str]
|
||||
id_country1: Optional[str]
|
||||
id_key2: Optional[str]
|
||||
id_number2: Optional[str]
|
||||
id_state2: Optional[str]
|
||||
id_country2: Optional[str]
|
||||
badge_number: Optional[str]
|
||||
class_type: Optional[str]
|
||||
unique_badge_number: Optional[str]
|
||||
driver_name: Optional[str] = None
|
||||
license_number: Optional[str] = None
|
||||
express_line_id: Optional[str] = None
|
||||
ace_id: Optional[str] = None
|
||||
birth_date: Optional[int] = None
|
||||
gender: Optional[str] = None
|
||||
birth_country: Optional[str] = None
|
||||
hazardous_material_auth: Optional[str] = None
|
||||
hazardous_material_state: Optional[str] = None
|
||||
first_name: Optional[str] = None
|
||||
last_name: Optional[str] = None
|
||||
id_key1: Optional[str] = None
|
||||
id_number1: Optional[str] = None
|
||||
id_state1: Optional[str] = None
|
||||
id_country1: Optional[str] = None
|
||||
id_key2: Optional[str] = None
|
||||
id_number2: Optional[str] = None
|
||||
id_state2: Optional[str] = None
|
||||
id_country2: Optional[str] = None
|
||||
badge_number: Optional[str] = None
|
||||
class_type: Optional[str] = None
|
||||
unique_badge_number: Optional[str] = None
|
||||
company_id: int
|
||||
tenant_id: int
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Drivers / Conductores (upload -> scan -> status -> commit).
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Rutas de importacion CSV para Conductores.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
DRV_IMPORT_FILE_PREFIX,
|
||||
DRV_IMPORT_META_PREFIX,
|
||||
DRV_IMPORT_STATUS_PREFIX,
|
||||
DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "drivers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Drivers import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not write processing status: {e}")
|
||||
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Drivers import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Insercion iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
updated: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,640 @@
|
||||
"""
|
||||
Tareas Celery para importacion CSV de Conductores.
|
||||
Flujo: scan_file (validacion) -> insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
|
||||
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
|
||||
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
|
||||
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
|
||||
DRV_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"drv_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
_MAX = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
"badge_number": 20,
|
||||
"class_type": 1,
|
||||
"unique_badge_number": 100,
|
||||
}
|
||||
|
||||
|
||||
def _parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
transporter_key = (row.get("TRANSPORTISTA") or "").strip()
|
||||
if not transporter_key:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"}
|
||||
if len(transporter_key) > _MAX["transporter_key"]:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"}
|
||||
|
||||
line_val = _parse_int(row.get("LINEA"))
|
||||
if line_val is None:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"}
|
||||
if line_val <= 0:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"}
|
||||
|
||||
driver_name = (row.get("CLAVE CONDUCTOR") or "").strip()
|
||||
if driver_name and len(driver_name) > _MAX["driver_name"]:
|
||||
return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("LICENCIA", _MAX["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", _MAX["express_line_id"]),
|
||||
("IDENTIFICACION ACE", _MAX["ace_id"]),
|
||||
("SEXO", _MAX["gender"]),
|
||||
("PAIS NACIMIENTO", _MAX["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]),
|
||||
("NOMBRE(S)", _MAX["first_name"]),
|
||||
("APELLIDO PATERNO", _MAX["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", _MAX["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", _MAX["id_number1"]),
|
||||
("ESTADO", _MAX["id_state1"]),
|
||||
("PAIS", _MAX["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", _MAX["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", _MAX["id_number2"]),
|
||||
("ESTADO 2", _MAX["id_state2"]),
|
||||
("PAIS 2", _MAX["id_country2"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
|
||||
fecha = row.get("FECHA NACIMIENTO")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_birth_date(fecha) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA NACIMIENTO",
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"])
|
||||
line = _parse_int(row.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
data = {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]),
|
||||
"license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]),
|
||||
"express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]),
|
||||
"ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]),
|
||||
"birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row.get("SEXO"), _MAX["gender"]),
|
||||
"birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]),
|
||||
"last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]),
|
||||
"id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]),
|
||||
"id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]),
|
||||
"id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]),
|
||||
"id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]),
|
||||
"id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]),
|
||||
"id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]),
|
||||
"id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]),
|
||||
"id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Drivers import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.drivers.services import DriverService
|
||||
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_driver_dto(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("transporter_key") or data.get("line") is None:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
key = f"{data['transporter_key']}:{data['line']}"
|
||||
if key in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[key] = i
|
||||
|
||||
existing = DriverService.get_driver_by_key_and_line(
|
||||
session, data["transporter_key"], data["line"], str(company_id), tenant_id
|
||||
)
|
||||
try:
|
||||
if existing:
|
||||
update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")}
|
||||
for field, value in update_fields.items():
|
||||
setattr(existing, field, value)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = DriverCreateDTO(**data)
|
||||
DriverService.create_driver(session, create_data)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Drivers import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Drivers import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros validos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Drivers import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Configuracion de plantilla CSV para Conductores (EstructuraCatConductor.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"drivers": [
|
||||
{"canonical": "TRANSPORTISTA", "aliases": ["TRANSPORTISTA CLAVE", "CLAVE TRANSPORTISTA", "TRANSPORTER"]},
|
||||
{"canonical": "LINEA", "aliases": ["LINE", "LINEA CONDUCTOR"]},
|
||||
{"canonical": "CLAVE CONDUCTOR", "aliases": ["CONDUCTOR", "DRIVER", "NOMBRE CONDUCTOR"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["LICENSE", "LICENCE"]},
|
||||
{"canonical": "PERMISO LINEA EXPRESS", "aliases": ["LINEA EXPRESS", "LINEA EXPRESS ID", "EXPRESS LINE"]},
|
||||
{"canonical": "IDENTIFICACION ACE", "aliases": ["ACE", "ACE ID", "IDENTIFICACION ACE ID"]},
|
||||
{"canonical": "FECHA NACIMIENTO", "aliases": ["FECHA NAC", "BIRTH DATE", "BIRTHDATE"]},
|
||||
{"canonical": "SEXO", "aliases": ["GENERO", "GENDER"]},
|
||||
{"canonical": "PAIS NACIMIENTO", "aliases": ["PAIS Nacimiento", "BIRTH COUNTRY"]},
|
||||
{"canonical": "TRANSPORTA MAT. PELIGROSO?", "aliases": ["MATERIAL PELIGROSO", "HAZMAT", "HAZARDOUS MATERIAL"]},
|
||||
{"canonical": "PERMISO MAT. PELIGROSO", "aliases": ["PERMISO MAT PELIGROSO", "HAZMAT PERMIT"]},
|
||||
{"canonical": "NOMBRE(S)", "aliases": ["NOMBRE", "NOMBRES", "FIRST NAME"]},
|
||||
{"canonical": "APELLIDO PATERNO", "aliases": ["APELLIDO", "LAST NAME", "APELLIDO P"]},
|
||||
{"canonical": "FORMA IDENTIFICACION 1", "aliases": ["FORMA IDENTIFICACION", "ID TIPO 1", "ID KEY 1"]},
|
||||
{"canonical": "NUM. IDENTIFICACION 1", "aliases": ["NUM IDENTIFICACION 1", "ID NUMERO 1", "ID NUMBER 1"]},
|
||||
{"canonical": "ESTADO", "aliases": ["ESTADO 1", "STATE 1"]},
|
||||
{"canonical": "PAIS", "aliases": ["PAIS 1", "COUNTRY 1"]},
|
||||
{"canonical": "FORMA IDENTIFICACION 2", "aliases": ["ID TIPO 2", "ID KEY 2"]},
|
||||
{"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]},
|
||||
{"canonical": "ESTADO 2", "aliases": ["STATE 2"]},
|
||||
{"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("drivers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -8,9 +8,13 @@ from sqlalchemy.orm import Session
|
||||
from .dto import DriverCreateDTO, DriverResponseDTO
|
||||
from .models import Driver
|
||||
from .services import DriverService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter(prefix="/drivers")
|
||||
|
||||
# CSV import (upload -> scan -> status -> commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / drivers / csv_import"])
|
||||
|
||||
|
||||
@router.get("/", response_model=Dict[str, Any])
|
||||
async def list_drivers(
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Trailers CSV import: upload -> scan -> status -> commit
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Rutas de importación CSV para Trailers y Cajas.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
TRL_IMPORT_FILE_PREFIX,
|
||||
TRL_IMPORT_META_PREFIX,
|
||||
TRL_IMPORT_STATUS_PREFIX,
|
||||
TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "trailers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Trailers import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not write processing status: {e}")
|
||||
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Trailers import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
updated: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,535 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Trailers y Cajas.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Upsert por trailer_number usando TrailerService.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRL_IMPORT_FILE_PREFIX = "trl_import_file:"
|
||||
TRL_IMPORT_META_PREFIX = "trl_import_meta:"
|
||||
TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:"
|
||||
TRL_IMPORT_STATUS_PREFIX = "trl_import_status:"
|
||||
TRL_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"trl_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Trailer model (a76.trailer)
|
||||
_MAX = {
|
||||
"trailer_number": 20,
|
||||
"ace_trailer_number": 10,
|
||||
"trailer_type_key": 2,
|
||||
"seal": 15,
|
||||
"entity_code": 1,
|
||||
"plate_number": 17,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"container_key": 3,
|
||||
}
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Trailer. Retorna error dict o None."""
|
||||
trailer_number = (row.get("NUMERO TRAILER") or "").strip()
|
||||
if not trailer_number:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"}
|
||||
if len(trailer_number) > _MAX["trailer_number"]:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_trailer_number"]),
|
||||
("TIPO TRAILER", _MAX["trailer_type_key"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("CODIGO ENTIDAD", _MAX["entity_code"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("CLAVE CONTENEDOR", _MAX["container_key"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
"""Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row."""
|
||||
trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"])
|
||||
if not trailer_number:
|
||||
return {}
|
||||
return {
|
||||
"trailer_number": trailer_number,
|
||||
"ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]),
|
||||
"trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]),
|
||||
}
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Trailers import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.trailers.services import TrailerService
|
||||
from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_trailer_dto(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("trailer_number"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
tn = data["trailer_number"]
|
||||
if tn in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": tn,
|
||||
"invoice": tn,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[tn] = i
|
||||
|
||||
existing = TrailerService.get_by_id(session, tn, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"})
|
||||
TrailerService.update(session, tn, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = TrailerCreateDTO(**data)
|
||||
TrailerService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Trailers import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Trailers import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Trailers import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Trailers / Cajas (EstructuraCatTrailers.xls).
|
||||
Mapeo: NUMERO TRAILER → trailer_number, CLAVE ACE → ace_trailer_number, etc.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"trailers": [
|
||||
{"canonical": "NUMERO TRAILER", "aliases": ["CLAVE TRAILER", "TRAILER NUMBER", "TRAILER", "NUMERO"]},
|
||||
{"canonical": "CLAVE ACE", "aliases": ["ACE", "NUMERO ACE", "ACE TRAILER"]},
|
||||
{"canonical": "TIPO TRAILER", "aliases": ["TIPO", "TRAILER TYPE", "TIPO CAJA"]},
|
||||
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
|
||||
{"canonical": "CODIGO ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE", "CODIGO DE ENTIDAD"]},
|
||||
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE NUMBER", "PLATE"]},
|
||||
{"canonical": "ESTADO", "aliases": ["STATE"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
|
||||
{"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("trailers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -1,11 +1,19 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO
|
||||
from .services import TrailerService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create router using TenantCRUDRoutes factory
|
||||
# Note: trailer_number is a string (not int) and is used as the primary key
|
||||
router = TenantCRUDRoutes(
|
||||
# Main router: trailers CRUD + CSV imports
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/trailers/imports", tags=["a76 / trailers / csv_import"])
|
||||
|
||||
# CRUD routes
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=TrailerService,
|
||||
create_schema=TrailerCreateDTO,
|
||||
update_schema=TrailerUpdateDTO,
|
||||
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
|
||||
prefix="/trailers",
|
||||
tags=[],
|
||||
resource_name="Trailer",
|
||||
id_name="trailer_number", # Using trailer_number instead of numeric ID
|
||||
id_type=str, # Specify that the ID is a string
|
||||
enable_list=True, # Enable GET /trailers with pagination
|
||||
enable_filters=True, # Enable filtering by plate_number and trailer_type_key
|
||||
id_name="trailer_number",
|
||||
id_type=str,
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
default_page_size=50,
|
||||
max_page_size=100,
|
||||
).router
|
||||
router.include_router(crud_router)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Vehicles / Transportes (upload → scan → status → commit).
|
||||
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
Rutas de importación CSV para Vehículos (Transportes).
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
VEHL_IMPORT_FILE_PREFIX,
|
||||
VEHL_IMPORT_META_PREFIX,
|
||||
VEHL_IMPORT_STATUS_PREFIX,
|
||||
VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "vehicles",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
# Fallback sin worker: ejecutar scan en un hilo y guardar resultado en Redis
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Vehicles import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
# Si el resultado del scan está en Redis (worker o hilo de fallback), usarlo
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
# Escribir "processing" en Redis para que el polling muestre estado hasta que termine el commit
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not write processing status: {e}")
|
||||
|
||||
# Ejecutar commit en hilo (mismo fallback que el scan: no depende del worker de Celery)
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Vehicles import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
# Mismo job_id para que el front siga haciendo polling y reciba el resultado de Redis
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,607 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Vehículos (Transportes).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VEHL_IMPORT_FILE_PREFIX = "veh_import_file:"
|
||||
VEHL_IMPORT_META_PREFIX = "veh_import_meta:"
|
||||
VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:"
|
||||
VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:"
|
||||
VEHL_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"veh_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Vehicle model (a76.vehicle)
|
||||
_MAX = {
|
||||
"vehicle_key": 14,
|
||||
"ace_vehicle_key": 10,
|
||||
"transporter_key": 23,
|
||||
"transport_identifier": 30,
|
||||
"transport_type": 2,
|
||||
"entity_code": 1,
|
||||
"transponder_number": 16,
|
||||
"dot_number": 8,
|
||||
"plate_number": 17,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"seal": 49,
|
||||
"insurance_company_name": 30,
|
||||
"insurance_number": 20,
|
||||
"series": 30,
|
||||
}
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
s = str(val).strip().replace(",", "")
|
||||
return Decimal(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_insurance_date(val: Any) -> Optional[int]:
|
||||
"""Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD."""
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
# Already integer-like
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
# Try DD/MM/YYYY or similar
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Vehículo. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > _MAX["vehicle_key"]:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"}
|
||||
|
||||
# Optional fields: max lengths only
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_vehicle_key"]),
|
||||
("CLAVE TRANSPORTE", _MAX["transporter_key"]),
|
||||
("VIN", _MAX["series"]),
|
||||
("TIPO TRANSPORTE", _MAX["transport_type"]),
|
||||
("CODIGO DE ENTIDAD", _MAX["entity_code"]),
|
||||
("TRANSPONDEDOR", _MAX["transponder_number"]),
|
||||
("NUMERO DOT", _MAX["dot_number"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("CIUDAD", _MAX["city"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]),
|
||||
("NUM. ASEGURADORA", _MAX["insurance_number"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
# MONTO ASEGURADO: must be numeric if present
|
||||
monto = row.get("MONTO ASEGURADO") or row.get("MONTO")
|
||||
if monto is not None and str(monto).strip():
|
||||
if _parse_decimal(monto) is None:
|
||||
return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"}
|
||||
|
||||
# FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None)
|
||||
# Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either
|
||||
# reject or set None. We reject invalid date to keep data quality.
|
||||
fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_insurance_date(fecha) is None:
|
||||
return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _do_scan(
|
||||
job_id: str,
|
||||
progress_callback: Optional[Any] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync.
|
||||
progress_callback(current, total, error_count) opcional.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row."""
|
||||
vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"])
|
||||
if not vehicle_key:
|
||||
return {}
|
||||
data = {
|
||||
"vehicle_key": vehicle_key,
|
||||
"ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]),
|
||||
"transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]),
|
||||
"series": _str_or_none(row.get("VIN"), _MAX["series"]),
|
||||
"transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]),
|
||||
"transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]),
|
||||
"dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"city": _str_or_none(row.get("CIUDAD"), _MAX["city"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]),
|
||||
"insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]),
|
||||
"insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")),
|
||||
"insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.vehicles.services import VehicleService
|
||||
from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_vehicle_dto(row_norm)
|
||||
if not data or not data.get("vehicle_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
vk = data["vehicle_key"]
|
||||
if vk in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[vk] = i
|
||||
|
||||
existing = VehicleService.get_by_id(session, vk, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"})
|
||||
VehicleService.update(session, vk, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = VehicleCreateDTO(**data)
|
||||
VehicleService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Vehicles import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Vehicles import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el commit en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Vehículos / Transportes (EstructuraCatTransportes.xls).
|
||||
Mapeo: CLAVE → vehicle_key, CLAVE ACE → ace_vehicle_key, etc.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"vehicles": [
|
||||
{"canonical": "CLAVE", "aliases": ["CLAVE VEHICULO", "VEHICLE KEY", "KEY"]},
|
||||
{"canonical": "CLAVE ACE", "aliases": ["ACE", "CLAVEACE"]},
|
||||
{"canonical": "CLAVE TRANSPORTE", "aliases": ["TRANSPORTE", "TRANSPORTISTA"]},
|
||||
{"canonical": "VIN", "aliases": ["SERIE", "SERIES", "NUMERO SERIE"]},
|
||||
{"canonical": "TIPO TRANSPORTE", "aliases": ["TIPO", "TRANSPORT TYPE"]},
|
||||
{"canonical": "CODIGO DE ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE"]},
|
||||
{"canonical": "TRANSPONDEDOR", "aliases": ["TRANSPONDER", "TRANSPONDER NUMBER"]},
|
||||
{"canonical": "NUMERO DOT", "aliases": ["DOT", "NUM DOT"]},
|
||||
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE", "PLATE NUMBER"]},
|
||||
{"canonical": "CIUDAD", "aliases": ["CITY"]},
|
||||
{"canonical": "ESTADO", "aliases": ["STATE"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
|
||||
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
|
||||
{"canonical": "EMPRESA ASEGURADORA", "aliases": ["ASEGURADORA", "INSURANCE COMPANY"]},
|
||||
{"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]},
|
||||
{"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]},
|
||||
{"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("vehicles")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -1,11 +1,19 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO
|
||||
from .services import VehicleService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create router using TenantCRUDRoutes factory
|
||||
# Note: vehicle_key is a string (not int) and is used as the primary key
|
||||
router = TenantCRUDRoutes(
|
||||
# Main router: vehicles CRUD + CSV imports
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/vehicles/imports", tags=["a76 / vehicles / csv_import"])
|
||||
|
||||
# CRUD routes
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=VehicleService,
|
||||
create_schema=VehicleCreateDTO,
|
||||
update_schema=VehicleUpdateDTO,
|
||||
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
|
||||
prefix="/vehicles",
|
||||
tags=[],
|
||||
resource_name="Vehicle",
|
||||
id_name="vehicle_key", # Using vehicle_key instead of numeric ID
|
||||
id_type=str, # Specify that the ID is a string
|
||||
enable_list=True, # Enable GET /vehicles with pagination
|
||||
enable_filters=True, # Enable filtering by plate_number and transport_type
|
||||
id_name="vehicle_key",
|
||||
id_type=str,
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
default_page_size=50,
|
||||
max_page_size=100,
|
||||
).router
|
||||
router.include_router(crud_router)
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import os
|
||||
from celery import Celery
|
||||
|
||||
# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen (mapper)
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
|
||||
CodePedimentoRegimen,
|
||||
)
|
||||
|
||||
# Import models in correct order for SQLAlchemy relationship resolution
|
||||
# CRITICAL: FaLineItem must be imported BEFORE LineItem
|
||||
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem # noqa: F401
|
||||
@@ -26,6 +33,15 @@ celery_app.conf.update(
|
||||
"api.v1.modules.a76.reports.movements.invoices.tasks",
|
||||
"api.v1.modules.a76.reports.exportacion.descargo.task",
|
||||
"api.v1.modules.a76.imports.tasks",
|
||||
"api.v1.modules.a76.customs_brokers.imports.tasks",
|
||||
"api.v1.modules.a76.clients_and_providers.imports.tasks",
|
||||
"api.v1.modules.a76.pedmientos.imports.tasks",
|
||||
"api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks",
|
||||
"api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks",
|
||||
"api.v1.modules.a76.classes.imports.tasks",
|
||||
"api.v1.modules.a76.parts.imports.tasks",
|
||||
"api.v1.modules.a76.boms.imports.tasks",
|
||||
"api.v1.modules.a76.transportation.vehicles.imports.tasks",
|
||||
"api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task",
|
||||
"api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task",
|
||||
"api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task",
|
||||
|
||||
@@ -11,11 +11,23 @@ from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
|
||||
|
||||
from .config import settings
|
||||
from .exceptions import BaseAPIException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cors_headers(request: Request) -> Dict[str, str]:
|
||||
"""CORS headers for error responses so browser does not block on 4xx/5xx."""
|
||||
origin = request.headers.get("origin")
|
||||
if not origin or origin not in settings.cors_origins_list:
|
||||
return {}
|
||||
return {
|
||||
"Access-Control-Allow-Origin": origin,
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
}
|
||||
|
||||
|
||||
async def base_exception_handler(
|
||||
request: Request,
|
||||
exc: BaseAPIException,
|
||||
@@ -36,10 +48,13 @@ async def base_exception_handler(
|
||||
if hasattr(exc, "errors") and exc.errors:
|
||||
logger.warning(f"Validation errors details: {exc.errors}")
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content=jsonable_encoder(exc.to_dict()),
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def validation_exception_handler(
|
||||
@@ -65,7 +80,7 @@ async def validation_exception_handler(
|
||||
extra={"errors": errors},
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
content={
|
||||
"error": "VALIDATION_ERROR",
|
||||
@@ -74,6 +89,9 @@ async def validation_exception_handler(
|
||||
"errors": errors,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def integrity_error_handler(
|
||||
@@ -105,7 +123,7 @@ async def integrity_error_handler(
|
||||
else:
|
||||
error_message = "Error de integridad en la base de datos"
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
content={
|
||||
"error": "DATABASE_INTEGRITY_ERROR",
|
||||
@@ -113,6 +131,9 @@ async def integrity_error_handler(
|
||||
"status_code": status.HTTP_409_CONFLICT,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def sqlalchemy_error_handler(
|
||||
@@ -131,7 +152,7 @@ async def sqlalchemy_error_handler(
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
content={
|
||||
"error": "DATABASE_ERROR",
|
||||
@@ -139,6 +160,9 @@ async def sqlalchemy_error_handler(
|
||||
"status_code": status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def http_exception_handler(
|
||||
@@ -174,7 +198,7 @@ async def general_exception_handler(
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
content={
|
||||
"error": "INTERNAL_SERVER_ERROR",
|
||||
@@ -182,6 +206,9 @@ async def general_exception_handler(
|
||||
"status_code": status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
def register_exception_handlers(app) -> None:
|
||||
|
||||
@@ -17,8 +17,13 @@ from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
|
||||
# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen para que
|
||||
# SQLAlchemy resuelva los nombres en relationship() al configurar el mapper
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
|
||||
CodePedimentoRegimen,
|
||||
)
|
||||
from api.v1.modules.public.reference_data.sectors.models import Sector
|
||||
from api.v1.modules.public.reference_data.states.models import State
|
||||
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode
|
||||
@@ -134,6 +139,20 @@ logger = logging.getLogger(__name__)
|
||||
register_exception_handlers(app)
|
||||
|
||||
|
||||
def _cors_headers_for_request(request: Request):
|
||||
"""Return CORS headers if request Origin is allowed (so error responses don't get blocked by browser)."""
|
||||
origin = request.headers.get("origin")
|
||||
if not origin:
|
||||
return {}
|
||||
allowed = settings.cors_origins_list
|
||||
if origin in allowed:
|
||||
return {
|
||||
"Access-Control-Allow-Origin": origin,
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
# Add validation error handler
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
@@ -141,12 +160,29 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
|
||||
f"Validation error for {request.method} {request.url.path}: {exc.errors()}"
|
||||
)
|
||||
logger.error(f"Request body: {await request.body()}")
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
content={"detail": exc.errors(), "body": exc.body},
|
||||
)
|
||||
for k, v in _cors_headers_for_request(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
# Add HTTP exception handler
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
logger.error(
|
||||
f"HTTP {exc.status_code} for {request.method} {request.url.path}: {exc.detail}"
|
||||
)
|
||||
response = JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={"detail": exc.detail},
|
||||
)
|
||||
for k, v in _cors_headers_for_request(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
def run_migrations():
|
||||
subprocess.run(["alembic", "upgrade", "head"], check=True)
|
||||
|
||||
|
||||
@@ -64,5 +64,6 @@
|
||||
"lucide-svelte": "^0.553.0",
|
||||
"marked": "^12.0.0",
|
||||
"svelte-sonner": "^1.0.7"
|
||||
}
|
||||
}
|
||||
},
|
||||
"packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017"
|
||||
}
|
||||
|
||||
@@ -315,6 +315,35 @@ export const api = {
|
||||
|
||||
delete: <T = any>(endpoint: string, options: RequestInit = {}) => fetchApi<T>(endpoint, { method: 'DELETE', ...options }),
|
||||
|
||||
/**
|
||||
* Download CSV template by template_id (generated from code, no static file).
|
||||
* Returns blob and suggested filename for the browser download.
|
||||
*/
|
||||
async getCsvTemplateDownload(
|
||||
templateId: string
|
||||
): Promise<{ blob: Blob; filename: string }> {
|
||||
const token = getToken();
|
||||
const headers: Record<string, string> = {};
|
||||
if (token) headers['Authorization'] = `Bearer ${token}`;
|
||||
const response = await fetch(`${API_BASE_URL}/v1/a76/csv-templates/${templateId}`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
credentials: 'include'
|
||||
});
|
||||
if (!response.ok) {
|
||||
const msg = response.status === 404 ? 'Plantilla no encontrada' : `Error ${response.status}`;
|
||||
throw new Error(msg);
|
||||
}
|
||||
const blob = await response.blob();
|
||||
let filename = `plantilla_${templateId}.csv`;
|
||||
const disposition = response.headers.get('Content-Disposition');
|
||||
if (disposition) {
|
||||
const match = /filename="?([^";\n]+)"?/.exec(disposition);
|
||||
if (match) filename = match[1].trim();
|
||||
}
|
||||
return { blob, filename };
|
||||
},
|
||||
|
||||
// Endpoints específicos
|
||||
auth: {
|
||||
login: (credentials: { username: string; password: string; tenant_slug: string }) =>
|
||||
@@ -341,6 +370,215 @@ export const api = {
|
||||
validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`)
|
||||
},
|
||||
|
||||
imports: {
|
||||
upload: (
|
||||
file: File,
|
||||
modelTarget: string,
|
||||
footerConfig: any,
|
||||
companyId: number,
|
||||
operationType: string,
|
||||
templateId?: string
|
||||
) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
if (footerConfig) {
|
||||
formData.append('footer_config', JSON.stringify(footerConfig));
|
||||
}
|
||||
if (templateId) {
|
||||
formData.append('template_id', templateId);
|
||||
}
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
company_id: String(companyId),
|
||||
operation_type: operationType || 'imp'
|
||||
}).toString();
|
||||
|
||||
return fetchApi(`/v1/a76/imports/upload/${modelTarget}?${queryParams}`, {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/imports/${jobId}/status`),
|
||||
commit: (jobId: string, modelTarget: string) =>
|
||||
api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget })
|
||||
},
|
||||
|
||||
// CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports)
|
||||
customsBrokerImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/customs-brokers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/customs-brokers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Clientes y Proveedores (flujo propio en clients_and_providers/imports)
|
||||
clientProviderImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/clients-providers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/clients-providers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports)
|
||||
exchangeRateImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/exchange-rate/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Fracción Americana (us_tariff_fractions/imports)
|
||||
americanFractionImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/us-tariff-fractions/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/us-tariff-fractions/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Pedimentos (pedimentos/imports)
|
||||
pedimentosImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/pedimentos/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/pedimentos/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Clases de Materiales (classes/imports)
|
||||
materialClassImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/classes/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/classes/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/classes/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Vehículos / Transportes (transportation/vehicles/imports)
|
||||
vehicleImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/transportation/vehicles/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/transportation/vehicles/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/transportation/vehicles/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Conductores (drivers/imports)
|
||||
driverImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/drivers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/drivers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) => api.post(`/v1/a76/drivers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Trailers y Cajas (transportation/trailers/imports)
|
||||
trailerImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/transportation/trailers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/transportation/trailers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Transportistas (transporters/imports)
|
||||
transporterImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/transporters/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/transporters/imports/${jobId}/status`),
|
||||
commit: (jobId: string) => api.post(`/v1/a76/transporters/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Números de parte (parts/imports)
|
||||
partNumberImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/parts/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/parts/imports/${jobId}/status`),
|
||||
commit: (jobId: string) => api.post(`/v1/a76/parts/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for BOMs (boms/imports)
|
||||
bomImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/boms/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/boms/imports/${jobId}/status`),
|
||||
commit: (jobId: string) => api.post(`/v1/a76/boms/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// Generic request for custom needs (like file uploads)
|
||||
request: <T = any>(endpoint: string, options: RequestInit = {}) => fetchApi<T>(endpoint, options)
|
||||
};
|
||||
|
||||
@@ -143,17 +143,53 @@
|
||||
|
||||
{#if scanResults.error_count > 0}
|
||||
<div
|
||||
class="rounded-md bg-destructive/5 border border-destructive/10 p-4 flex items-start gap-3"
|
||||
class="rounded-md bg-destructive/5 border border-destructive/10 p-4 flex items-start gap-3 mb-4"
|
||||
>
|
||||
<XCircle class="w-5 h-5 text-destructive mt-0.5 shrink-0" />
|
||||
<div class="text-sm text-destructive-foreground/90">
|
||||
<p class="font-semibold mb-1">Se detectaron problemas en el archivo</p>
|
||||
<p>
|
||||
Las filas con errores serán omitidas automáticamente. Solo se importarán los
|
||||
registros válidos.
|
||||
Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para
|
||||
importar solo las filas válidas (las erróneas se omitirán).
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{#if scanResults.errors && scanResults.errors.length > 0}
|
||||
<div class="border rounded-lg overflow-hidden shadow-sm">
|
||||
<div class="bg-muted/50 px-4 py-2 border-b flex justify-between items-center">
|
||||
<h5 class="text-xs font-bold text-foreground uppercase tracking-wide">
|
||||
Detalle de errores (para corregir en el CSV)
|
||||
</h5>
|
||||
<span
|
||||
class="text-[10px] bg-secondary text-secondary-foreground px-2 py-0.5 rounded-full border"
|
||||
>
|
||||
{scanResults.errors.length} error(es)
|
||||
</span>
|
||||
</div>
|
||||
<div class="max-h-60 overflow-y-auto bg-card relative">
|
||||
<table class="w-full text-xs text-left">
|
||||
<thead
|
||||
class="text-muted-foreground font-medium bg-muted/30 sticky top-0 z-10 shadow-sm backdrop-blur-sm"
|
||||
>
|
||||
<tr>
|
||||
<th class="px-4 py-2 w-16">Línea</th>
|
||||
<th class="px-4 py-2 w-40">Columna</th>
|
||||
<th class="px-4 py-2">Mensaje</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y">
|
||||
{#each scanResults.errors as err}
|
||||
<tr class="hover:bg-muted/30 transition-colors">
|
||||
<td class="px-4 py-2 font-mono text-muted-foreground">{err.line}</td>
|
||||
<td class="px-4 py-2 font-mono font-medium text-foreground">{err.col || '-'}</td>
|
||||
<td class="px-4 py-2 text-destructive">{err.msg || '-'}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
<div class="rounded-md bg-primary/5 border border-primary/10 p-4 flex items-start gap-3">
|
||||
<CheckCircle2 class="w-5 h-5 text-primary mt-0.5 shrink-0" />
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { UploadCloud, Lock } from 'lucide-svelte';
|
||||
import { cn } from '$lib/utils';
|
||||
import { toast } from 'svelte-sonner';
|
||||
import { api } from '$lib/api';
|
||||
|
||||
let {
|
||||
items,
|
||||
@@ -88,23 +89,31 @@
|
||||
}
|
||||
}
|
||||
|
||||
function handleContextMenu(e: MouseEvent, item: CsvUploadItem) {
|
||||
async function handleContextMenu(e: MouseEvent, item: CsvUploadItem) {
|
||||
if (item.disabled) {
|
||||
e.preventDefault();
|
||||
return;
|
||||
}
|
||||
if (!item.templateUrl) return;
|
||||
|
||||
e.preventDefault();
|
||||
|
||||
const link = document.createElement('a');
|
||||
link.href = item.templateUrl;
|
||||
link.download = item.templateUrl.split('/').pop() || 'plantilla.xls';
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
if (!item.templateId) return;
|
||||
|
||||
toast.info(`Descargando plantilla para ${item.title}...`);
|
||||
try {
|
||||
toast.info(`Descargando plantilla para ${item.title}...`);
|
||||
const { blob, filename } = await api.getCsvTemplateDownload(item.templateId);
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
URL.revokeObjectURL(url);
|
||||
toast.success(`Plantilla descargada: ${filename}`);
|
||||
} catch (err) {
|
||||
toast.error(err instanceof Error ? err.message : 'Error al descargar la plantilla');
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -123,7 +132,7 @@
|
||||
ondragover={(e) => handleDragOver(e, item.disabled)}
|
||||
ondrop={(e) => handleDrop(e, item)}
|
||||
oncontextmenu={(e) => handleContextMenu(e, item)}
|
||||
roles="button"
|
||||
role="button"
|
||||
tabindex={item.disabled ? -1 : 0}
|
||||
onclick={() => handleClick(item.id, item.disabled)}
|
||||
onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)}
|
||||
|
||||
@@ -8,6 +8,20 @@ export type { CustomsBroker };
|
||||
|
||||
export function createColumns(onSuccess?: () => void): ColumnDef<CustomsBroker>[] {
|
||||
return [
|
||||
{
|
||||
accessorKey: "id",
|
||||
header: "ID",
|
||||
cell: ({ row }) => {
|
||||
const idSnippet = createRawSnippet<[{ id: number }]>((getId) => {
|
||||
const { id } = getId();
|
||||
return {
|
||||
render: () =>
|
||||
`<code class="relative rounded bg-muted px-[0.3rem] py-[0.2rem] font-mono text-sm">${id}</code>`
|
||||
};
|
||||
});
|
||||
return renderSnippet(idSnippet, { id: row.original.id });
|
||||
}
|
||||
},
|
||||
{
|
||||
accessorKey: "broker_key",
|
||||
header: "Clave",
|
||||
|
||||
@@ -1,366 +1,388 @@
|
||||
import {
|
||||
User,
|
||||
Users,
|
||||
FileText,
|
||||
Truck,
|
||||
Container,
|
||||
Ship,
|
||||
Plane,
|
||||
Package,
|
||||
Briefcase,
|
||||
Globe,
|
||||
CreditCard,
|
||||
DollarSign,
|
||||
Calendar,
|
||||
Hash,
|
||||
MapPin,
|
||||
ShieldCheck,
|
||||
FileDigit,
|
||||
Scale,
|
||||
} from 'lucide-svelte';
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
export interface CsvUploadItem {
|
||||
id: string;
|
||||
title: string;
|
||||
icon: any;
|
||||
group?: string; // For grouping within a tab
|
||||
modelTarget?: string; // The backend model this maps to
|
||||
description?: string;
|
||||
templateUrl?: string; // Path to the template file in static/
|
||||
disabled?: boolean; // New property to mark items as "Coming Soon"
|
||||
}
|
||||
|
||||
export interface CsvUploadField {
|
||||
name: string;
|
||||
label: string;
|
||||
type: 'text' | 'select' | 'boolean' | 'date' | 'radio';
|
||||
options?: { label: string; value: string | boolean | number }[];
|
||||
required?: boolean;
|
||||
defaultValue?: any;
|
||||
}
|
||||
|
||||
// Map of Tab ID -> Array of Fields
|
||||
export const tabSettings: Record<string, CsvUploadField[]> = {
|
||||
catalogos: [
|
||||
{
|
||||
name: 'mode',
|
||||
label: 'Modo de Carga',
|
||||
type: 'radio',
|
||||
options: [
|
||||
{ label: 'Actualizar', value: 'update' },
|
||||
{ label: 'Reemplazar', value: 'replace' }
|
||||
],
|
||||
defaultValue: 'update'
|
||||
}
|
||||
],
|
||||
transportes: [
|
||||
{
|
||||
name: 'mode',
|
||||
label: 'Modo de Carga',
|
||||
type: 'radio',
|
||||
options: [
|
||||
{ label: 'Actualizar', value: 'update' },
|
||||
{ label: 'Reemplazar', value: 'replace' }
|
||||
],
|
||||
defaultValue: 'update'
|
||||
}
|
||||
],
|
||||
importacion: [
|
||||
{
|
||||
name: 'autonumber_remesas',
|
||||
label: 'Autonumerar Remesas',
|
||||
type: 'boolean',
|
||||
defaultValue: false
|
||||
},
|
||||
{
|
||||
name: 'recalculate_dates',
|
||||
label: 'Recalcular Fechas',
|
||||
type: 'boolean',
|
||||
defaultValue: false
|
||||
},
|
||||
{
|
||||
name: 'dateFormat',
|
||||
label: 'Formato de Fecha',
|
||||
type: 'select',
|
||||
options: [
|
||||
{ label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' },
|
||||
{ label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' },
|
||||
{ label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' }
|
||||
],
|
||||
defaultValue: 'dd/mm/yyyy'
|
||||
}
|
||||
],
|
||||
exportacion: [
|
||||
{
|
||||
name: 'invoice_type',
|
||||
label: 'Tipo de Factura',
|
||||
type: 'select',
|
||||
options: [
|
||||
{ label: 'AFIJO', value: 'AFIJO' },
|
||||
{ label: 'NORMAL', value: 'NORMAL' },
|
||||
],
|
||||
defaultValue: 'AFIJO',
|
||||
},
|
||||
{
|
||||
name: 'is_regime_change',
|
||||
label: 'Es Cambio de Régimen',
|
||||
type: 'boolean',
|
||||
defaultValue: false,
|
||||
},
|
||||
{
|
||||
name: 'dateFormat',
|
||||
label: 'Formato de Fecha',
|
||||
type: 'select',
|
||||
options: [
|
||||
{ label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' },
|
||||
{ label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' },
|
||||
{ label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' }
|
||||
],
|
||||
defaultValue: 'dd/mm/yyyy'
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
// --- DATA DEFINITIONS (Items only, no config) ---
|
||||
|
||||
export const catalogosConfig: CsvUploadItem[] = [
|
||||
{
|
||||
id: 'customs_brokers',
|
||||
title: 'Agentes Aduanales',
|
||||
icon: User,
|
||||
modelTarget: 'CustomsBroker',
|
||||
templateUrl: '/csv/EstructuraCatAgenteAduanal.xls'
|
||||
},
|
||||
{
|
||||
id: 'clients_providers',
|
||||
title: 'Clientes y Proveedores',
|
||||
icon: Users,
|
||||
modelTarget: 'ClientProvider',
|
||||
templateUrl: '/csv/EstructuraCatClienteProv.xls'
|
||||
},
|
||||
{
|
||||
id: 'exchange_rates',
|
||||
title: 'Tipo de Cambios',
|
||||
icon: DollarSign,
|
||||
modelTarget: 'ExchangeRate',
|
||||
templateUrl: '/csv/EstructuraCatTiposCambio.xls'
|
||||
},
|
||||
{
|
||||
id: 'american_fractions',
|
||||
title: 'Fracc. Ame.',
|
||||
icon: Globe,
|
||||
modelTarget: 'AmericanFraction',
|
||||
templateUrl: '/csv/EstructuraCatFraccAme.xls'
|
||||
},
|
||||
{
|
||||
id: 'material_classes',
|
||||
title: 'Clases de Materiales',
|
||||
icon: Package,
|
||||
modelTarget: 'MaterialClass',
|
||||
templateUrl: '/csv/EstructuraCatClasesAF.xls'
|
||||
},
|
||||
{
|
||||
id: 'items',
|
||||
title: 'Partidas (Permisos)',
|
||||
icon: FileText,
|
||||
group: 'Permisos',
|
||||
modelTarget: 'ItemPermission',
|
||||
templateUrl: '/csv/EstructuraCatPartesAF.xls'
|
||||
},
|
||||
{
|
||||
id: 'headers',
|
||||
title: 'Encabezados (Permisos)',
|
||||
icon: FileText,
|
||||
group: 'Permisos',
|
||||
modelTarget: 'HeaderPermission',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'historical_fractions',
|
||||
title: 'Fracciones Históricas',
|
||||
icon: Calendar,
|
||||
modelTarget: 'HistoricalFraction',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'pedimentos',
|
||||
title: 'Pedimentos',
|
||||
icon: FileDigit,
|
||||
modelTarget: 'Pedimento',
|
||||
templateUrl: '/csv/EstructuraCatPedimentos.xls'
|
||||
},
|
||||
];
|
||||
|
||||
export const transportesConfig: CsvUploadItem[] = [
|
||||
{
|
||||
id: 'transports',
|
||||
title: 'Transportes',
|
||||
icon: Truck,
|
||||
modelTarget: 'Transport',
|
||||
templateUrl: '/csv/EstructuraCatTransportes.xls'
|
||||
},
|
||||
{
|
||||
id: 'drivers',
|
||||
title: 'Conductores',
|
||||
icon: User,
|
||||
modelTarget: 'Driver',
|
||||
templateUrl: '/csv/EstructuraCatConductor.xls'
|
||||
},
|
||||
{
|
||||
id: 'trailers',
|
||||
title: 'Trailers y Cajas',
|
||||
icon: Container,
|
||||
modelTarget: 'Trailer',
|
||||
templateUrl: '/csv/EstructuraCatTrailers.xls'
|
||||
},
|
||||
];
|
||||
|
||||
export const importacionConfig: CsvUploadItem[] = [
|
||||
// Impo Temp
|
||||
{
|
||||
id: 'imp_temp_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Impo. Temp.',
|
||||
modelTarget: 'invoice_header',
|
||||
templateUrl: '/csv/EstructuraEncFacImpoTemp.xls'
|
||||
},
|
||||
{
|
||||
id: 'imp_temp_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Impo. Temp.',
|
||||
modelTarget: 'invoice_details',
|
||||
templateUrl: '/csv/EstructuraParFacImpoTempAF.xls'
|
||||
},
|
||||
{
|
||||
id: 'imp_temp_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Impo. Temp.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
// Impo Def
|
||||
{
|
||||
id: 'imp_def_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
templateUrl: '/csv/EstructuraEncFacImpoDef.xls'
|
||||
},
|
||||
{
|
||||
id: 'imp_def_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
templateUrl: '/csv/EstructuraParFacImpoDefAF.xls'
|
||||
},
|
||||
{
|
||||
id: 'imp_def_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
// Compras Mex
|
||||
{
|
||||
id: 'comp_mex_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'comp_mex_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'comp_mex_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
];
|
||||
|
||||
export const exportacionConfig: CsvUploadItem[] = [
|
||||
// Expo Def / Cam. Reg.
|
||||
{
|
||||
id: 'exp_def_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls'
|
||||
},
|
||||
{
|
||||
id: 'exp_def_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
templateUrl: '/csv/EstructuraParExpoCamReg.xls'
|
||||
},
|
||||
{
|
||||
id: 'exp_def_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'exp_def_nodes',
|
||||
title: 'NODES',
|
||||
icon: Briefcase,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'Nodes',
|
||||
disabled: true,
|
||||
},
|
||||
// Expo Rep
|
||||
{
|
||||
id: 'exp_rep_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Expo. Rep.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'exp_rep_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Expo. Rep.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'exp_rep_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Expo. Rep.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
// Manifiesto
|
||||
{
|
||||
id: 'manifest_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Manifiesto',
|
||||
modelTarget: 'Manifest',
|
||||
disabled: true,
|
||||
},
|
||||
];
|
||||
import {
|
||||
User,
|
||||
Users,
|
||||
FileText,
|
||||
Truck,
|
||||
Container,
|
||||
Ship,
|
||||
Plane,
|
||||
Package,
|
||||
Briefcase,
|
||||
Globe,
|
||||
CreditCard,
|
||||
DollarSign,
|
||||
Calendar,
|
||||
Hash,
|
||||
MapPin,
|
||||
ShieldCheck,
|
||||
FileDigit,
|
||||
Scale,
|
||||
} from 'lucide-svelte';
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
export interface CsvUploadItem {
|
||||
id: string;
|
||||
title: string;
|
||||
icon: any;
|
||||
group?: string; // For grouping within a tab
|
||||
modelTarget?: string; // The backend model this maps to
|
||||
description?: string;
|
||||
/** Backend template id for CSV download (e.g. customs_brokers, part_numbers). No physical file. */
|
||||
templateId?: string;
|
||||
disabled?: boolean; // New property to mark items as "Coming Soon"
|
||||
}
|
||||
|
||||
export interface CsvUploadField {
|
||||
name: string;
|
||||
label: string;
|
||||
type: 'text' | 'select' | 'boolean' | 'date' | 'radio';
|
||||
options?: { label: string; value: string | boolean | number }[];
|
||||
required?: boolean;
|
||||
defaultValue?: any;
|
||||
}
|
||||
|
||||
// Map of Tab ID -> Array of Fields
|
||||
export const tabSettings: Record<string, CsvUploadField[]> = {
|
||||
catalogos: [
|
||||
{
|
||||
name: 'mode',
|
||||
label: 'Modo de Carga',
|
||||
type: 'radio',
|
||||
options: [
|
||||
{ label: 'Actualizar', value: 'update' },
|
||||
{ label: 'Reemplazar', value: 'replace' }
|
||||
],
|
||||
defaultValue: 'update'
|
||||
}
|
||||
],
|
||||
transportes: [
|
||||
{
|
||||
name: 'mode',
|
||||
label: 'Modo de Carga',
|
||||
type: 'radio',
|
||||
options: [
|
||||
{ label: 'Actualizar', value: 'update' },
|
||||
{ label: 'Reemplazar', value: 'replace' }
|
||||
],
|
||||
defaultValue: 'update'
|
||||
}
|
||||
],
|
||||
importacion: [
|
||||
{
|
||||
name: 'autonumber_remesas',
|
||||
label: 'Autonumerar Remesas',
|
||||
type: 'boolean',
|
||||
defaultValue: false
|
||||
},
|
||||
{
|
||||
name: 'recalculate_dates',
|
||||
label: 'Recalcular Fechas',
|
||||
type: 'boolean',
|
||||
defaultValue: false
|
||||
},
|
||||
{
|
||||
name: 'dateFormat',
|
||||
label: 'Formato de Fecha',
|
||||
type: 'select',
|
||||
options: [
|
||||
{ label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' },
|
||||
{ label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' },
|
||||
{ label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' }
|
||||
],
|
||||
defaultValue: 'dd/mm/yyyy'
|
||||
}
|
||||
],
|
||||
exportacion: [
|
||||
{
|
||||
name: 'invoice_type',
|
||||
label: 'Tipo de Factura',
|
||||
type: 'select',
|
||||
options: [
|
||||
{ label: 'AFIJO', value: 'AFIJO' },
|
||||
{ label: 'NORMAL', value: 'NORMAL' },
|
||||
],
|
||||
defaultValue: 'AFIJO',
|
||||
},
|
||||
{
|
||||
name: 'is_regime_change',
|
||||
label: 'Es Cambio de Régimen',
|
||||
type: 'boolean',
|
||||
defaultValue: false,
|
||||
},
|
||||
{
|
||||
name: 'dateFormat',
|
||||
label: 'Formato de Fecha',
|
||||
type: 'select',
|
||||
options: [
|
||||
{ label: 'DD/MM/YYYY', value: 'dd/mm/yyyy' },
|
||||
{ label: 'MM/DD/YYYY', value: 'mm/dd/yyyy' },
|
||||
{ label: 'YYYY-MM-DD', value: 'yyyy-mm-dd' }
|
||||
],
|
||||
defaultValue: 'dd/mm/yyyy'
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
// --- DATA DEFINITIONS (Items only, no config) ---
|
||||
|
||||
export const catalogosConfig: CsvUploadItem[] = [
|
||||
{
|
||||
id: 'customs_brokers',
|
||||
title: 'Agentes Aduanales',
|
||||
icon: User,
|
||||
modelTarget: 'CustomsBroker',
|
||||
templateId: 'customs_brokers'
|
||||
},
|
||||
{
|
||||
id: 'clients_providers',
|
||||
title: 'Clientes y Proveedores',
|
||||
icon: Users,
|
||||
modelTarget: 'ClientProvider',
|
||||
templateId: 'clients_providers'
|
||||
},
|
||||
{
|
||||
id: 'exchange_rates',
|
||||
title: 'Tipo de Cambios',
|
||||
icon: DollarSign,
|
||||
modelTarget: 'ExchangeRate',
|
||||
templateId: 'exchange_rates'
|
||||
},
|
||||
{
|
||||
id: 'american_fractions',
|
||||
title: 'Fracc. Ame.',
|
||||
icon: Globe,
|
||||
modelTarget: 'AmericanFraction',
|
||||
templateId: 'american_fractions'
|
||||
},
|
||||
{
|
||||
id: 'material_classes',
|
||||
title: 'Clases de Materiales',
|
||||
icon: Package,
|
||||
modelTarget: 'MaterialClass',
|
||||
templateId: 'material_classes'
|
||||
},
|
||||
{
|
||||
id: 'part_numbers',
|
||||
title: 'Números de parte',
|
||||
icon: Hash,
|
||||
modelTarget: 'Part',
|
||||
templateId: 'part_numbers',
|
||||
},
|
||||
{
|
||||
id: 'boms',
|
||||
title: 'BOMs',
|
||||
icon: Briefcase,
|
||||
modelTarget: 'Bom',
|
||||
templateId: 'boms',
|
||||
},
|
||||
{
|
||||
id: 'items',
|
||||
title: 'Partidas (Permisos)',
|
||||
icon: FileText,
|
||||
group: 'Permisos',
|
||||
modelTarget: 'ItemPermission',
|
||||
templateId: 'part_numbers'
|
||||
},
|
||||
{
|
||||
id: 'headers',
|
||||
title: 'Encabezados (Permisos)',
|
||||
icon: FileText,
|
||||
group: 'Permisos',
|
||||
modelTarget: 'HeaderPermission',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'historical_fractions',
|
||||
title: 'Fracciones Históricas',
|
||||
icon: Calendar,
|
||||
modelTarget: 'HistoricalFraction',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'pedimentos',
|
||||
title: 'Pedimentos',
|
||||
icon: FileDigit,
|
||||
modelTarget: 'Pedimento',
|
||||
templateId: 'pedimentos'
|
||||
},
|
||||
];
|
||||
|
||||
export const transportesConfig: CsvUploadItem[] = [
|
||||
{
|
||||
id: 'transporters',
|
||||
title: 'Transportistas',
|
||||
icon: Ship,
|
||||
modelTarget: 'Transporter',
|
||||
// No templateId: backend transporters/imports not implemented yet
|
||||
},
|
||||
{
|
||||
id: 'transports',
|
||||
title: 'Transportes',
|
||||
icon: Truck,
|
||||
modelTarget: 'Transport',
|
||||
templateId: 'transports'
|
||||
},
|
||||
{
|
||||
id: 'drivers',
|
||||
title: 'Conductores',
|
||||
icon: User,
|
||||
modelTarget: 'Driver',
|
||||
templateId: 'drivers'
|
||||
},
|
||||
{
|
||||
id: 'trailers',
|
||||
title: 'Trailers y Cajas',
|
||||
icon: Container,
|
||||
modelTarget: 'Trailer',
|
||||
templateId: 'trailers'
|
||||
},
|
||||
];
|
||||
|
||||
export const importacionConfig: CsvUploadItem[] = [
|
||||
// Impo Temp
|
||||
{
|
||||
id: 'imp_temp_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Impo. Temp.',
|
||||
modelTarget: 'invoice_header',
|
||||
templateId: 'imp_temp_header'
|
||||
},
|
||||
{
|
||||
id: 'imp_temp_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Impo. Temp.',
|
||||
modelTarget: 'invoice_details',
|
||||
templateId: 'imp_temp_details'
|
||||
},
|
||||
{
|
||||
id: 'imp_temp_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Impo. Temp.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
// Impo Def
|
||||
{
|
||||
id: 'imp_def_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'invoice_header',
|
||||
templateId: 'imp_def_header'
|
||||
},
|
||||
{
|
||||
id: 'imp_def_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'invoice_details',
|
||||
templateId: 'imp_def_details'
|
||||
},
|
||||
{
|
||||
id: 'imp_def_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
// Compras Mex
|
||||
{
|
||||
id: 'comp_mex_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'invoice_header',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'comp_mex_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'invoice_details',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'comp_mex_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
];
|
||||
|
||||
export const exportacionConfig: CsvUploadItem[] = [
|
||||
// Expo Def / Cam. Reg.
|
||||
{
|
||||
id: 'exp_def_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'invoice_header',
|
||||
templateId: 'exp_def_header'
|
||||
},
|
||||
{
|
||||
id: 'exp_def_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'invoice_details',
|
||||
templateId: 'exp_def_details'
|
||||
},
|
||||
{
|
||||
id: 'exp_def_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'exp_def_nodes',
|
||||
title: 'NODES',
|
||||
icon: Briefcase,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'Nodes',
|
||||
disabled: true,
|
||||
},
|
||||
// Expo Rep
|
||||
{
|
||||
id: 'exp_rep_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Expo. Rep.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'exp_rep_details',
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Expo. Rep.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
id: 'exp_rep_series',
|
||||
title: 'Series',
|
||||
icon: Hash,
|
||||
group: 'Expo. Rep.',
|
||||
modelTarget: 'InvoiceSeries',
|
||||
disabled: true,
|
||||
},
|
||||
// Manifiesto
|
||||
{
|
||||
id: 'manifest_header',
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Manifiesto',
|
||||
modelTarget: 'Manifest',
|
||||
disabled: true,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -1,209 +1,549 @@
|
||||
<script lang="ts">
|
||||
import * as Tabs from '$lib/components/ui/tabs/index.js';
|
||||
import UploadLauncherGrid from '$lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte';
|
||||
import ConfigFooter from '$lib/components/dashboard/csv-upload/ConfigFooter.svelte';
|
||||
import ProcessingResultModal from '$lib/components/dashboard/csv-upload/ProcessingResultModal.svelte';
|
||||
import {
|
||||
catalogosConfig,
|
||||
transportesConfig,
|
||||
importacionConfig,
|
||||
exportacionConfig,
|
||||
tabSettings,
|
||||
type CsvUploadItem
|
||||
} from '$lib/config/csv-upload';
|
||||
import { api } from '$lib/api';
|
||||
import { toast } from 'svelte-sonner';
|
||||
import { companyStore } from '$lib/stores/company.svelte';
|
||||
|
||||
// We no longer need modal state
|
||||
let activeTab = $state('catalogos');
|
||||
|
||||
let isUploading = $state(false);
|
||||
let currentJobId = $state<string | null>(null);
|
||||
let activeModelTarget = $state<string | null>(null);
|
||||
let scanResults = $state<any>(null);
|
||||
let commitResults = $state<any>(null);
|
||||
let showResultModal = $state(false);
|
||||
|
||||
// Initialize settings for all tabs upfront to avoid reactivity loops
|
||||
let allSettings = $state<Record<string, any>>(() => {
|
||||
const initial: Record<string, any> = {};
|
||||
for (const tab in tabSettings) {
|
||||
initial[tab] = {};
|
||||
tabSettings[tab].forEach((f) => {
|
||||
initial[tab][f.name] = f.defaultValue;
|
||||
});
|
||||
}
|
||||
return initial;
|
||||
});
|
||||
|
||||
async function handleUpload(file: File, config: CsvUploadItem) {
|
||||
isUploading = true;
|
||||
activeModelTarget = config.modelTarget || null;
|
||||
scanResults = null;
|
||||
const currentSettings = allSettings[activeTab] || {};
|
||||
const companyId = companyStore.activeCompany?.id || 1;
|
||||
const opType = activeTab === 'exportacion' ? 'exp' : 'imp';
|
||||
|
||||
const res = await api.imports.upload(
|
||||
file,
|
||||
config.modelTarget || '',
|
||||
currentSettings,
|
||||
companyId,
|
||||
opType
|
||||
);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error('Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollStatus() {
|
||||
if (!currentJobId) return;
|
||||
|
||||
const res = await api.imports.status(currentJobId);
|
||||
if (res.data?.status === 'waiting_confirmation') {
|
||||
scanResults = res.data;
|
||||
showResultModal = true;
|
||||
toast.success('Escaneo completado. Revisa los resultados.');
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'failed') {
|
||||
toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido'));
|
||||
isUploading = false;
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
} else if (res.data?.status === 'warning') {
|
||||
// Caso cuando no se insertaron registros pero hay información de rechazo
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const totalSkipped = skippedInvalid + skippedFk;
|
||||
|
||||
if (inserted === 0) {
|
||||
toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`);
|
||||
} else {
|
||||
toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`);
|
||||
}
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'finished') {
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const skippedDetails = res.data?.skipped_details || [];
|
||||
|
||||
if (inserted > 0) {
|
||||
toast.success(`Importación completada: ${inserted} registros insertados`);
|
||||
if (skippedInvalid > 0 || skippedFk > 0) {
|
||||
const totalSkipped = skippedInvalid + skippedFk;
|
||||
toast.warning(`${totalSkipped} registros fueron rechazados`);
|
||||
}
|
||||
} else {
|
||||
toast.error('No se insertaron registros. Revisa los errores a continuación.');
|
||||
}
|
||||
isUploading = false;
|
||||
} else {
|
||||
// Continue polling
|
||||
setTimeout(pollStatus, 2000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col h-[calc(100vh-4rem)] -m-4 overflow-hidden">
|
||||
<!-- Scrollable Content Area -->
|
||||
<div class="flex-1 overflow-y-auto p-4 md:p-8 space-y-4">
|
||||
<div class="flex items-center gap-4">
|
||||
<h1 class="text-lg font-semibold md:text-2xl">Importación Masiva de Datos (CSV)</h1>
|
||||
</div>
|
||||
|
||||
<Tabs.Root bind:value={activeTab} class="w-full">
|
||||
<Tabs.List class="grid w-full grid-cols-2 md:grid-cols-4 lg:w-auto">
|
||||
<Tabs.Trigger value="catalogos">Catálogos</Tabs.Trigger>
|
||||
<Tabs.Trigger value="transportes">Transportes</Tabs.Trigger>
|
||||
<Tabs.Trigger value="importacion">Importación</Tabs.Trigger>
|
||||
<Tabs.Trigger value="exportacion">Exportación</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
|
||||
<div class="mt-6">
|
||||
<Tabs.Content value="catalogos" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Catálogos Generales</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={catalogosConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="transportes" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Logística y Transporte</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={transportesConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="importacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Importación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={importacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="exportacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Exportación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={exportacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
</div>
|
||||
</Tabs.Root>
|
||||
|
||||
<div class="h-4"></div>
|
||||
</div>
|
||||
|
||||
<!-- Fixed Footer Area -->
|
||||
{#if allSettings[activeTab]}
|
||||
<div class="flex-none z-20">
|
||||
<ConfigFooter {activeTab} bind:settings={allSettings[activeTab]} />
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<ProcessingResultModal
|
||||
bind:open={showResultModal}
|
||||
{scanResults}
|
||||
{commitResults}
|
||||
{isUploading}
|
||||
onConfirm={async () => {
|
||||
if (currentJobId && activeModelTarget) {
|
||||
try {
|
||||
isUploading = true;
|
||||
const res = await api.imports.commit(currentJobId, activeModelTarget);
|
||||
if (res.data?.commit_job_id) {
|
||||
currentJobId = res.data.commit_job_id;
|
||||
pollStatus();
|
||||
}
|
||||
} catch (err) {
|
||||
toast.error('Error al iniciar la importación');
|
||||
isUploading = false;
|
||||
}
|
||||
}
|
||||
}}
|
||||
onCancel={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
onClose={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
/>
|
||||
<script lang="ts">
|
||||
import * as Tabs from '$lib/components/ui/tabs/index.js';
|
||||
import UploadLauncherGrid from '$lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte';
|
||||
import ConfigFooter from '$lib/components/dashboard/csv-upload/ConfigFooter.svelte';
|
||||
import ProcessingResultModal from '$lib/components/dashboard/csv-upload/ProcessingResultModal.svelte';
|
||||
import {
|
||||
catalogosConfig,
|
||||
transportesConfig,
|
||||
importacionConfig,
|
||||
exportacionConfig,
|
||||
tabSettings,
|
||||
type CsvUploadItem
|
||||
} from '$lib/config/csv-upload';
|
||||
import { api } from '$lib/api';
|
||||
import { toast } from 'svelte-sonner';
|
||||
import { companyStore } from '$lib/stores/company.svelte';
|
||||
|
||||
// We no longer need modal state
|
||||
let activeTab = $state('catalogos');
|
||||
|
||||
let isUploading = $state(false);
|
||||
let currentJobId = $state<string | null>(null);
|
||||
let activeModelTarget = $state<string | null>(null);
|
||||
let scanResults = $state<any>(null);
|
||||
let commitResults = $state<any>(null);
|
||||
let showResultModal = $state(false);
|
||||
// Cuando es true, usamos API de importación de Agentes Aduanales (customs_brokers/imports)
|
||||
let useCustomsBrokerImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Clientes y Proveedores (clients_and_providers/imports)
|
||||
let useClientProviderImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Tipos de Cambio (exchange_rate/imports)
|
||||
let useExchangeRateImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Fracción Americana (us_tariff_fractions/imports)
|
||||
let useAmericanFractionImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Pedimentos (pedimentos/imports)
|
||||
let usePedimentosImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Clases de Materiales (classes/imports)
|
||||
let useMaterialClassesImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Vehículos / Transportes (vehicles/imports)
|
||||
let useVehicleImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Conductores (drivers/imports)
|
||||
let useDriverImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Trailers y Cajas (trailers/imports)
|
||||
let useTrailerImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Transportistas (transporters/imports)
|
||||
let useTransporterImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Números de parte (parts/imports)
|
||||
let usePartNumbersImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de BOMs (boms/imports)
|
||||
let useBomImport = $state(false);
|
||||
|
||||
// Initialize settings for all tabs upfront to avoid reactivity loops
|
||||
let allSettings = $state<Record<string, any>>(() => {
|
||||
const initial: Record<string, any> = {};
|
||||
for (const tab in tabSettings) {
|
||||
initial[tab] = {};
|
||||
tabSettings[tab].forEach((f) => {
|
||||
initial[tab][f.name] = f.defaultValue;
|
||||
});
|
||||
}
|
||||
return initial;
|
||||
});
|
||||
|
||||
async function handleUpload(file: File, config: CsvUploadItem) {
|
||||
console.log('handleUpload started', { file, config });
|
||||
isUploading = true;
|
||||
activeModelTarget = config.modelTarget || null;
|
||||
scanResults = null;
|
||||
useCustomsBrokerImport = config.id === 'customs_brokers';
|
||||
useClientProviderImport = config.id === 'clients_providers';
|
||||
useExchangeRateImport = config.id === 'exchange_rates';
|
||||
useAmericanFractionImport = config.id === 'american_fractions';
|
||||
usePedimentosImport = config.id === 'pedimentos';
|
||||
useMaterialClassesImport = config.id === 'material_classes';
|
||||
useVehicleImport = config.id === 'transports';
|
||||
useDriverImport = config.id === 'drivers';
|
||||
useTrailerImport = config.id === 'trailers';
|
||||
useTransporterImport = config.id === 'transporters';
|
||||
usePartNumbersImport = config.id === 'part_numbers';
|
||||
useBomImport = config.id === 'boms';
|
||||
|
||||
const companyId = companyStore.activeCompany?.id || 1;
|
||||
|
||||
if (useCustomsBrokerImport) {
|
||||
try {
|
||||
const res = await api.customsBrokerImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useClientProviderImport) {
|
||||
try {
|
||||
const res = await api.clientProviderImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useExchangeRateImport) {
|
||||
try {
|
||||
const res = await api.exchangeRateImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useAmericanFractionImport) {
|
||||
try {
|
||||
const res = await api.americanFractionImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (usePedimentosImport) {
|
||||
try {
|
||||
const res = await api.pedimentosImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useMaterialClassesImport) {
|
||||
try {
|
||||
const res = await api.materialClassImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useVehicleImport) {
|
||||
try {
|
||||
const res = await api.vehicleImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useDriverImport) {
|
||||
try {
|
||||
const res = await api.driverImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useTrailerImport) {
|
||||
try {
|
||||
const res = await api.trailerImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useTransporterImport) {
|
||||
try {
|
||||
const res = await api.transporterImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (usePartNumbersImport) {
|
||||
try {
|
||||
const res = await api.partNumberImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useBomImport) {
|
||||
try {
|
||||
const res = await api.bomImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const currentSettings = allSettings[activeTab] || {};
|
||||
const footerConfig = { ...currentSettings };
|
||||
if (activeTab === 'importacion') {
|
||||
footerConfig.invoice_type = config.id?.startsWith('imp_def_') ? 'DEF' : 'TEM';
|
||||
}
|
||||
const opType = activeTab === 'exportacion' ? 'exp' : 'imp';
|
||||
|
||||
try {
|
||||
const res = await api.imports.upload(
|
||||
file,
|
||||
config.modelTarget || '',
|
||||
footerConfig,
|
||||
companyId,
|
||||
opType,
|
||||
config.id
|
||||
);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollStatus() {
|
||||
if (!currentJobId) return;
|
||||
|
||||
try {
|
||||
const res = useCustomsBrokerImport
|
||||
? await api.customsBrokerImports.status(currentJobId)
|
||||
: useClientProviderImport
|
||||
? await api.clientProviderImports.status(currentJobId)
|
||||
: useExchangeRateImport
|
||||
? await api.exchangeRateImports.status(currentJobId)
|
||||
: useAmericanFractionImport
|
||||
? await api.americanFractionImports.status(currentJobId)
|
||||
: usePedimentosImport
|
||||
? await api.pedimentosImports.status(currentJobId)
|
||||
: useMaterialClassesImport
|
||||
? await api.materialClassImports.status(currentJobId)
|
||||
: useVehicleImport
|
||||
? await api.vehicleImports.status(currentJobId)
|
||||
: useDriverImport
|
||||
? await api.driverImports.status(currentJobId)
|
||||
: useTrailerImport
|
||||
? await api.trailerImports.status(currentJobId)
|
||||
: useTransporterImport
|
||||
? await api.transporterImports.status(currentJobId)
|
||||
: usePartNumbersImport
|
||||
? await api.partNumberImports.status(currentJobId)
|
||||
: useBomImport
|
||||
? await api.bomImports.status(currentJobId)
|
||||
: await api.imports.status(currentJobId);
|
||||
console.log('Poll response', res);
|
||||
if (res.error && !res.data) {
|
||||
toast.error(res.error || 'Error al consultar el estado');
|
||||
isUploading = false;
|
||||
currentJobId = null;
|
||||
return;
|
||||
}
|
||||
if (res.data?.status === 'waiting_confirmation') {
|
||||
scanResults = res.data;
|
||||
showResultModal = true;
|
||||
toast.success('Escaneo completado. Revisa los resultados.');
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'failed' || res.data?.status === 'FAILURE') {
|
||||
const errRaw = res.data.error;
|
||||
const errText =
|
||||
typeof errRaw === 'string'
|
||||
? errRaw.includes('finished') && errRaw.includes('inserted')
|
||||
? 'La importación pudo completarse. Revisa el listado de registros.'
|
||||
: errRaw
|
||||
: (errRaw?.message ?? 'Error desconocido');
|
||||
toast.error('Error en el procesamiento: ' + errText);
|
||||
isUploading = false;
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
} else if (res.data?.status === 'warning') {
|
||||
// Caso cuando no se insertaron registros pero hay información de rechazo
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const skippedDup = res.data?.skipped_duplicate || 0;
|
||||
const totalSkipped = skippedInvalid + skippedFk + skippedDup;
|
||||
|
||||
if (inserted === 0) {
|
||||
toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`);
|
||||
} else {
|
||||
toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`);
|
||||
}
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'finished') {
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const skippedDup = res.data?.skipped_duplicate || 0;
|
||||
const skippedDetails = res.data?.skipped_details || [];
|
||||
|
||||
if (inserted > 0) {
|
||||
toast.success(`Importación completada: ${inserted} registros insertados`);
|
||||
if (skippedInvalid > 0 || skippedFk > 0 || skippedDup > 0) {
|
||||
const totalSkipped = skippedInvalid + skippedFk + skippedDup;
|
||||
toast.warning(`${totalSkipped} registros fueron rechazados`);
|
||||
}
|
||||
} else {
|
||||
toast.error('No se insertaron registros. Revisa los errores a continuación.');
|
||||
}
|
||||
isUploading = false;
|
||||
} else {
|
||||
// Continue polling
|
||||
console.log('Status not final, polling again in 2s...', res.data?.status);
|
||||
setTimeout(pollStatus, 2000);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Poll exception', e);
|
||||
// Retry on network error? Or fail?
|
||||
// For now, let's keep retrying a few times or hard fail.
|
||||
// Let's just log and retry.
|
||||
setTimeout(pollStatus, 2000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col h-[calc(100vh-4rem)] -m-4 overflow-hidden">
|
||||
<!-- Scrollable Content Area -->
|
||||
<div class="flex-1 overflow-y-auto p-4 md:p-8 space-y-4">
|
||||
<div class="flex items-center gap-4">
|
||||
<h1 class="text-lg font-semibold md:text-2xl">Importación Masiva de Datos (CSV)</h1>
|
||||
</div>
|
||||
|
||||
<Tabs.Root bind:value={activeTab} class="w-full">
|
||||
<Tabs.List class="grid w-full grid-cols-2 md:grid-cols-4 lg:w-auto">
|
||||
<Tabs.Trigger value="catalogos">Catálogos</Tabs.Trigger>
|
||||
<Tabs.Trigger value="transportes">Transportes</Tabs.Trigger>
|
||||
<Tabs.Trigger value="importacion">Importación</Tabs.Trigger>
|
||||
<Tabs.Trigger value="exportacion">Exportación</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
|
||||
<div class="mt-6">
|
||||
<Tabs.Content value="catalogos" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Catálogos Generales</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={catalogosConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="transportes" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Logística y Transporte</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={transportesConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="importacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Importación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={importacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="exportacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Exportación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={exportacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
</div>
|
||||
</Tabs.Root>
|
||||
|
||||
<div class="h-4"></div>
|
||||
</div>
|
||||
|
||||
<!-- Fixed Footer Area -->
|
||||
{#if allSettings[activeTab]}
|
||||
<div class="flex-none z-20">
|
||||
<ConfigFooter {activeTab} bind:settings={allSettings[activeTab]} />
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if scanResults || commitResults}
|
||||
<ProcessingResultModal
|
||||
bind:open={showResultModal}
|
||||
{scanResults}
|
||||
{commitResults}
|
||||
{isUploading}
|
||||
onConfirm={async () => {
|
||||
if (!currentJobId) return;
|
||||
try {
|
||||
isUploading = true;
|
||||
const res = useCustomsBrokerImport
|
||||
? await api.customsBrokerImports.commit(currentJobId)
|
||||
: useClientProviderImport
|
||||
? await api.clientProviderImports.commit(currentJobId)
|
||||
: useExchangeRateImport
|
||||
? await api.exchangeRateImports.commit(currentJobId)
|
||||
: useAmericanFractionImport
|
||||
? await api.americanFractionImports.commit(currentJobId)
|
||||
: usePedimentosImport
|
||||
? await api.pedimentosImports.commit(currentJobId)
|
||||
: useMaterialClassesImport
|
||||
? await api.materialClassImports.commit(currentJobId)
|
||||
: useVehicleImport
|
||||
? await api.vehicleImports.commit(currentJobId)
|
||||
: useDriverImport
|
||||
? await api.driverImports.commit(currentJobId)
|
||||
: useTrailerImport
|
||||
? await api.trailerImports.commit(currentJobId)
|
||||
: useTransporterImport
|
||||
? await api.transporterImports.commit(currentJobId)
|
||||
: usePartNumbersImport
|
||||
? await api.partNumberImports.commit(currentJobId)
|
||||
: useBomImport
|
||||
? await api.bomImports.commit(currentJobId)
|
||||
: await api.imports.commit(currentJobId, activeModelTarget || '');
|
||||
if (res.data?.commit_job_id) {
|
||||
currentJobId = res.data.commit_job_id;
|
||||
pollStatus();
|
||||
}
|
||||
} catch (err) {
|
||||
toast.error('Error al iniciar la importación');
|
||||
isUploading = false;
|
||||
}
|
||||
}}
|
||||
onCancel={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
onClose={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
TIPO(MEX=Mexicano,AME=AMERICANO) CLAVE AADUANAL PATENTE NOMBRE RFC DIRECCION CODIGO POSTAL CIUDAD ESTADO PAIS TELEFONO NUMERO FAX CORREO ELECTRONICO CURP
|
||||
@@ -1 +0,0 @@
|
||||
CLAVE CLASE DESCRIPCION ESPA<50>OL DESCRIPCION INGLES TIPO DE MATERIAL U.M. COMERCIAL FRACCION ARANCELARIA FRACCION AMERICANA TASA DE DEPRECIACION REVISION FISICA (1/0) CODIGO DE PRODUCTO/SERVICIO CP
|
||||
@@ -1 +0,0 @@
|
||||
PROCEDENCIA CLIENTE(E=Extranjero, N=Nacional) TIPO(C=Cliente,P=Proveedor,A=Ambos) CLAVE CLIENTE NOMBRE RFC CALLES NUM. EXTERIOR CODIGO POSTAL COLONIA o PARQUE IND. CIUDAD ESTADO PAIS TELEFONO NUMERO FAX CORREO ELECTRONICO CURP TIPO DE PROGRAMA SECON NUMERO DE PROGRAMA SECON FECHA AUT. SECON ##/##/#### ES PROGRAMA PROSEC? (SI o NO) NUMERO DE PROGRAMA PROSEC VINCULACION ES EMPRESA CERTIFICADA? REGISTRO DE EMPRESA CERT. INFORMACION ADICIONAL CONTACTO CLAVE MANUFACTURERO TAX I.D. CLAVE BROKER AMERICANO EXPO CLAVE BROKER AMERICANO IMPO CLAVE TRANSFERENCIA A.A. TRANSFORMADOR/SUBMAQUILA CLAVE INTERFACE
|
||||
@@ -1 +0,0 @@
|
||||
TRANSPORTISTA LINEA CLAVE CONDUCTOR LICENCIA PERMISO LINEA EXPRESS IDENTIFICACION ACE FECHA NACIMIENTO SEXO PAIS NACIMIENTO TRANSPORTA MAT. PELIGROSO? PERMISO MAT. PELIGROSO NOMBRE(S) APELLIDO PATERNO FORMA IDENTIFICACION 1 NUM. IDENTIFICACION 1 ESTADO PAIS FORMA IDENTIFICACION 2 NUM. IDENTIFICACION 2 ESTADO PAIS
|
||||
@@ -1 +0,0 @@
|
||||
FRACCION ARANCELARIA PREFIJO UNIDAD DE MEDIDA DESCRIPCION TIPO DE ADVALOREM ADVALOREM % ADVALOREM DLLS
|
||||
@@ -1 +0,0 @@
|
||||
NUMERO DE PARTE DESCRIPCION EN ESPA<50>OL DESCRIPCION EN INGLES CLASE UNIDAD DE MEDIDA COMERCIAL COSTO UNITARIO TIPO MONEDA COSTO CLAVE MONEDA PESO UNITARIO TIPO PESO FRACCION PAIS PREFERENCIA SECTOR RUTA DE LA IMAGEN
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user