Merge branch 'feature/csv-cajas-trailers' into feature/faltantes-csv

This commit is contained in:
hreyes
2026-03-03 07:59:02 -07:00
83 changed files with 8813 additions and 635 deletions

6
.gitignore vendored
View File

@@ -24,9 +24,11 @@ wheels/
*.egg
.pnpm-store/
# Environment
# Environment (no subir: cada quien puede usar puertos distintos vía .env)
.env
.env.local
backend/.env
frontend/.env
backend/SCRIPTS/
# IDEs
.vscode/
@@ -67,3 +69,5 @@ postgres-data/
backend/uploads/
docker-compose.yml
.mypy_cache/
backend/celerybeat-schedule

0
backend/api/__init__.py Normal file
View File

View File

View File

View File

View File

@@ -2,6 +2,20 @@
Módulo de Class
"""
from .routes import router
from typing import Any
__all__ = ["router"]
def __getattr__(name: str) -> Any:
"""
Lazy export to avoid circular imports during Celery init.
This package is imported when models are loaded (e.g. a76.items.models),
so importing FastAPI routes at import-time can break Celery startup.
"""
if name == "router":
from .routes import router
return router
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

View File

@@ -0,0 +1 @@
# CSV import for Clases de Materiales (upload → scan → status → commit).

View File

@@ -0,0 +1,151 @@
"""
Rutas de importación CSV para Clases de Materiales.
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
CLS_IMPORT_FILE_PREFIX,
CLS_IMPORT_META_PREFIX,
CLS_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"Classes import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "material_classes",
}
try:
r = _get_redis()
r.set(
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=CLS_IMPORT_REDIS_TTL,
)
r.set(
f"{CLS_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=CLS_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Classes import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Classes import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
task = insert_valid_rows.delay(job_id)
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": task.id,
}

View File

@@ -0,0 +1,21 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,528 @@
"""
Tareas Celery para importación CSV de Clases de Materiales.
Flujo: scan_file (validación) → insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List, Set
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
CLS_IMPORT_REDIS_TTL = 3600
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
f"{CLS_IMPORT_META_PREFIX}{job_id}",
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _validate_row_class(
row: Dict[str, Any],
line_num: int,
valid_material_keys: Optional[Set[str]] = None,
valid_uom_codes: Optional[Set[str]] = None,
) -> Optional[Dict[str, Any]]:
class_code = (row.get("CLASE") or "").strip()
if not class_code:
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
if len(class_code) > 8:
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
desc_es = (row.get("DESCRIPCIONE") or "").strip()
if desc_es and len(desc_es) > 500:
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
desc_en = (row.get("DESCRIPCIONI") or "").strip()
if desc_en and len(desc_en) > 500:
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
material_key = (row.get("CLAVEMAT") or "").strip()
if material_key:
if len(material_key) > 10:
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
if valid_material_keys is not None and material_key not in valid_material_keys:
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
uom = (row.get("UNIMED") or "").strip()
if uom:
if len(uom) > 5:
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
if valid_uom_codes is not None and uom not in valid_uom_codes:
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
fraction = (row.get("FRACCION") or "").strip()
if fraction and len(fraction) > 20:
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
us_fraction = (row.get("FRACCIONAME") or "").strip()
if us_fraction and len(us_fraction) > 16:
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
sub_key = (row.get("CLAVESUB") or "").strip()
if sub_key and len(sub_key) > 5:
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
if iva_exempt and len(iva_exempt) > 4:
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
rev_fisica = row.get("REVFISICA")
if rev_fisica is not None and rev_fisica != "":
try:
v = int(rev_fisica)
if v < -32768 or v > 32767:
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
except (ValueError, TypeError):
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
return None
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
logger.info(f"Classes import: starting scan for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"Classes import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
valid_material_keys: Set[str] = set()
valid_uom_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.public.reference_data.material_types.models import MaterialType
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
for m in session.query(MaterialType.key).all():
valid_material_keys.add(m[0])
for u in (
session.query(UnitOfMeasure.code)
.filter(
UnitOfMeasure.tenant_id == tenant_id,
UnitOfMeasure.company_id == company_id,
)
.all()
):
valid_uom_codes.add(u[0])
except Exception as e:
logger.warning(f"Classes import: could not load FK sets: {e}")
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_class(
row_norm, i,
valid_material_keys=valid_material_keys,
valid_uom_codes=valid_uom_codes,
)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"Classes import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=CLS_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _int_or_none(val: Any) -> Optional[int]:
if val is None or val == "":
return None
try:
return int(val)
except (ValueError, TypeError):
return None
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
logger.info(f"Classes import: starting commit for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.classes.models import Class
valid_material_keys: Set[str] = set()
valid_uom_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.public.reference_data.material_types.models import MaterialType
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
for m in session.query(MaterialType.key).all():
valid_material_keys.add(m[0])
for u in (
session.query(UnitOfMeasure.code)
.filter(
UnitOfMeasure.tenant_id == tenant_id,
UnitOfMeasure.company_id == company_id,
)
.all()
):
valid_uom_codes.add(u[0])
except Exception as e:
logger.warning(f"Classes import: could not load FK sets: {e}")
inserted_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
response = None
try:
with CoreSessionLocal() as session:
existing_by_code: Dict[str, Class] = {}
for c in (
session.query(Class)
.filter(
Class.tenant_id == tenant_id,
Class.company_id == company_id,
)
.all()
):
existing_by_code[c.class_code] = c
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_class(
row_norm, i,
valid_material_keys=valid_material_keys,
valid_uom_codes=valid_uom_codes,
)
if err:
skipped_invalid += 1
skipped_details.append(
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
)
continue
class_code = _str_or_none(row_norm.get("CLASE"), 8)
if not class_code:
skipped_invalid += 1
continue
existing = existing_by_code.get(class_code)
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
if material_key and material_key not in valid_material_keys:
material_key = None
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
if unit_of_measure and unit_of_measure not in valid_uom_codes:
unit_of_measure = None
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
physical_review = _int_or_none(row_norm.get("REVFISICA"))
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
if existing:
existing.description_es = desc_es
existing.description_en = desc_en
existing.material_key = material_key
existing.unit_of_measure = unit_of_measure
existing.fraction = fraction
existing.us_fraction = us_fraction
existing.sub_key = sub_key
existing.physical_review = physical_review
existing.iva_exempt_fraction = iva_exempt_fraction
session.add(existing)
inserted_count += 1
else:
new_class = Class(
tenant_id=tenant_id,
company_id=company_id,
class_code=class_code,
description_es=desc_es,
description_en=desc_en,
material_key=material_key,
unit_of_measure=unit_of_measure,
fraction=fraction,
us_fraction=us_fraction,
sub_key=sub_key,
physical_review=physical_review,
iva_exempt_fraction=iva_exempt_fraction,
)
session.add(new_class)
existing_by_code[class_code] = new_class
inserted_count += 1
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"Classes import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
total_skipped = skipped_invalid
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
except Exception as e:
logger.error(f"Classes import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return response

View File

@@ -0,0 +1,45 @@
"""
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
"""
from typing import Dict, List, Any, Optional
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"material_classes": [
{"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]},
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]},
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]},
{"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
{"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]},
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]},
],
}
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
cols = TEMPLATE_COLUMNS.get("material_classes")
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
lookup = build_normalized_lookup(normalize_header_fn)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t
from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse
from .service import ClassService
from .imports.routes import router as imports_router
# Create a new router for custom endpoints
router = APIRouter()
# CSV import (upload → scan → status → commit)
router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"])
# Add consolidated catalog endpoints FIRST (before generic CRUD routes)
# This ensures they have priority over the generic /{id} route
@router.get(

View File

@@ -0,0 +1,2 @@
# CSV import flow for Clientes y Proveedores (Client Providers).
# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit.

View File

@@ -0,0 +1,161 @@
"""
Rutas de importación CSV para Clientes y Proveedores.
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
CP_IMPORT_FILE_PREFIX,
CP_IMPORT_META_PREFIX,
CP_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
"""
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"CP import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "client_providers",
}
try:
r = _get_redis()
r.set(
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=CP_IMPORT_REDIS_TTL,
)
r.set(
f"{CP_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=CP_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"CP import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"CP import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Polling: estado del escaneo o del commit.
"""
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("CP import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
"""
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
"""
task = insert_valid_rows.delay(job_id)
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": task.id,
}

View File

@@ -0,0 +1,25 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class CommitRequest(BaseModel):
pass # no body needed for single model
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,500 @@
"""
Tareas Celery para importación CSV de Clientes y Proveedores.
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
from api.v1.modules.a76.clients_and_providers.models import (
ClientProvider,
ClientProviderAddress,
ClientOrProviderEnum,
)
logger = logging.getLogger(__name__)
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
CP_IMPORT_META_PREFIX = "cp_import_meta:"
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"CP import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
f"{CP_IMPORT_META_PREFIX}{job_id}",
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"CP import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
if not val or not str(val).strip():
return None
v = str(val).strip().lower()
if v in ("client", "cliente", "c"):
return ClientOrProviderEnum.CLIENT
if v in ("provider", "proveedor", "p"):
return ClientOrProviderEnum.PROVIDER
if v in ("both", "ambos", "b", "cliente y proveedor"):
return ClientOrProviderEnum.BOTH
return None
def _parse_active(val: Optional[str]) -> bool:
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
if not val or not str(val).strip():
return True
v = str(val).strip().lower()
if v in ("1", "true", "si", "", "yes", "s", "x"):
return True
if v in ("0", "false", "no", "n"):
return False
return True
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
rfc = (row.get("RFC") or "").strip()
if not rfc:
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
if len(rfc) > 30:
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
tipo_raw = (row.get("TIPO") or "").strip()
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
return {
"line": line_num,
"col": "TIPO",
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
}
name = (row.get("NOMBRE") or "").strip()
if len(name) > 256:
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
short_name = (row.get("SHORT_NAME") or "").strip()
if short_name and len(short_name) > 10:
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
curp = (row.get("CURP") or "").strip()
if curp and len(curp) > 19:
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
return None
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
"""
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
"""
logger.info(f"CP import: starting scan for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"CP import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_client_provider(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"CP import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=CP_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
"""
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
Upsert por (tenant_id, company_id, rfc).
"""
logger.info(f"CP import: starting commit for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"CP import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
inserted_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
response = None
try:
with CoreSessionLocal() as session:
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
existing_by_rfc: Dict[str, ClientProvider] = {}
for cp in (
session.query(ClientProvider)
.filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
)
.all()
):
key = (cp.rfc or "").strip()
existing_by_rfc[key] = cp
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_client_provider(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
rfc = _str_or_none(row_norm.get("RFC"), 30)
if not rfc:
skipped_invalid += 1
skipped_details.append({"line": i, "reason": "RFC requerido"})
continue
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
if client_or_provider is None:
client_or_provider = ClientOrProviderEnum.BOTH
is_active = _parse_active(row_norm.get("ACTIVO"))
existing = existing_by_rfc.get(rfc)
if existing:
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
existing.client_or_provider = client_or_provider
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
existing.is_active = is_active
session.add(existing)
inserted_count += 1
else:
new_cp = ClientProvider(
tenant_id=tenant_id,
company_id=company_id,
rfc=rfc,
name=_str_or_none(row_norm.get("NOMBRE"), 256),
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
curp=_str_or_none(row_norm.get("CURP"), 19),
client_or_provider=client_or_provider,
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
position=_str_or_none(row_norm.get("POSICION"), 30),
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
is_active=is_active,
)
session.add(new_cp)
session.flush()
existing_by_rfc[rfc] = new_cp
inserted_count += 1
# Opcional: crear dirección si hay email/teléfono/dirección
email = _str_or_none(row_norm.get("EMAIL"), 100)
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
if email or phone or address_str:
addr = ClientProviderAddress(
client_id=new_cp.id,
tenant_id=tenant_id,
company_id=company_id,
streets=address_str,
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
city=_str_or_none(row_norm.get("CIUDAD"), 30),
state=_str_or_none(row_norm.get("ESTADO"), 30),
country=_str_or_none(row_norm.get("PAIS"), 3),
phone=phone,
email=email,
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
)
session.add(addr)
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"CP import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
total_skipped = skipped_invalid
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
except Exception as e:
logger.error(f"CP import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
meta_path = file_path.replace(".csv", ".meta.json")
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"CP import cleanup failed: {cleanup_err}")
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return response

View File

@@ -0,0 +1,56 @@
"""
Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls).
Solo se leen columnas definidas aquí; el resto se ignora.
Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls).
"""
from typing import Dict, List, Any, Optional
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"client_providers": [
{"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]},
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]},
{"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]},
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
{"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]},
{"canonical": "CURP", "aliases": []},
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]},
{"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]},
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
{"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]},
{"canonical": "ESTADO", "aliases": []},
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
{"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]},
{"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]},
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
{"canonical": "INCOTERM", "aliases": []},
{"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]},
],
}
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
"""normalized_header -> canonical_name para plantilla client_providers."""
cols = TEMPLATE_COLUMNS.get("client_providers")
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
lookup = build_normalized_lookup(normalize_header_fn)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -20,10 +20,14 @@ from .dto import (
)
from .service import ClientProviderService
from .models import ClientProvider
from .imports.routes import router as imports_router
# Create main router to add custom endpoints
router = APIRouter(prefix="/clients-providers")
# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit)
router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"])
@router.get("/", response_model=ClientProviderPaginatedResponseDTO)
async def get_clients_and_providers(

View File

@@ -0,0 +1,2 @@
# CSV import flow for Agentes Aduanales (Customs Brokers).
# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit.

View File

@@ -0,0 +1,161 @@
"""
Rutas de importación CSV para Agentes Aduanales.
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
CB_IMPORT_FILE_PREFIX,
CB_IMPORT_META_PREFIX,
CB_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
"""
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"CB import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "customs_brokers",
}
try:
r = _get_redis()
r.set(
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=CB_IMPORT_REDIS_TTL,
)
r.set(
f"{CB_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=CB_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"CB import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"CB import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Polling: estado del escaneo o del commit.
"""
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("CB import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
"""
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
"""
task = insert_valid_rows.delay(job_id)
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": task.id,
}

View File

@@ -0,0 +1,22 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class CommitRequest(BaseModel):
pass # no body needed for single model
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,447 @@
"""
Tareas Celery para importación CSV de Agentes Aduanales.
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
# Redis keys (prefijo propio para no colisionar con a76.imports)
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
CB_IMPORT_META_PREFIX = "cb_import_meta:"
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"CB import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
f"{CB_IMPORT_META_PREFIX}{job_id}",
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"CB import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
clave = (row.get("CLAVE") or "").strip()
if not clave:
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
if len(clave) > 5:
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
if not re.match(r"^[a-zA-Z0-9]+$", clave):
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
licencia = (row.get("LICENCIA") or "").strip()
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
return None
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
"""
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
"""
logger.info(f"CB import: starting scan for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"CB import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_customs_broker(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"CB import scan failed: {e}")
return {"status": "failed", "error": str(e)}
# Guardar números de línea con error en Redis para insert_valid_rows
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=CB_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
"""
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
"""
logger.info(f"CB import: starting commit for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"CB import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
inserted_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
response = None
try:
with CoreSessionLocal() as session:
existing_by_key: Dict[str, CustomsBroker] = {}
for b in (
session.query(CustomsBroker)
.filter(
CustomsBroker.tenant_id == tenant_id,
CustomsBroker.company_id == company_id,
)
.all()
):
existing_by_key[b.broker_key] = b
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_customs_broker(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
clave = (row_norm.get("CLAVE") or "").strip()[:5]
if not clave:
skipped_invalid += 1
continue
existing = existing_by_key.get(clave)
if existing:
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
lic = (row_norm.get("LICENCIA") or "").strip()
existing.license = lic[:4] if lic and lic.isdigit() else None
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
session.add(existing)
inserted_count += 1
else:
lic = (row_norm.get("LICENCIA") or "").strip()
license_val = lic[:4] if lic and lic.isdigit() else None
new_broker = CustomsBroker(
tenant_id=tenant_id,
company_id=company_id,
broker_key=clave,
type=_str_or_none(row_norm.get("TIPO"), 9),
name=_str_or_none(row_norm.get("NOMBRE"), 80),
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
city=_str_or_none(row_norm.get("CIUDAD"), 30),
state=_str_or_none(row_norm.get("ESTADO"), 30),
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
fax=_str_or_none(row_norm.get("FAX"), 30),
email=_str_or_none(row_norm.get("EMAIL"), 100),
country=_str_or_none(row_norm.get("PAIS"), 3),
tax_id=_str_or_none(row_norm.get("RFC"), 30),
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
position=_str_or_none(row_norm.get("POSICION"), 30),
license=license_val,
company=_str_or_none(row_norm.get("EMPRESA"), 200),
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
)
session.add(new_broker)
existing_by_key[clave] = new_broker
inserted_count += 1
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"CB import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
total_skipped = skipped_invalid
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
except Exception as e:
logger.error(f"CB import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
# Limpieza
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
meta_path = file_path.replace(".csv", ".meta.json")
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"CB import cleanup failed: {cleanup_err}")
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return response

View File

@@ -0,0 +1,55 @@
"""
Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls).
Solo se leen columnas definidas aquí; el resto se ignora.
"""
from typing import Dict, List, Any, Optional
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"customs_brokers": [
{"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]},
{"canonical": "TIPO"},
{"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]},
{"canonical": "DIRECCION", "aliases": ["DOMICILIO"]},
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
{"canonical": "CIUDAD"},
{"canonical": "ESTADO"},
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]},
{"canonical": "FAX"},
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]},
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]},
{"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]},
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
{"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]},
{"canonical": "EMPRESA", "aliases": ["COMPANY"]},
{"canonical": "CONTACTO", "aliases": ["CONTACT"]},
],
}
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
"""normalized_header -> canonical_name para plantilla customs_brokers."""
cols = TEMPLATE_COLUMNS.get("customs_brokers")
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
lookup = build_normalized_lookup(normalize_header_fn)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
from . import dto, services
from .imports.routes import router as imports_router
router = APIRouter()
# CSV import (mismo flujo que a76.imports: upload → scan → commit)
router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"])
customs_broker_crud = TenantCRUDRoutes(
service=services.CustomsBrokerService,
create_schema=dto.CustomsBrokerCreateDTO,

View File

@@ -0,0 +1 @@
# Exchange rate CSV import module

View File

@@ -0,0 +1,160 @@
"""
Rutas de importación CSV para Tipos de Cambio.
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
ER_IMPORT_FILE_PREFIX,
ER_IMPORT_META_PREFIX,
ER_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
"""
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"ER import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "exchange_rates",
}
try:
r = _get_redis()
r.set(
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=ER_IMPORT_REDIS_TTL,
)
r.set(
f"{ER_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=ER_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"ER import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"ER import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Polling: estado del escaneo o del commit.
"""
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("ER import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
"""
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
"""
task = insert_valid_rows.delay(job_id)
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": task.id,
}

View File

@@ -0,0 +1,25 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class CommitRequest(BaseModel):
pass # no body needed for single model
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,465 @@
"""
Tareas Celery para importación CSV de Tipos de Cambio.
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from datetime import datetime, time
from decimal import Decimal
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
ER_IMPORT_FILE_PREFIX = "er_import_file:"
ER_IMPORT_META_PREFIX = "er_import_meta:"
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
TEMPLATE_ID = "exchange_rates"
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"ER import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
f"{ER_IMPORT_META_PREFIX}{job_id}",
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"ER import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _parse_date(val: Optional[str]) -> Optional[datetime]:
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
if not val or not str(val).strip():
return None
raw = str(val).strip()
for fmt in DATE_FORMATS:
try:
parsed = datetime.strptime(raw, fmt)
return datetime.combine(parsed.date(), time.min)
except ValueError:
continue
return None
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
fecha_raw = (row.get("FECHA") or "").strip()
if not fecha_raw:
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
if _parse_date(fecha_raw) is None:
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
valor_raw = (row.get("VALOR") or "").strip()
if not valor_raw:
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
try:
v = float(valor_raw.replace(",", "."))
if v <= 0:
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
except ValueError:
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
if local_raw and len(local_raw) > 7:
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
if foreign_raw and len(foreign_raw) > 7:
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
return None
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
"""
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
"""
logger.info(f"ER import: starting scan for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"ER import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
err = _validate_row_exchange_rate(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"ER import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=ER_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
"""
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
"""
logger.info(f"ER import: starting commit for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"ER import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
inserted_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
response = None
try:
with CoreSessionLocal() as session:
existing_by_date: Dict[tuple, ExchangeRate] = {}
for er in (
session.query(ExchangeRate)
.filter(
ExchangeRate.tenant_id == tenant_id,
ExchangeRate.company_id == company_id,
)
.all()
):
d = er.date.date() if hasattr(er.date, "date") else er.date
existing_by_date[(tenant_id, company_id, d)] = er
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
err = _validate_row_exchange_rate(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
parsed_date = _parse_date(row_norm.get("FECHA"))
if not parsed_date:
skipped_invalid += 1
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
continue
try:
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
value_decimal = Decimal(str(round(v, 6)))
except (ValueError, TypeError):
skipped_invalid += 1
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
continue
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
if local_currency:
local_currency = local_currency.upper()
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
if foreign_currency:
foreign_currency = foreign_currency.upper()
key_date = parsed_date.date()
existing = existing_by_date.get((tenant_id, company_id, key_date))
if existing:
existing.value = value_decimal
existing.local_currency = local_currency or None
existing.foreign_currency = foreign_currency or None
session.add(existing)
inserted_count += 1
else:
new_er = ExchangeRate(
tenant_id=tenant_id,
company_id=company_id,
date=parsed_date,
value=value_decimal,
local_currency=local_currency,
foreign_currency=foreign_currency,
)
session.add(new_er)
existing_by_date[(tenant_id, company_id, key_date)] = new_er
inserted_count += 1
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"ER import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
total_skipped = skipped_invalid
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
except Exception as e:
logger.error(f"ER import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
meta_path = file_path.replace(".csv", ".meta.json")
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"ER import cleanup failed: {cleanup_err}")
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return response

View File

@@ -0,0 +1,42 @@
"""
Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls).
Solo se leen columnas definidas aquí; el resto se ignora.
"""
from typing import Dict, List, Any, Optional
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"exchange_rates": [
{"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]},
{"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]},
{"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]},
{"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]},
],
}
def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]:
"""normalized_header -> canonical_name para plantilla exchange_rates."""
cols = TEMPLATE_COLUMNS.get(template_id)
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]:
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
lookup = build_normalized_lookup(normalize_header_fn, template_id)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -34,6 +34,10 @@ crud_router = route_handler.router
from fastapi import APIRouter
custom_router = APIRouter(prefix="/exchange-rate", tags=[])
# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered
from .imports.routes import router as imports_router
custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"])
@custom_router.get("/test-ping")
async def test_ping():
return {"message": "pong"}

View File

@@ -0,0 +1 @@
# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit

View File

@@ -0,0 +1,160 @@
"""
Rutas de importación CSV para Fracción Americana (US Tariff Fractions).
Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
FA_IMPORT_FILE_PREFIX,
FA_IMPORT_META_PREFIX,
FA_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
"""
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"FA import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "us_tariff_fractions",
}
try:
r = _get_redis()
r.set(
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=FA_IMPORT_REDIS_TTL,
)
r.set(
f"{FA_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=FA_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"FA import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"FA import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Polling: estado del escaneo o del commit.
"""
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = task_result.info or {}
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("FA import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
"""
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
"""
task = insert_valid_rows.delay(job_id)
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": task.id,
}

View File

@@ -0,0 +1,25 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class CommitRequest(BaseModel):
pass # no body needed for single model
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,474 @@
"""
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from decimal import Decimal
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
FA_IMPORT_META_PREFIX = "fa_import_meta:"
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
TEMPLATE_ID = "us_tariff_fractions"
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"FA import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
f"{FA_IMPORT_META_PREFIX}{job_id}",
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"FA import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _normalize_code(raw: Optional[str]) -> str:
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
if not raw:
return ""
s = str(raw).strip().replace(".", "").replace("-", "")
return s[:16] if len(s) > 16 else s
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
if not code_raw:
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
code_norm = _normalize_code(code_raw)
if not code_norm:
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
if len(code_norm) > 16:
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
prefix_raw = (row.get("PREFIJO") or "").strip()
if prefix_raw and len(prefix_raw) > 10:
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
if um_raw and len(um_raw) > 10:
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
if tipo_raw and len(tipo_raw) > 10:
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
adv_pct = row.get("ADVALOREM_PCT")
if adv_pct is not None and str(adv_pct).strip():
try:
v = float(str(adv_pct).strip().replace(",", "."))
if v < 0:
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
except ValueError:
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
adv_dlls = row.get("ADVALOREM_DLLS")
if adv_dlls is not None and str(adv_dlls).strip():
try:
v = float(str(adv_dlls).strip().replace(",", "."))
if v < 0:
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
except ValueError:
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
return None
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
"""
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
"""
logger.info(f"FA import: starting scan for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"FA import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
err = _validate_row_us_tariff_fraction(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"FA import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=FA_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _parse_float(val: Any) -> Optional[float]:
if val is None or str(val).strip() == "":
return None
try:
return float(str(val).strip().replace(",", "."))
except (ValueError, TypeError):
return None
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
"""
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
"""
logger.info(f"FA import: starting commit for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"FA import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
inserted_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
response = None
try:
with CoreSessionLocal() as session:
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
err = _validate_row_us_tariff_fraction(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
if not code:
skipped_invalid += 1
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
continue
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
description = _str_or_none(row_norm.get("DESCRIPCION"))
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
existing = (
session.query(USTariffFraction)
.filter(
USTariffFraction.tenant_id == tenant_id,
USTariffFraction.company_id == company_id,
USTariffFraction.code == code,
)
.first()
)
if existing:
existing.prefix = prefix
existing.type_code = type_code
existing.ad_valorem = ad_valorem
existing.fixed_cost = fixed_cost
existing.unit_of_measure = unit_of_measure
existing.description = description
session.add(existing)
inserted_count += 1
else:
new_row = USTariffFraction(
tenant_id=tenant_id,
company_id=company_id,
code=code,
prefix=prefix,
type_code=type_code,
ad_valorem=ad_valorem,
fixed_cost=fixed_cost,
unit_of_measure=unit_of_measure,
description=description,
)
session.add(new_row)
inserted_count += 1
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"FA import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
total_skipped = skipped_invalid
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
except Exception as e:
logger.error(f"FA import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
meta_path_clean = file_path.replace(".csv", ".meta.json")
if os.path.exists(meta_path_clean):
os.remove(meta_path_clean)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"FA import cleanup failed: {cleanup_err}")
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return response

View File

@@ -0,0 +1,47 @@
"""
Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls).
Solo se leen columnas definidas aquí; el resto se ignora.
"""
from typing import Dict, List, Any, Optional
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"us_tariff_fractions": [
{"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]},
{"canonical": "PREFIJO"},
{"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]},
{"canonical": "DESCRIPCION"},
{"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]},
{"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]},
{"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]},
],
}
def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]:
"""normalized_header -> canonical_name para plantilla us_tariff_fractions."""
cols = TEMPLATE_COLUMNS.get(template_id)
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(
row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions"
) -> Dict[str, Any]:
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
lookup = build_normalized_lookup(normalize_header_fn, template_id)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -17,23 +17,28 @@ from .dto import (
)
from .service import USTariffFractionService
# Create router using TenantCRUDRoutes factory for basic CRUD operations
# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router)
crud_router = TenantCRUDRoutes(
service=USTariffFractionService,
create_schema=USTariffFractionCreateDTO,
update_schema=USTariffFractionUpdateDTO,
response_schema=USTariffFractionResponseDTO,
prefix="/us-tariff-fractions",
prefix="",
tags=["a76 / general catalogs / us tariff fractions"],
resource_name="US Tariff Fraction",
id_name="id",
enable_list=False, # We implement our custom list endpoint
)
router = crud_router.router
# Master router with prefix so all routes live under /us-tariff-fractions
from .imports.routes import router as imports_router
main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"])
main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"])
main_router.include_router(crud_router.router)
# Custom list endpoint with search filter
@router.get(
# Custom list endpoint with search filter (under /us-tariff-fractions/)
@main_router.get(
"/",
response_model=Dict[str, Any],
summary="List US Tariff Fractions",
@@ -65,3 +70,6 @@ async def list_us_tariff_fractions(
"page_size": page_size,
"pages": (total + page_size - 1) // page_size,
}
router = main_router

View File

@@ -1,5 +1,6 @@
from datetime import datetime
from uuid import uuid4
import base64
import os
import json
import logging
@@ -12,24 +13,39 @@ from core.config import settings
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .tasks import scan_file, insert_valid_rows
from .tasks import (
scan_file,
insert_valid_rows,
IMPORT_FILE_KEY_PREFIX,
IMPORT_META_KEY_PREFIX,
IMPORT_REDIS_TTL,
)
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
"""Redis client (same broker as Celery so worker can read)."""
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload/{model_target}", response_model=ImportJobResponse)
async def upload_import_file(
model_target: Literal["invoice_header", "invoice_details"],
file: UploadFile = File(...),
footer_config: Optional[str] = Form(None), # JSON string with settings
company_id: int = Query(..., description="Company ID"), # Required for context
template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas
company_id: int = Query(..., description="Company ID"), # Required for context
operation_type: Optional[str] = Query("imp"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Step 1: Upload CSV, save to temp, trigger scan task.
Si se envía template_id, solo se leen las columnas de esa plantilla.
"""
# 1. Validate Access & Get Tenant
try:
@@ -40,40 +56,51 @@ async def upload_import_file(
if not file.filename.endswith(".csv"):
raise HTTPException(status_code=400, detail="Only .csv files allowed")
job_id = str(uuid4())
# Ensure directory exists (Safety check)
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"{job_id}.csv")
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"footer_config": footer_config,
"operation_type": operation_type,
"template_id": template_id,
}
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
try:
# Save CSV
contents = await file.read()
redis_client = _get_redis()
redis_client.set(
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
base64.b64encode(contents),
ex=IMPORT_REDIS_TTL,
)
redis_client.set(
f"{IMPORT_META_KEY_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Redis store error: {e}")
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"{job_id}.csv")
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
with open(file_path, "wb") as f:
f.write(contents)
# Save Metadata (Context)
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"footer_config": footer_config,
"operation_type": operation_type,
}
with open(meta_path, "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.error(f"File save error: {e}")
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
logger.warning(f"Local file save failed (worker will use Redis): {e}")
# Trigger Celery Task (Async)
# Use our job_id as the Celery task_id for easier tracking
scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id)
# Trigger Celery Task (Async). Worker loads file from Redis.
scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
@@ -84,24 +111,56 @@ async def upload_import_file(
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Poll this endpoint to get % progress or final report.
Poll to get progress or final report. Always returns an object with "status".
"""
# In a real app, query Redis or DB.
# For MVP, we might mock or use Celery AsyncResult if backend shares Redis.
task_result = celery_app.AsyncResult(job_id)
if task_result.state == 'PENDING':
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
elif task_result.state == 'PROGRESS':
if task_result.state == "PROGRESS":
return {
"status": "processing",
"progress": task_result.info.get('current', 0),
"total": task_result.info.get('total', 0)
"status": "processing",
"progress": (task_result.info or {}).get("current", 0),
"total": (task_result.info or {}).get("total", 0),
}
elif task_result.state == 'SUCCESS':
return task_result.result # Should return the report
else:
return {"status": task_result.state, "error": str(task_result.info)}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb:
logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg and len(lines) > 1:
err_msg = lines[-2] + " " + (lines[-1] or "")
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg:
result = getattr(task_result, "result", None)
info = getattr(task_result, "info", None)
if result is not None and not isinstance(result, dict):
err_msg = str(result)
elif isinstance(result, dict) and (result.get("error") or result.get("message")):
err_msg = result.get("error") or result.get("message")
if not err_msg and isinstance(info, str):
err_msg = info
elif not err_msg and isinstance(info, dict) and "error" in info:
err_msg = str(info["error"])
if not err_msg:
err_msg = "Task failed"
return {"status": "failed", "error": err_msg}
@router.post("/{job_id}/commit")

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,118 @@
"""
Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean.
La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora.
Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models).
"""
from typing import Dict, List, Any, Optional
# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV).
# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que
# espera la lógica de validación e insert (tasks.py).
# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA).
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
# --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) ---
"imp_temp_header": [
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
{"canonical": "FECHA EMISION"},
{"canonical": "CLAVE PROVEEDOR"},
{"canonical": "CLAVE VENDIDO A"},
{"canonical": "CLAVE ENVIADO A"},
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
{"canonical": "ADUANA DE CRUCE"},
{"canonical": "CLAVE MONEDA"},
{"canonical": "CLAVE INCOTERM"},
{"canonical": "TIPO MONEDA"},
{"canonical": "TIPO DE CAMBIO"},
{"canonical": "TIPO PESO"},
{"canonical": "TIPO TRANSPORTE"},
{"canonical": "REMESA"},
{"canonical": "AGENTE ADUANAL"},
{"canonical": "FLETES"},
{"canonical": "VALOR SEGUROS"},
{"canonical": "SEGUROS"},
{"canonical": "EMBALAJES"},
{"canonical": "OTROS INCREMENTABLES"},
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
{"canonical": "FACTURA ALTERNA"},
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
{"canonical": "OBSERVACIONES E"},
{"canonical": "OBSERVACIONES I"},
{"canonical": "E DOCUMENT"},
{"canonical": "NUM OPERACION"},
{"canonical": "CLAVE TRANSPORTISTA"},
{"canonical": "NOMBRE CONDUCTOR"},
{"canonical": "NUMERO TRANSPORTE"},
{"canonical": "PRECINTO"},
],
# --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura ---
"imp_def_header": None, # se resuelve igual que imp_temp_header
# --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura ---
"exp_def_header": None,
# --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) ---
"imp_temp_details": [
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
{"canonical": "CANTIDAD"},
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
{"canonical": "DESCRIPCION"},
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
{"canonical": "FRACCION"},
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
],
# --- Partidas: Impo Def y Expo - misma estructura ---
"imp_def_details": None,
"exp_def_details": None,
}
def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]:
cols = TEMPLATE_COLUMNS.get(template_id)
if cols is not None:
return cols
if template_id in ("imp_def_header", "exp_def_header"):
return TEMPLATE_COLUMNS.get("imp_temp_header")
if template_id in ("imp_def_details", "exp_def_details"):
return TEMPLATE_COLUMNS.get("imp_temp_details")
return None
def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]:
"""
Construye un diccionario: normalized_header -> canonical_name.
normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos).
"""
cols = _resolve_template_columns(template_id)
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]:
"""
A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict
solo con las columnas de la plantilla, usando nombres canónicos.
Así la plantilla se respeta: solo entran columnas definidas en la plantilla.
"""
lookup = build_normalized_lookup(template_id, normalize_header_fn)
if not lookup:
# Sin template definido: comportamiento legacy (normalizar todo)
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.public.reference_data.transport_types.models import TransportType
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode

View File

@@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel):
operation_type: Optional[OperationType] = Field(
..., description="Operation type: imp/exp/sm/ctm"
)
@field_validator("operation_type", mode="before")
@classmethod
def normalize_operation_type(cls, v):
"""Accept DB string (e.g. 'IMP') and coerce to enum value ('imp')."""
if isinstance(v, str):
return v.lower() if v else v
return v
invoice_type: Optional[str] = Field(
None, max_length=5, description="Invoice type key"
)
@@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel):
class InvoiceFinancialsBase(BaseModel):
"""Base fields for Financials"""
currency: Currency = Field(None, max_length=7, description="Currency code")
currency: Optional[Currency] = Field(None, max_length=7, description="Currency code")
currency_type: Optional[str] = Field("USD", description="Currency type")
exchange_rate: Decimal = Field(0.00, description="Exchange rate")
exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate")
exchange_rate_mm: Optional[Decimal] = Field(
None, description="Exchange rate currency to currency"
)

View File

@@ -67,11 +67,13 @@ class InvoiceService:
# Apply filters if provided
if filters:
if filters.get("status"):
if filters.get("status") is not None:
query = query.filter(models.InvoiceHeader.is_updated == filters["status"])
if filters.get("operation_type"):
ot = filters["operation_type"]
ot_val = ot.value if hasattr(ot, "value") else ot
query = query.filter(
models.InvoiceHeader.operation_type == filters["operation_type"]
models.InvoiceHeader.operation_type == ot_val
)
if filters.get("invoice_type"):
query = query.filter(
@@ -89,10 +91,9 @@ class InvoiceService:
f"%{filters['pedimento']}%"
)
)
if (
not filters.get("invoice_type")
and filters.get("operation_type") == "exp"
):
ot_exp = filters.get("operation_type")
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
if not filters.get("invoice_type") and ot_exp_val == "exp":
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
if filters.get("manifest_number"):

View File

@@ -0,0 +1 @@
# CSV import for Pedimentos (upload → scan → commit)

View File

@@ -0,0 +1,160 @@
"""
Rutas de importación CSV para Pedimentos.
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
PED_IMPORT_FILE_PREFIX,
PED_IMPORT_META_PREFIX,
PED_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
"""
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"Pedimentos import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "pedimentos",
}
try:
r = _get_redis()
r.set(
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=PED_IMPORT_REDIS_TTL,
)
r.set(
f"{PED_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=PED_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Pedimentos import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Pedimentos import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Polling: estado del escaneo o del commit.
"""
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = task_result.info or {}
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
"""
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
"""
task = insert_valid_rows.delay(job_id)
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": task.id,
}

View File

@@ -0,0 +1,25 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class CommitRequest(BaseModel):
pass # no body needed for single model
class ImportJobStatus(BaseModel):
status: str
job_id: Optional[str] = None
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,563 @@
"""
Tareas Celery para importación CSV de Pedimentos.
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from decimal import Decimal, InvalidOperation
from datetime import datetime
from typing import Dict, Any, Optional, List, Set
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
# Redis keys (prefijo propio para no colisionar con otros imports)
PED_IMPORT_FILE_PREFIX = "ped_import_file:"
PED_IMPORT_META_PREFIX = "ped_import_meta:"
PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:"
PED_IMPORT_REDIS_TTL = 3600 # 1 hour
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"ped_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
f"{PED_IMPORT_META_PREFIX}{job_id}",
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _validate_row_pedimento(
row: Dict[str, Any],
line_num: int,
valid_client_ids: Set[int],
valid_regimes: Set[str],
valid_pedimento_codes: Set[str],
) -> Optional[Dict[str, Any]]:
"""Valida una fila para Pedimento. Retorna error dict o None."""
# Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN
year = (row.get("AÑO") or "").strip()
if not year:
return {"line": line_num, "col": "AÑO", "msg": "Requerido"}
if len(year) > 2:
return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"}
customs_office = (row.get("ADUANA") or "").strip()
if not customs_office:
return {"line": line_num, "col": "ADUANA", "msg": "Requerido"}
if len(customs_office) > 3:
return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"}
license_val = (row.get("PATENTE") or "").strip()
if not license_val:
return {"line": line_num, "col": "PATENTE", "msg": "Requerido"}
if len(license_val) > 4:
return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"}
pedimento_number = (row.get("NUMERO") or "").strip()
if not pedimento_number:
return {"line": line_num, "col": "NUMERO", "msg": "Requerido"}
if len(pedimento_number) > 7:
return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"}
client_id_str = (row.get("CLIENTE_ID") or "").strip()
if not client_id_str:
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"}
try:
client_id = int(client_id_str)
except ValueError:
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"}
if client_id not in valid_client_ids:
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"}
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()
if not pedimento_code:
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"}
if len(pedimento_code) > 2:
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"}
if pedimento_code not in valid_pedimento_codes:
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"}
regime = (row.get("REGIMEN") or "").strip()
if not regime:
return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"}
if len(regime) > 3:
return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"}
if regime not in valid_regimes:
return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"}
# Optional numeric/string fields - validate format if present
status = (row.get("ESTATUS") or "").strip()
if status and len(status) > 30:
return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"}
for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]:
val = (row.get(col) or "").strip()
if not val:
continue
try:
Decimal(val.replace(",", "."))
except (InvalidOperation, ValueError):
return {"line": line_num, "col": col, "msg": "Valor numérico inválido"}
operation_type = (row.get("TIPO_OPERACION") or "").strip().lower()
if operation_type and operation_type not in ("imp", "exp", ""):
return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"}
pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""):
return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"}
return None
def _parse_decimal(val: Any) -> Optional[Decimal]:
if val is None or (isinstance(val, str) and not val.strip()):
return None
try:
return Decimal(str(val).strip().replace(",", "."))
except (InvalidOperation, ValueError):
return None
def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]:
"""Build dict for PedimentosCreate from normalized CSV row (canonical names)."""
year = (row.get("AÑO") or "").strip()[:2]
customs_office = (row.get("ADUANA") or "").strip()[:3]
license_val = (row.get("PATENTE") or "").strip()[:4]
pedimento_number = (row.get("NUMERO") or "").strip()[:7]
client_id_str = (row.get("CLIENTE_ID") or "").strip()
client_id = int(client_id_str) if client_id_str else None
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2]
regime = (row.get("REGIMEN") or "").strip()[:3]
data = {
"year": year,
"customs_office": customs_office,
"license": license_val,
"pedimento_number": pedimento_number,
"client_id": client_id,
"pedimento_code": pedimento_code,
"regime": regime,
}
op = (row.get("TIPO_OPERACION") or "").strip().lower()
if op in ("imp", "exp"):
data["operation_type"] = op
ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
if ptype in ("normal", "consolidated", "complementary", "automobile"):
data["pedimento_type"] = ptype
status = (row.get("ESTATUS") or "").strip()
if status:
data["status"] = status[:30]
data["usd_value"] = _parse_decimal(row.get("VALOR_USD"))
data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO"))
data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO"))
data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO"))
obs = (row.get("OBSERVACIONES") or "").strip()
if obs:
data["observations"] = obs
return data
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
"""
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
"""
logger.info(f"Pedimentos import: starting scan for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"Pedimentos import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
# Load valid FK sets for validation
valid_client_ids: Set[int] = set()
valid_regimes: Set[str] = set()
valid_pedimento_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
for cp in session.query(ClientProvider).filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
).all():
valid_client_ids.add(cp.id)
for r in session.query(RegimenPedimento).all():
valid_regimes.add(r.code)
for pc in session.query(PedimentoCode).all():
valid_pedimento_codes.add(pc.code)
except Exception as e:
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header, "pedimentos")
err = _validate_row_pedimento(
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"Pedimentos import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=PED_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
"""
Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create.
"""
logger.info(f"Pedimentos import: starting commit for job {job_id}")
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
# Reload FK sets for commit-time validation
valid_client_ids: Set[int] = set()
valid_regimes: Set[str] = set()
valid_pedimento_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
for cp in session.query(ClientProvider).filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
).all():
valid_client_ids.add(cp.id)
for r in session.query(RegimenPedimento).all():
valid_regimes.add(r.code)
for pc in session.query(PedimentoCode).all():
valid_pedimento_codes.add(pc.code)
except Exception as e:
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
return {"status": "failed", "error": str(e)}
from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate
from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate
from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService
inserted_count = 0
skipped_invalid = 0
skipped_missing_fk = 0
skipped_duplicate = 0
skipped_details: List[Dict[str, Any]] = []
response = None
try:
with CoreSessionLocal() as session:
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header, "pedimentos")
err = _validate_row_pedimento(
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
)
if err:
skipped_invalid += 1
skipped_details.append(
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
)
continue
try:
data = _row_to_pedimentos_create(row_norm)
# Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
data["pedimento_dates"] = PedimentoDatesCreate(
entry_date=datetime.now(),
end_date=datetime.now(),
)
create_data = PedimentosCreate(**data)
PedimentosService.create(session, create_data, tenant_id, company_id)
inserted_count += 1
except ValueError as ve:
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
skipped_duplicate += 1
skipped_details.append({"line": i, "reason": str(ve)})
else:
skipped_invalid += 1
skipped_details.append({"line": i, "reason": str(ve)})
except Exception as e:
logger.warning(f"Pedimentos import line {i}: {e}")
skipped_invalid += 1
skipped_details.append({"line": i, "reason": str(e)})
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": skipped_missing_fk,
"skipped_duplicate": skipped_duplicate,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": skipped_missing_fk,
"skipped_duplicate": skipped_duplicate,
"skipped_details": skipped_details,
}
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": skipped_missing_fk,
"skipped_duplicate": skipped_duplicate,
"skipped_details": skipped_details,
}
except Exception as e:
logger.error(f"Pedimentos import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
meta_path = file_path.replace(".csv", ".meta.json")
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}")
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": skipped_missing_fk,
"skipped_duplicate": skipped_duplicate,
"skipped_details": skipped_details,
}
return response

View File

@@ -0,0 +1,53 @@
"""
Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls).
Solo se leen columnas definidas aquí; el resto se ignora.
"""
from typing import Dict, List, Any, Optional
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"pedimentos": [
{"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]},
{"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]},
{"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]},
{"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]},
{"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]},
{"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]},
{"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]},
{"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]},
{"canonical": "REGIMEN", "aliases": ["REGIME"]},
{"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]},
{"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]},
{"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]},
{"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]},
{"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]},
{"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]},
],
}
def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]:
"""normalized_header -> canonical_name para plantilla pedimentos."""
cols = TEMPLATE_COLUMNS.get(template_id)
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]:
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
lookup = build_normalized_lookup(normalize_header_fn, template_id)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import (
from .routes.pedimento_transport_means import router as pedimento_transport_means_router
from .routes.pedimento_validation import router as pedimento_validation_router
from .routes.pedimentos import router as pedimentos_router
from .imports.routes import router as pedimentos_imports_router
router = APIRouter()
@@ -117,3 +118,8 @@ router.include_router(
router.include_router(
pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"]
)
router.include_router(
pedimentos_imports_router,
prefix="/pedimentos/imports",
tags=["a76 / pedimentos / csv_import"],
)

View File

@@ -12,7 +12,7 @@ from .general_catalogs.router import router as general_catalogs_router
from .invoices.routes import router as invoices_router
from .items.routes import router as items_router
from .classes import router as classes_router
from .classes import router as classes_router
from .clients_and_providers import router as client_and_provider_router
from .imports.routes import router as imports_router
from .invoice_settings.routes import router as invoice_settings_router

View File

@@ -6,28 +6,28 @@ from pydantic import BaseModel
class DriverBaseDTO(BaseModel):
transporter_key: str
line: int
driver_name: Optional[str]
license_number: Optional[str]
express_line_id: Optional[str]
ace_id: Optional[str]
birth_date: Optional[int]
gender: Optional[str]
birth_country: Optional[str]
hazardous_material_auth: Optional[str]
hazardous_material_state: Optional[str]
first_name: Optional[str]
last_name: Optional[str]
id_key1: Optional[str]
id_number1: Optional[str]
id_state1: Optional[str]
id_country1: Optional[str]
id_key2: Optional[str]
id_number2: Optional[str]
id_state2: Optional[str]
id_country2: Optional[str]
badge_number: Optional[str]
class_type: Optional[str]
unique_badge_number: Optional[str]
driver_name: Optional[str] = None
license_number: Optional[str] = None
express_line_id: Optional[str] = None
ace_id: Optional[str] = None
birth_date: Optional[int] = None
gender: Optional[str] = None
birth_country: Optional[str] = None
hazardous_material_auth: Optional[str] = None
hazardous_material_state: Optional[str] = None
first_name: Optional[str] = None
last_name: Optional[str] = None
id_key1: Optional[str] = None
id_number1: Optional[str] = None
id_state1: Optional[str] = None
id_country1: Optional[str] = None
id_key2: Optional[str] = None
id_number2: Optional[str] = None
id_state2: Optional[str] = None
id_country2: Optional[str] = None
badge_number: Optional[str] = None
class_type: Optional[str] = None
unique_badge_number: Optional[str] = None
company_id: int
tenant_id: int

View File

@@ -0,0 +1 @@
# CSV import for Drivers / Conductores (upload -> scan -> status -> commit).

View File

@@ -0,0 +1,188 @@
"""
Rutas de importacion CSV para Conductores.
Flujo: upload -> scan -> status (polling) -> commit.
"""
import base64
import json
import logging
import os
import threading
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
run_scan_sync,
run_commit_sync,
DRV_IMPORT_FILE_PREFIX,
DRV_IMPORT_META_PREFIX,
DRV_IMPORT_STATUS_PREFIX,
DRV_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"Drivers import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "drivers",
}
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=DRV_IMPORT_REDIS_TTL,
)
r.set(
f"{DRV_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Drivers import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Drivers import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
def run_scan_background():
try:
run_scan_sync(job_id)
except Exception as e:
logger.exception(f"Drivers import: background scan failed: {e}")
threading.Thread(target=run_scan_background, daemon=True).start()
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
try:
r = _get_redis()
raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
return data
except Exception as e:
logger.debug(f"Drivers import: could not read status from Redis: {e}")
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.debug(f"Drivers import: could not write processing status: {e}")
def run_commit_background():
try:
run_commit_sync(job_id)
except Exception as e:
logger.exception(f"Drivers import: background commit failed: {e}")
threading.Thread(target=run_commit_background, daemon=True).start()
return {
"status": "committing",
"message": "Insercion iniciada.",
"commit_job_id": job_id,
}

View File

@@ -0,0 +1,23 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
updated: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_duplicate: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,640 @@
"""
Tareas Celery para importacion CSV de Conductores.
Flujo: scan_file (validacion) -> insert_valid_rows (commit).
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
DRV_IMPORT_REDIS_TTL = 3600
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"Drivers import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"drv_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"Drivers import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
f"{DRV_IMPORT_META_PREFIX}{job_id}",
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"Drivers import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
_MAX = {
"transporter_key": 5,
"driver_name": 80,
"license_number": 29,
"express_line_id": 17,
"ace_id": 20,
"gender": 1,
"birth_country": 3,
"hazardous_material_auth": 2,
"hazardous_material_state": 30,
"first_name": 20,
"last_name": 20,
"id_key1": 40,
"id_number1": 20,
"id_state1": 30,
"id_country1": 3,
"id_key2": 40,
"id_number2": 20,
"id_state2": 30,
"id_country2": 3,
"badge_number": 20,
"class_type": 1,
"unique_badge_number": 100,
}
def _parse_int(val: Any) -> Optional[int]:
if val is None or (isinstance(val, str) and not val.strip()):
return None
s = str(val).strip()
if re.match(r"^\d+$", s):
return int(s)
if re.match(r"^\d+\.0+$", s):
try:
return int(float(s))
except ValueError:
return None
return None
def _parse_birth_date(val: Any) -> Optional[int]:
if val is None or (isinstance(val, str) and not val.strip()):
return None
s = str(val).strip()
if re.match(r"^\d{8}$", s):
try:
return int(s)
except ValueError:
return None
if re.match(r"^\d+\.0+$", s):
try:
return int(float(s))
except ValueError:
return None
for sep in ["/", "-", "."]:
if sep in s:
parts = s.split(sep)
if len(parts) == 3:
try:
a, b, c = [p.strip() for p in parts]
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
return int(c) * 10000 + int(b) * 100 + int(a)
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
return int(a) * 10000 + int(b) * 100 + int(c)
except (ValueError, TypeError):
return None
break
return None
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _dedupe_headers(headers: List[str]) -> List[str]:
counts: Dict[str, int] = {}
unique: List[str] = []
for header in headers:
name = str(header or "").strip() or "COL"
count = counts.get(name, 0) + 1
counts[name] = count
if count == 1:
unique.append(name)
else:
unique.append(f"{name} {count}")
return unique
def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
transporter_key = (row.get("TRANSPORTISTA") or "").strip()
if not transporter_key:
return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"}
if len(transporter_key) > _MAX["transporter_key"]:
return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"}
line_val = _parse_int(row.get("LINEA"))
if line_val is None:
return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"}
if line_val <= 0:
return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"}
driver_name = (row.get("CLAVE CONDUCTOR") or "").strip()
if driver_name and len(driver_name) > _MAX["driver_name"]:
return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"}
for col, max_len in [
("LICENCIA", _MAX["license_number"]),
("PERMISO LINEA EXPRESS", _MAX["express_line_id"]),
("IDENTIFICACION ACE", _MAX["ace_id"]),
("SEXO", _MAX["gender"]),
("PAIS NACIMIENTO", _MAX["birth_country"]),
("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]),
("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]),
("NOMBRE(S)", _MAX["first_name"]),
("APELLIDO PATERNO", _MAX["last_name"]),
("FORMA IDENTIFICACION 1", _MAX["id_key1"]),
("NUM. IDENTIFICACION 1", _MAX["id_number1"]),
("ESTADO", _MAX["id_state1"]),
("PAIS", _MAX["id_country1"]),
("FORMA IDENTIFICACION 2", _MAX["id_key2"]),
("NUM. IDENTIFICACION 2", _MAX["id_number2"]),
("ESTADO 2", _MAX["id_state2"]),
("PAIS 2", _MAX["id_country2"]),
]:
val = (row.get(col) or "").strip()
if val and len(val) > max_len:
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
fecha = row.get("FECHA NACIMIENTO")
if fecha is not None and str(fecha).strip():
if _parse_birth_date(fecha) is None:
return {
"line": line_num,
"col": "FECHA NACIMIENTO",
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
}
return None
def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"])
line = _parse_int(row.get("LINEA"))
if not transporter_key or line is None:
return {}
data = {
"transporter_key": transporter_key,
"line": line,
"driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]),
"license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]),
"express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]),
"ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]),
"birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")),
"gender": _str_or_none(row.get("SEXO"), _MAX["gender"]),
"birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]),
"hazardous_material_auth": _str_or_none(
row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"]
),
"hazardous_material_state": _str_or_none(
row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"]
),
"first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]),
"last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]),
"id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]),
"id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]),
"id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]),
"id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]),
"id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]),
"id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]),
"id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]),
"id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]),
"company_id": company_id,
"tenant_id": tenant_id,
}
return data
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"Drivers import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.reader(f_in, dialect=dialect)
try:
headers = next(reader)
except StopIteration:
headers = []
headers = _dedupe_headers(headers)
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
for i, row in enumerate(dict_reader, start=1):
if progress_callback and i % 500 == 0:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_driver(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"Drivers import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Drivers import: failed to store error lines: {e}")
result = {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
return result
def run_scan_sync(job_id: str) -> Dict[str, Any]:
result = _do_scan(job_id, progress_callback=None)
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
return result
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
logger.info(f"Drivers import: starting scan for job {job_id}")
def on_progress(current: int, total: int, errors: int) -> None:
self.update_state(
state="PROGRESS",
meta={"current": current, "total": total, "errors": errors},
)
result = _do_scan(job_id, progress_callback=on_progress)
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
return result
def _do_commit(job_id: str) -> Dict[str, Any]:
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"Drivers import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.transportation.drivers.services import DriverService
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
inserted_count = 0
updated_count = 0
skipped_invalid = 0
skipped_duplicate = 0
skipped_details: List[Dict[str, Any]] = []
seen_keys_in_file: Dict[str, int] = {}
try:
with CoreSessionLocal() as session:
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.reader(f, dialect=dialect)
try:
headers = next(reader)
except StopIteration:
headers = []
headers = _dedupe_headers(headers)
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
for i, row in enumerate(dict_reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_driver(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
"invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
data = _row_to_driver_dto(row_norm, tenant_id, company_id)
if not data or not data.get("transporter_key") or data.get("line") is None:
skipped_invalid += 1
continue
key = f"{data['transporter_key']}:{data['line']}"
if key in seen_keys_in_file:
skipped_duplicate += 1
skipped_details.append(
{
"line": i,
"driver_key": key,
"invoice": key,
"reason": "Clave duplicada en el archivo (se usa la primera)",
}
)
continue
seen_keys_in_file[key] = i
existing = DriverService.get_driver_by_key_and_line(
session, data["transporter_key"], data["line"], str(company_id), tenant_id
)
try:
if existing:
update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")}
for field, value in update_fields.items():
setattr(existing, field, value)
session.add(existing)
updated_count += 1
else:
create_data = DriverCreateDTO(**data)
DriverService.create_driver(session, create_data)
inserted_count += 1
except Exception as db_err:
session.rollback()
skipped_invalid += 1
skipped_details.append(
{"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)}
)
continue
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"Drivers import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
except Exception as e:
logger.error(f"Drivers import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"Drivers import cleanup failed: {cleanup_err}")
total_ok = inserted_count + updated_count
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
return {
"status": "warning",
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
}
if total_ok == 0:
return {
"status": "failed",
"error": "No hay registros validos en el archivo CSV",
"inserted": 0,
"updated": 0,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return {
"status": "finished",
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
def run_commit_sync(job_id: str) -> Dict[str, Any]:
result = _do_commit(job_id)
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
return result
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
logger.info(f"Drivers import: starting commit for job {job_id}")
result = _do_commit(job_id)
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
return result

View File

@@ -0,0 +1,56 @@
"""
Configuracion de plantilla CSV para Conductores (EstructuraCatConductor.xls).
"""
from typing import Dict, List, Any
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"drivers": [
{"canonical": "TRANSPORTISTA", "aliases": ["TRANSPORTISTA CLAVE", "CLAVE TRANSPORTISTA", "TRANSPORTER"]},
{"canonical": "LINEA", "aliases": ["LINE", "LINEA CONDUCTOR"]},
{"canonical": "CLAVE CONDUCTOR", "aliases": ["CONDUCTOR", "DRIVER", "NOMBRE CONDUCTOR"]},
{"canonical": "LICENCIA", "aliases": ["LICENSE", "LICENCE"]},
{"canonical": "PERMISO LINEA EXPRESS", "aliases": ["LINEA EXPRESS", "LINEA EXPRESS ID", "EXPRESS LINE"]},
{"canonical": "IDENTIFICACION ACE", "aliases": ["ACE", "ACE ID", "IDENTIFICACION ACE ID"]},
{"canonical": "FECHA NACIMIENTO", "aliases": ["FECHA NAC", "BIRTH DATE", "BIRTHDATE"]},
{"canonical": "SEXO", "aliases": ["GENERO", "GENDER"]},
{"canonical": "PAIS NACIMIENTO", "aliases": ["PAIS Nacimiento", "BIRTH COUNTRY"]},
{"canonical": "TRANSPORTA MAT. PELIGROSO?", "aliases": ["MATERIAL PELIGROSO", "HAZMAT", "HAZARDOUS MATERIAL"]},
{"canonical": "PERMISO MAT. PELIGROSO", "aliases": ["PERMISO MAT PELIGROSO", "HAZMAT PERMIT"]},
{"canonical": "NOMBRE(S)", "aliases": ["NOMBRE", "NOMBRES", "FIRST NAME"]},
{"canonical": "APELLIDO PATERNO", "aliases": ["APELLIDO", "LAST NAME", "APELLIDO P"]},
{"canonical": "FORMA IDENTIFICACION 1", "aliases": ["FORMA IDENTIFICACION", "ID TIPO 1", "ID KEY 1"]},
{"canonical": "NUM. IDENTIFICACION 1", "aliases": ["NUM IDENTIFICACION 1", "ID NUMERO 1", "ID NUMBER 1"]},
{"canonical": "ESTADO", "aliases": ["ESTADO 1", "STATE 1"]},
{"canonical": "PAIS", "aliases": ["PAIS 1", "COUNTRY 1"]},
{"canonical": "FORMA IDENTIFICACION 2", "aliases": ["ID TIPO 2", "ID KEY 2"]},
{"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]},
{"canonical": "ESTADO 2", "aliases": ["STATE 2"]},
{"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]},
],
}
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
cols = TEMPLATE_COLUMNS.get("drivers")
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
lookup = build_normalized_lookup(normalize_header_fn)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -8,9 +8,13 @@ from sqlalchemy.orm import Session
from .dto import DriverCreateDTO, DriverResponseDTO
from .models import Driver
from .services import DriverService
from .imports.routes import router as imports_router
router = APIRouter(prefix="/drivers")
# CSV import (upload -> scan -> status -> commit)
router.include_router(imports_router, prefix="/imports", tags=["a76 / drivers / csv_import"])
@router.get("/", response_model=Dict[str, Any])
async def list_drivers(

View File

@@ -0,0 +1 @@
# Trailers CSV import: upload -> scan -> status -> commit

View File

@@ -0,0 +1,188 @@
"""
Rutas de importación CSV para Trailers y Cajas.
Flujo: upload -> scan -> status (polling) -> commit.
"""
import base64
import json
import logging
import os
import threading
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
run_scan_sync,
run_commit_sync,
TRL_IMPORT_FILE_PREFIX,
TRL_IMPORT_META_PREFIX,
TRL_IMPORT_STATUS_PREFIX,
TRL_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"Trailers import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "trailers",
}
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=TRL_IMPORT_REDIS_TTL,
)
r.set(
f"{TRL_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Trailers import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Trailers import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
def run_scan_background():
try:
run_scan_sync(job_id)
except Exception as e:
logger.exception(f"Trailers import: background scan failed: {e}")
threading.Thread(target=run_scan_background, daemon=True).start()
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
try:
r = _get_redis()
raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
return data
except Exception as e:
logger.debug(f"Trailers import: could not read status from Redis: {e}")
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.debug(f"Trailers import: could not write processing status: {e}")
def run_commit_background():
try:
run_commit_sync(job_id)
except Exception as e:
logger.exception(f"Trailers import: background commit failed: {e}")
threading.Thread(target=run_commit_background, daemon=True).start()
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": job_id,
}

View File

@@ -0,0 +1,23 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
updated: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_duplicate: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,535 @@
"""
Tareas Celery para importación CSV de Trailers y Cajas.
Flujo: scan_file (validación) → insert_valid_rows (commit).
Upsert por trailer_number usando TrailerService.
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
TRL_IMPORT_FILE_PREFIX = "trl_import_file:"
TRL_IMPORT_META_PREFIX = "trl_import_meta:"
TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:"
TRL_IMPORT_STATUS_PREFIX = "trl_import_status:"
TRL_IMPORT_REDIS_TTL = 3600
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"Trailers import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"trl_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"Trailers import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
f"{TRL_IMPORT_META_PREFIX}{job_id}",
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"Trailers import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
# Max lengths from Trailer model (a76.trailer)
_MAX = {
"trailer_number": 20,
"ace_trailer_number": 10,
"trailer_type_key": 2,
"seal": 15,
"entity_code": 1,
"plate_number": 17,
"state": 30,
"country": 3,
"container_key": 3,
}
def _dedupe_headers(headers: List[str]) -> List[str]:
counts: Dict[str, int] = {}
unique: List[str] = []
for header in headers:
name = str(header or "").strip() or "COL"
count = counts.get(name, 0) + 1
counts[name] = count
if count == 1:
unique.append(name)
else:
unique.append(f"{name} {count}")
return unique
def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
"""Valida una fila para Trailer. Retorna error dict o None."""
trailer_number = (row.get("NUMERO TRAILER") or "").strip()
if not trailer_number:
return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"}
if len(trailer_number) > _MAX["trailer_number"]:
return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"}
for col, max_len in [
("CLAVE ACE", _MAX["ace_trailer_number"]),
("TIPO TRAILER", _MAX["trailer_type_key"]),
("PRECINTO", _MAX["seal"]),
("CODIGO ENTIDAD", _MAX["entity_code"]),
("PLACAS", _MAX["plate_number"]),
("ESTADO", _MAX["state"]),
("PAIS", _MAX["country"]),
("CLAVE CONTENEDOR", _MAX["container_key"]),
]:
val = (row.get(col) or "").strip()
if val and len(val) > max_len:
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
return None
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
"""Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row."""
trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"])
if not trailer_number:
return {}
return {
"trailer_number": trailer_number,
"ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]),
"trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]),
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
"entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]),
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
"container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]),
}
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"Trailers import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.reader(f_in, dialect=dialect)
try:
headers = next(reader)
except StopIteration:
headers = []
headers = _dedupe_headers(headers)
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
for i, row in enumerate(dict_reader, start=1):
if progress_callback and i % 500 == 0:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_trailer(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"Trailers import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Trailers import: failed to store error lines: {e}")
result = {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
return result
def run_scan_sync(job_id: str) -> Dict[str, Any]:
result = _do_scan(job_id, progress_callback=None)
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
return result
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
logger.info(f"Trailers import: starting scan for job {job_id}")
def on_progress(current: int, total: int, errors: int) -> None:
self.update_state(
state="PROGRESS",
meta={"current": current, "total": total, "errors": errors},
)
result = _do_scan(job_id, progress_callback=on_progress)
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
return result
def _do_commit(job_id: str) -> Dict[str, Any]:
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"Trailers import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.transportation.trailers.services import TrailerService
from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO
inserted_count = 0
updated_count = 0
skipped_invalid = 0
skipped_duplicate = 0
skipped_details: List[Dict[str, Any]] = []
seen_keys_in_file: Dict[str, int] = {}
try:
with CoreSessionLocal() as session:
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.reader(f, dialect=dialect)
try:
headers = next(reader)
except StopIteration:
headers = []
headers = _dedupe_headers(headers)
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
for i, row in enumerate(dict_reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_trailer(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
"invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
data = _row_to_trailer_dto(row_norm, tenant_id, company_id)
if not data or not data.get("trailer_number"):
skipped_invalid += 1
continue
tn = data["trailer_number"]
if tn in seen_keys_in_file:
skipped_duplicate += 1
skipped_details.append(
{
"line": i,
"trailer_number": tn,
"invoice": tn,
"reason": "Clave duplicada en el archivo (se usa la primera)",
}
)
continue
seen_keys_in_file[tn] = i
existing = TrailerService.get_by_id(session, tn, tenant_id, company_id)
try:
if existing:
update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"})
TrailerService.update(session, tn, tenant_id, update_data, company_id)
updated_count += 1
else:
create_data = TrailerCreateDTO(**data)
TrailerService.create(session, create_data, tenant_id, company_id)
inserted_count += 1
except Exception as db_err:
session.rollback()
skipped_invalid += 1
skipped_details.append(
{"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)}
)
continue
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"Trailers import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
except Exception as e:
logger.error(f"Trailers import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"Trailers import cleanup failed: {cleanup_err}")
total_ok = inserted_count + updated_count
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
return {
"status": "warning",
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
}
if total_ok == 0:
return {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"updated": 0,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return {
"status": "finished",
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
def run_commit_sync(job_id: str) -> Dict[str, Any]:
result = _do_commit(job_id)
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
return result
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
logger.info(f"Trailers import: starting commit for job {job_id}")
result = _do_commit(job_id)
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
return result

View File

@@ -0,0 +1,45 @@
"""
Configuración de plantilla CSV para Trailers / Cajas (EstructuraCatTrailers.xls).
Mapeo: NUMERO TRAILER → trailer_number, CLAVE ACE → ace_trailer_number, etc.
"""
from typing import Dict, List, Any
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"trailers": [
{"canonical": "NUMERO TRAILER", "aliases": ["CLAVE TRAILER", "TRAILER NUMBER", "TRAILER", "NUMERO"]},
{"canonical": "CLAVE ACE", "aliases": ["ACE", "NUMERO ACE", "ACE TRAILER"]},
{"canonical": "TIPO TRAILER", "aliases": ["TIPO", "TRAILER TYPE", "TIPO CAJA"]},
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
{"canonical": "CODIGO ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE", "CODIGO DE ENTIDAD"]},
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE NUMBER", "PLATE"]},
{"canonical": "ESTADO", "aliases": ["STATE"]},
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
{"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]},
],
}
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
cols = TEMPLATE_COLUMNS.get("trailers")
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
lookup = build_normalized_lookup(normalize_header_fn)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -1,11 +1,19 @@
from fastapi import APIRouter
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO
from .services import TrailerService
from .imports.routes import router as imports_router
# Create router using TenantCRUDRoutes factory
# Note: trailer_number is a string (not int) and is used as the primary key
router = TenantCRUDRoutes(
# Main router: trailers CRUD + CSV imports
router = APIRouter()
# CSV import (upload → scan → status → commit)
router.include_router(imports_router, prefix="/trailers/imports", tags=["a76 / trailers / csv_import"])
# CRUD routes
crud_router = TenantCRUDRoutes(
service=TrailerService,
create_schema=TrailerCreateDTO,
update_schema=TrailerUpdateDTO,
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
prefix="/trailers",
tags=[],
resource_name="Trailer",
id_name="trailer_number", # Using trailer_number instead of numeric ID
id_type=str, # Specify that the ID is a string
enable_list=True, # Enable GET /trailers with pagination
enable_filters=True, # Enable filtering by plate_number and trailer_type_key
id_name="trailer_number",
id_type=str,
enable_list=True,
enable_filters=True,
default_page_size=50,
max_page_size=100,
).router
router.include_router(crud_router)

View File

@@ -0,0 +1 @@
# CSV import for Vehicles / Transportes (upload → scan → status → commit).

View File

@@ -0,0 +1,192 @@
"""
Rutas de importación CSV para Vehículos (Transportes).
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
import threading
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
from sqlalchemy.orm import Session
from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
run_scan_sync,
run_commit_sync,
VEHL_IMPORT_FILE_PREFIX,
VEHL_IMPORT_META_PREFIX,
VEHL_IMPORT_STATUS_PREFIX,
VEHL_IMPORT_REDIS_TTL,
)
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload", response_model=ImportJobResponse)
async def upload_import_file(
file: UploadFile = File(...),
company_id: int = Query(..., description="Company ID"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
try:
tenant_id = validate_access_to_resource(db, company_id, current_user)
except Exception as e:
logger.error(f"Vehicles import: access validation failed: {e}")
raise HTTPException(status_code=403, detail="Invalid company access")
if not file.filename or not file.filename.lower().endswith(".csv"):
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
job_id = str(uuid4())
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"template_id": "vehicles",
}
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=VEHL_IMPORT_REDIS_TTL,
)
r.set(
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Vehicles import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Vehicles import: local file save failed: {e}")
scan_file.apply_async(args=[job_id], task_id=job_id)
# Fallback sin worker: ejecutar scan en un hilo y guardar resultado en Redis
def run_scan_background():
try:
run_scan_sync(job_id)
except Exception as e:
logger.exception(f"Vehicles import: background scan failed: {e}")
threading.Thread(target=run_scan_background, daemon=True).start()
return ImportJobResponse(
job_id=job_id,
status="queued",
message="Archivo subido. Escaneo iniciado.",
)
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
# Si el resultado del scan está en Redis (worker o hilo de fallback), usarlo
try:
r = _get_redis()
raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
return data
except Exception as e:
logger.debug(f"Vehicles import: could not read status from Redis: {e}")
task_result = celery_app.AsyncResult(job_id)
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
if task_result.state == "PROGRESS":
info = (task_result.info or {})
return {
"status": "processing",
"progress": info.get("current", 0),
"total": info.get("total", 0),
}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg and result is not None:
if not isinstance(result, dict):
err_msg = str(result)
elif result.get("error") or result.get("message"):
err_msg = result.get("error") or result.get("message")
return {"status": "failed", "error": err_msg or "Task failed"}
@router.post("/{job_id}/commit")
async def commit_import_job(job_id: str):
# Escribir "processing" en Redis para que el polling muestre estado hasta que termine el commit
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.debug(f"Vehicles import: could not write processing status: {e}")
# Ejecutar commit en hilo (mismo fallback que el scan: no depende del worker de Celery)
def run_commit_background():
try:
run_commit_sync(job_id)
except Exception as e:
logger.exception(f"Vehicles import: background commit failed: {e}")
threading.Thread(target=run_commit_background, daemon=True).start()
# Mismo job_id para que el front siga haciendo polling y reciba el resultado de Redis
return {
"status": "committing",
"message": "Inserción iniciada.",
"commit_job_id": job_id,
}

View File

@@ -0,0 +1,22 @@
from pydantic import BaseModel
from typing import Optional
class ImportJobResponse(BaseModel):
job_id: str
status: str
message: str
class ImportJobStatus(BaseModel):
status: str
job_id: str
total_rows: Optional[int] = 0
error_count: Optional[int] = 0
valid_rows: Optional[int] = 0
error: Optional[str] = None
inserted: Optional[int] = 0
skipped_invalid: Optional[int] = 0
skipped_missing_fk: Optional[int] = 0
skipped_duplicate: Optional[int] = 0
skipped_details: Optional[list] = None

View File

@@ -0,0 +1,607 @@
"""
Tareas Celery para importación CSV de Vehículos (Transportes).
Flujo: scan_file (validación) → insert_valid_rows (commit).
Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key.
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from decimal import Decimal
from typing import Dict, Any, Optional, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from .template_config import row_from_template
logger = logging.getLogger(__name__)
VEHL_IMPORT_FILE_PREFIX = "veh_import_file:"
VEHL_IMPORT_META_PREFIX = "veh_import_meta:"
VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:"
VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:"
VEHL_IMPORT_REDIS_TTL = 3600
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return os.path.join(os.getcwd(), "uploads", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"Vehicles import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"veh_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"Vehicles import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
# Max lengths from Vehicle model (a76.vehicle)
_MAX = {
"vehicle_key": 14,
"ace_vehicle_key": 10,
"transporter_key": 23,
"transport_identifier": 30,
"transport_type": 2,
"entity_code": 1,
"transponder_number": 16,
"dot_number": 8,
"plate_number": 17,
"city": 30,
"state": 30,
"country": 3,
"seal": 49,
"insurance_company_name": 30,
"insurance_number": 20,
"series": 30,
}
def _parse_decimal(val: Any) -> Optional[Decimal]:
if val is None or (isinstance(val, str) and not val.strip()):
return None
try:
s = str(val).strip().replace(",", "")
return Decimal(s)
except Exception:
return None
def _parse_insurance_date(val: Any) -> Optional[int]:
"""Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD."""
if val is None or (isinstance(val, str) and not val.strip()):
return None
s = str(val).strip()
if not s:
return None
# Already integer-like
if re.match(r"^\d{8}$", s):
try:
return int(s)
except ValueError:
pass
# Try DD/MM/YYYY or similar
for sep in ["/", "-", "."]:
if sep in s:
parts = s.split(sep)
if len(parts) == 3:
try:
a, b, c = [p.strip() for p in parts]
if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year
return int(c) * 10000 + int(b) * 100 + int(a)
if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year
return int(a) * 10000 + int(b) * 100 + int(c)
except (ValueError, TypeError):
pass
break
return None
def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
"""Valida una fila para Vehículo. Retorna error dict o None."""
clave = (row.get("CLAVE") or "").strip()
if not clave:
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
if len(clave) > _MAX["vehicle_key"]:
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"}
# Optional fields: max lengths only
for col, max_len in [
("CLAVE ACE", _MAX["ace_vehicle_key"]),
("CLAVE TRANSPORTE", _MAX["transporter_key"]),
("VIN", _MAX["series"]),
("TIPO TRANSPORTE", _MAX["transport_type"]),
("CODIGO DE ENTIDAD", _MAX["entity_code"]),
("TRANSPONDEDOR", _MAX["transponder_number"]),
("NUMERO DOT", _MAX["dot_number"]),
("PLACAS", _MAX["plate_number"]),
("CIUDAD", _MAX["city"]),
("ESTADO", _MAX["state"]),
("PAIS", _MAX["country"]),
("PRECINTO", _MAX["seal"]),
("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]),
("NUM. ASEGURADORA", _MAX["insurance_number"]),
]:
val = (row.get(col) or "").strip()
if val and len(val) > max_len:
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
# MONTO ASEGURADO: must be numeric if present
monto = row.get("MONTO ASEGURADO") or row.get("MONTO")
if monto is not None and str(monto).strip():
if _parse_decimal(monto) is None:
return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"}
# FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None)
# Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either
# reject or set None. We reject invalid date to keep data quality.
fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")
if fecha is not None and str(fecha).strip():
if _parse_insurance_date(fecha) is None:
return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
return None
def _do_scan(
job_id: str,
progress_callback: Optional[Any] = None,
) -> Dict[str, Any]:
"""
Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync.
progress_callback(current, total, error_count) opcional.
"""
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"Vehicles import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
if progress_callback and i % 500 == 0:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_vehicle(row_norm, i)
if err:
error_count += 1
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
processed_rows += 1
except Exception as e:
logger.error(f"Vehicles import scan failed: {e}")
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}")
result = {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
return result
def run_scan_sync(job_id: str) -> Dict[str, Any]:
"""
Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis.
Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery.
"""
result = _do_scan(job_id, progress_callback=None)
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
return result
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
"""
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
"""
logger.info(f"Vehicles import: starting scan for job {job_id}")
def on_progress(current: int, total: int, errors: int) -> None:
self.update_state(
state="PROGRESS",
meta={"current": current, "total": total, "errors": errors},
)
result = _do_scan(job_id, progress_callback=on_progress)
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
return result
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]:
"""Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row."""
vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"])
if not vehicle_key:
return {}
data = {
"vehicle_key": vehicle_key,
"ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]),
"transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]),
"series": _str_or_none(row.get("VIN"), _MAX["series"]),
"transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]),
"entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]),
"transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]),
"dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]),
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
"city": _str_or_none(row.get("CIUDAD"), _MAX["city"]),
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
"insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]),
"insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]),
"insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")),
"insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")),
}
return data
def _do_commit(job_id: str) -> Dict[str, Any]:
"""
Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync.
"""
file_path = _ensure_worker_has_file_from_redis(job_id)
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv")
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
base_dir = os.path.dirname(file_path)
error_dir = base_dir.replace("temp", "errors")
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"Vehicles import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
from api.v1.modules.a76.transportation.vehicles.services import VehicleService
from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO
inserted_count = 0
updated_count = 0
skipped_invalid = 0
skipped_duplicate = 0
skipped_details: List[Dict[str, Any]] = []
seen_keys_in_file: Dict[str, int] = {}
try:
with CoreSessionLocal() as session:
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_vehicle(row_norm, i)
if err:
skipped_invalid += 1
skipped_details.append(
{
"line": i,
"vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
"invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
}
)
continue
data = _row_to_vehicle_dto(row_norm)
if not data or not data.get("vehicle_key"):
skipped_invalid += 1
continue
vk = data["vehicle_key"]
if vk in seen_keys_in_file:
skipped_duplicate += 1
skipped_details.append(
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"}
)
continue
seen_keys_in_file[vk] = i
existing = VehicleService.get_by_id(session, vk, tenant_id, company_id)
try:
if existing:
update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"})
VehicleService.update(session, vk, tenant_id, update_data, company_id)
updated_count += 1
else:
create_data = VehicleCreateDTO(**data)
VehicleService.create(session, create_data, tenant_id, company_id)
inserted_count += 1
except Exception as db_err:
session.rollback()
skipped_invalid += 1
skipped_details.append(
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)}
)
continue
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"Vehicles import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
except Exception as e:
logger.error(f"Vehicles import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"Vehicles import cleanup failed: {cleanup_err}")
total_ok = inserted_count + updated_count
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
return {
"status": "warning",
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
}
if total_ok == 0:
return {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"updated": 0,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
return {
"status": "finished",
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_duplicate,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
def run_commit_sync(job_id: str) -> Dict[str, Any]:
"""
Ejecuta el commit en el proceso actual y guarda el resultado en Redis.
Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery.
"""
result = _do_commit(job_id)
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
return result
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
"""
Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService.
"""
logger.info(f"Vehicles import: starting commit for job {job_id}")
result = _do_commit(job_id)
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
json.dumps(result).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
return result

View File

@@ -0,0 +1,53 @@
"""
Configuración de plantilla CSV para Vehículos / Transportes (EstructuraCatTransportes.xls).
Mapeo: CLAVE → vehicle_key, CLAVE ACE → ace_vehicle_key, etc.
"""
from typing import Dict, List, Any
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
"vehicles": [
{"canonical": "CLAVE", "aliases": ["CLAVE VEHICULO", "VEHICLE KEY", "KEY"]},
{"canonical": "CLAVE ACE", "aliases": ["ACE", "CLAVEACE"]},
{"canonical": "CLAVE TRANSPORTE", "aliases": ["TRANSPORTE", "TRANSPORTISTA"]},
{"canonical": "VIN", "aliases": ["SERIE", "SERIES", "NUMERO SERIE"]},
{"canonical": "TIPO TRANSPORTE", "aliases": ["TIPO", "TRANSPORT TYPE"]},
{"canonical": "CODIGO DE ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE"]},
{"canonical": "TRANSPONDEDOR", "aliases": ["TRANSPONDER", "TRANSPONDER NUMBER"]},
{"canonical": "NUMERO DOT", "aliases": ["DOT", "NUM DOT"]},
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE", "PLATE NUMBER"]},
{"canonical": "CIUDAD", "aliases": ["CITY"]},
{"canonical": "ESTADO", "aliases": ["STATE"]},
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
{"canonical": "EMPRESA ASEGURADORA", "aliases": ["ASEGURADORA", "INSURANCE COMPANY"]},
{"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]},
{"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]},
{"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]},
],
}
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
cols = TEMPLATE_COLUMNS.get("vehicles")
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
lookup = build_normalized_lookup(normalize_header_fn)
if not lookup:
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

View File

@@ -1,11 +1,19 @@
from fastapi import APIRouter
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO
from .services import VehicleService
from .imports.routes import router as imports_router
# Create router using TenantCRUDRoutes factory
# Note: vehicle_key is a string (not int) and is used as the primary key
router = TenantCRUDRoutes(
# Main router: vehicles CRUD + CSV imports
router = APIRouter()
# CSV import (upload → scan → status → commit)
router.include_router(imports_router, prefix="/vehicles/imports", tags=["a76 / vehicles / csv_import"])
# CRUD routes
crud_router = TenantCRUDRoutes(
service=VehicleService,
create_schema=VehicleCreateDTO,
update_schema=VehicleUpdateDTO,
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
prefix="/vehicles",
tags=[],
resource_name="Vehicle",
id_name="vehicle_key", # Using vehicle_key instead of numeric ID
id_type=str, # Specify that the ID is a string
enable_list=True, # Enable GET /vehicles with pagination
enable_filters=True, # Enable filtering by plate_number and transport_type
id_name="vehicle_key",
id_type=str,
enable_list=True,
enable_filters=True,
default_page_size=50,
max_page_size=100,
).router
router.include_router(crud_router)

View File

@@ -1,6 +1,13 @@
import os
from celery import Celery
# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen (mapper)
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
CodePedimentoRegimen,
)
# Import models in correct order for SQLAlchemy relationship resolution
# CRITICAL: FaLineItem must be imported BEFORE LineItem
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem # noqa: F401
@@ -26,6 +33,13 @@ celery_app.conf.update(
"api.v1.modules.a76.reports.movements.invoices.tasks",
"api.v1.modules.a76.reports.exportacion.descargo.task",
"api.v1.modules.a76.imports.tasks",
"api.v1.modules.a76.customs_brokers.imports.tasks",
"api.v1.modules.a76.clients_and_providers.imports.tasks",
"api.v1.modules.a76.pedmientos.imports.tasks",
"api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks",
"api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks",
"api.v1.modules.a76.classes.imports.tasks",
"api.v1.modules.a76.transportation.vehicles.imports.tasks",
"api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task",
"api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task",
"api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task",

View File

@@ -11,11 +11,23 @@ from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from .config import settings
from .exceptions import BaseAPIException
logger = logging.getLogger(__name__)
def _cors_headers(request: Request) -> Dict[str, str]:
"""CORS headers for error responses so browser does not block on 4xx/5xx."""
origin = request.headers.get("origin")
if not origin or origin not in settings.cors_origins_list:
return {}
return {
"Access-Control-Allow-Origin": origin,
"Access-Control-Allow-Credentials": "true",
}
async def base_exception_handler(
request: Request,
exc: BaseAPIException,
@@ -36,10 +48,13 @@ async def base_exception_handler(
if hasattr(exc, "errors") and exc.errors:
logger.warning(f"Validation errors details: {exc.errors}")
return JSONResponse(
response = JSONResponse(
status_code=exc.status_code,
content=jsonable_encoder(exc.to_dict()),
)
for k, v in _cors_headers(request).items():
response.headers[k] = v
return response
async def validation_exception_handler(
@@ -65,7 +80,7 @@ async def validation_exception_handler(
extra={"errors": errors},
)
return JSONResponse(
response = JSONResponse(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
content={
"error": "VALIDATION_ERROR",
@@ -74,6 +89,9 @@ async def validation_exception_handler(
"errors": errors,
},
)
for k, v in _cors_headers(request).items():
response.headers[k] = v
return response
async def integrity_error_handler(
@@ -105,7 +123,7 @@ async def integrity_error_handler(
else:
error_message = "Error de integridad en la base de datos"
return JSONResponse(
response = JSONResponse(
status_code=status.HTTP_409_CONFLICT,
content={
"error": "DATABASE_INTEGRITY_ERROR",
@@ -113,6 +131,9 @@ async def integrity_error_handler(
"status_code": status.HTTP_409_CONFLICT,
},
)
for k, v in _cors_headers(request).items():
response.headers[k] = v
return response
async def sqlalchemy_error_handler(
@@ -131,7 +152,7 @@ async def sqlalchemy_error_handler(
exc_info=True,
)
return JSONResponse(
response = JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={
"error": "DATABASE_ERROR",
@@ -139,6 +160,9 @@ async def sqlalchemy_error_handler(
"status_code": status.HTTP_500_INTERNAL_SERVER_ERROR,
},
)
for k, v in _cors_headers(request).items():
response.headers[k] = v
return response
async def http_exception_handler(
@@ -174,7 +198,7 @@ async def general_exception_handler(
exc_info=True,
)
return JSONResponse(
response = JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={
"error": "INTERNAL_SERVER_ERROR",
@@ -182,6 +206,9 @@ async def general_exception_handler(
"status_code": status.HTTP_500_INTERNAL_SERVER_ERROR,
},
)
for k, v in _cors_headers(request).items():
response.headers[k] = v
return response
def register_exception_handlers(app) -> None:

View File

@@ -17,8 +17,13 @@ from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType
from api.v1.modules.public.reference_data.material_types.models import MaterialType
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen para que
# SQLAlchemy resuelva los nombres en relationship() al configurar el mapper
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
CodePedimentoRegimen,
)
from api.v1.modules.public.reference_data.sectors.models import Sector
from api.v1.modules.public.reference_data.states.models import State
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode
@@ -134,6 +139,20 @@ logger = logging.getLogger(__name__)
register_exception_handlers(app)
def _cors_headers_for_request(request: Request):
"""Return CORS headers if request Origin is allowed (so error responses don't get blocked by browser)."""
origin = request.headers.get("origin")
if not origin:
return {}
allowed = settings.cors_origins_list
if origin in allowed:
return {
"Access-Control-Allow-Origin": origin,
"Access-Control-Allow-Credentials": "true",
}
return {}
# Add validation error handler
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
@@ -141,12 +160,29 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
f"Validation error for {request.method} {request.url.path}: {exc.errors()}"
)
logger.error(f"Request body: {await request.body()}")
return JSONResponse(
response = JSONResponse(
status_code=status.HTTP_400_BAD_REQUEST,
content={"detail": exc.errors(), "body": exc.body},
)
for k, v in _cors_headers_for_request(request).items():
response.headers[k] = v
return response
# Add HTTP exception handler
@app.exception_handler(HTTPException)
async def http_exception_handler(request: Request, exc: HTTPException):
logger.error(
f"HTTP {exc.status_code} for {request.method} {request.url.path}: {exc.detail}"
)
response = JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail},
)
for k, v in _cors_headers_for_request(request).items():
response.headers[k] = v
return response
def run_migrations():
subprocess.run(["alembic", "upgrade", "head"], check=True)

View File

@@ -64,5 +64,6 @@
"lucide-svelte": "^0.553.0",
"marked": "^12.0.0",
"svelte-sonner": "^1.0.7"
}
}
},
"packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017"
}

View File

@@ -341,6 +341,173 @@ export const api = {
validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`)
},
imports: {
upload: (
file: File,
modelTarget: string,
footerConfig: any,
companyId: number,
operationType: string,
templateId?: string
) => {
const formData = new FormData();
formData.append('file', file);
if (footerConfig) {
formData.append('footer_config', JSON.stringify(footerConfig));
}
if (templateId) {
formData.append('template_id', templateId);
}
const queryParams = new URLSearchParams({
company_id: String(companyId),
operation_type: operationType || 'imp'
}).toString();
return fetchApi(`/v1/a76/imports/upload/${modelTarget}?${queryParams}`, {
method: 'POST',
body: formData
});
},
status: (jobId: string) => api.get(`/v1/a76/imports/${jobId}/status`),
commit: (jobId: string, modelTarget: string) =>
api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget })
},
// CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports)
customsBrokerImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/customs-brokers/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/customs-brokers/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {})
},
// CSV import for Clientes y Proveedores (flujo propio en clients_and_providers/imports)
clientProviderImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/clients-providers/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/clients-providers/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {})
},
// CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports)
exchangeRateImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/exchange-rate/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {})
},
// CSV import for Fracción Americana (us_tariff_fractions/imports)
americanFractionImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/us-tariff-fractions/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/us-tariff-fractions/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {})
},
// CSV import for Pedimentos (pedimentos/imports)
pedimentosImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/pedimentos/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/pedimentos/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {})
},
// CSV import for Clases de Materiales (classes/imports)
materialClassImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/classes/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/classes/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/classes/imports/${jobId}/commit`, {})
},
// CSV import for Vehículos / Transportes (transportation/vehicles/imports)
vehicleImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/transportation/vehicles/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/transportation/vehicles/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/transportation/vehicles/imports/${jobId}/commit`, {})
},
// CSV import for Conductores (drivers/imports)
driverImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/drivers/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/drivers/imports/${jobId}/status`),
commit: (jobId: string) => api.post(`/v1/a76/drivers/imports/${jobId}/commit`, {})
},
// CSV import for Trailers y Cajas (transportation/trailers/imports)
trailerImports: {
upload: (file: File, companyId: number) => {
const formData = new FormData();
formData.append('file', file);
return fetchApi(
`/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}`,
{ method: 'POST', body: formData }
);
},
status: (jobId: string) => api.get(`/v1/a76/transportation/trailers/imports/${jobId}/status`),
commit: (jobId: string) =>
api.post(`/v1/a76/transportation/trailers/imports/${jobId}/commit`, {})
},
// Generic request for custom needs (like file uploads)
request: <T = any>(endpoint: string, options: RequestInit = {}) => fetchApi<T>(endpoint, options)
};

View File

@@ -143,17 +143,53 @@
{#if scanResults.error_count > 0}
<div
class="rounded-md bg-destructive/5 border border-destructive/10 p-4 flex items-start gap-3"
class="rounded-md bg-destructive/5 border border-destructive/10 p-4 flex items-start gap-3 mb-4"
>
<XCircle class="w-5 h-5 text-destructive mt-0.5 shrink-0" />
<div class="text-sm text-destructive-foreground/90">
<p class="font-semibold mb-1">Se detectaron problemas en el archivo</p>
<p>
Las filas con errores serán omitidas automáticamente. Solo se importarán los
registros válidos.
Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para
importar solo las filas válidas (las erróneas se omitirán).
</p>
</div>
</div>
{#if scanResults.errors && scanResults.errors.length > 0}
<div class="border rounded-lg overflow-hidden shadow-sm">
<div class="bg-muted/50 px-4 py-2 border-b flex justify-between items-center">
<h5 class="text-xs font-bold text-foreground uppercase tracking-wide">
Detalle de errores (para corregir en el CSV)
</h5>
<span
class="text-[10px] bg-secondary text-secondary-foreground px-2 py-0.5 rounded-full border"
>
{scanResults.errors.length} error(es)
</span>
</div>
<div class="max-h-60 overflow-y-auto bg-card relative">
<table class="w-full text-xs text-left">
<thead
class="text-muted-foreground font-medium bg-muted/30 sticky top-0 z-10 shadow-sm backdrop-blur-sm"
>
<tr>
<th class="px-4 py-2 w-16">Línea</th>
<th class="px-4 py-2 w-40">Columna</th>
<th class="px-4 py-2">Mensaje</th>
</tr>
</thead>
<tbody class="divide-y">
{#each scanResults.errors as err}
<tr class="hover:bg-muted/30 transition-colors">
<td class="px-4 py-2 font-mono text-muted-foreground">{err.line}</td>
<td class="px-4 py-2 font-mono font-medium text-foreground">{err.col || '-'}</td>
<td class="px-4 py-2 text-destructive">{err.msg || '-'}</td>
</tr>
{/each}
</tbody>
</table>
</div>
</div>
{/if}
{:else}
<div class="rounded-md bg-primary/5 border border-primary/10 p-4 flex items-start gap-3">
<CheckCircle2 class="w-5 h-5 text-primary mt-0.5 shrink-0" />

View File

@@ -123,7 +123,7 @@
ondragover={(e) => handleDragOver(e, item.disabled)}
ondrop={(e) => handleDrop(e, item)}
oncontextmenu={(e) => handleContextMenu(e, item)}
roles="button"
role="button"
tabindex={item.disabled ? -1 : 0}
onclick={() => handleClick(item.id, item.disabled)}
onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)}

View File

@@ -8,6 +8,20 @@ export type { CustomsBroker };
export function createColumns(onSuccess?: () => void): ColumnDef<CustomsBroker>[] {
return [
{
accessorKey: "id",
header: "ID",
cell: ({ row }) => {
const idSnippet = createRawSnippet<[{ id: number }]>((getId) => {
const { id } = getId();
return {
render: () =>
`<code class="relative rounded bg-muted px-[0.3rem] py-[0.2rem] font-mono text-sm">${id}</code>`
};
});
return renderSnippet(idSnippet, { id: row.original.id });
}
},
{
accessorKey: "broker_key",
header: "Clave",

View File

@@ -249,7 +249,7 @@ export const importacionConfig: CsvUploadItem[] = [
title: 'Encabezado',
icon: FileText,
group: 'Impo. Def.',
modelTarget: 'InvoiceHeader',
modelTarget: 'invoice_header',
templateUrl: '/csv/EstructuraEncFacImpoDef.xls'
},
{
@@ -257,7 +257,7 @@ export const importacionConfig: CsvUploadItem[] = [
title: 'Partidas',
icon: Package,
group: 'Impo. Def.',
modelTarget: 'InvoiceSalesDetails',
modelTarget: 'invoice_details',
templateUrl: '/csv/EstructuraParFacImpoDefAF.xls'
},
{
@@ -274,7 +274,7 @@ export const importacionConfig: CsvUploadItem[] = [
title: 'Encabezado',
icon: FileText,
group: 'Compras Mex.',
modelTarget: 'InvoiceHeader',
modelTarget: 'invoice_header',
disabled: true,
},
{
@@ -282,7 +282,7 @@ export const importacionConfig: CsvUploadItem[] = [
title: 'Partidas',
icon: Package,
group: 'Compras Mex.',
modelTarget: 'InvoiceSalesDetails',
modelTarget: 'invoice_details',
disabled: true,
},
{
@@ -302,7 +302,7 @@ export const exportacionConfig: CsvUploadItem[] = [
title: 'Encabezado',
icon: FileText,
group: 'Expo. Def./Cam. Reg.',
modelTarget: 'InvoiceHeader',
modelTarget: 'invoice_header',
templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls'
},
{
@@ -310,7 +310,7 @@ export const exportacionConfig: CsvUploadItem[] = [
title: 'Partidas',
icon: Package,
group: 'Expo. Def./Cam. Reg.',
modelTarget: 'InvoiceSalesDetails',
modelTarget: 'invoice_details',
templateUrl: '/csv/EstructuraParExpoCamReg.xls'
},
{

View File

@@ -1,209 +1,474 @@
<script lang="ts">
import * as Tabs from '$lib/components/ui/tabs/index.js';
import UploadLauncherGrid from '$lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte';
import ConfigFooter from '$lib/components/dashboard/csv-upload/ConfigFooter.svelte';
import ProcessingResultModal from '$lib/components/dashboard/csv-upload/ProcessingResultModal.svelte';
import {
catalogosConfig,
transportesConfig,
importacionConfig,
exportacionConfig,
tabSettings,
type CsvUploadItem
} from '$lib/config/csv-upload';
import { api } from '$lib/api';
import { toast } from 'svelte-sonner';
import { companyStore } from '$lib/stores/company.svelte';
// We no longer need modal state
let activeTab = $state('catalogos');
let isUploading = $state(false);
let currentJobId = $state<string | null>(null);
let activeModelTarget = $state<string | null>(null);
let scanResults = $state<any>(null);
let commitResults = $state<any>(null);
let showResultModal = $state(false);
// Initialize settings for all tabs upfront to avoid reactivity loops
let allSettings = $state<Record<string, any>>(() => {
const initial: Record<string, any> = {};
for (const tab in tabSettings) {
initial[tab] = {};
tabSettings[tab].forEach((f) => {
initial[tab][f.name] = f.defaultValue;
});
}
return initial;
});
async function handleUpload(file: File, config: CsvUploadItem) {
isUploading = true;
activeModelTarget = config.modelTarget || null;
scanResults = null;
const currentSettings = allSettings[activeTab] || {};
const companyId = companyStore.activeCompany?.id || 1;
const opType = activeTab === 'exportacion' ? 'exp' : 'imp';
const res = await api.imports.upload(
file,
config.modelTarget || '',
currentSettings,
companyId,
opType
);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error('Error al subir el archivo');
isUploading = false;
}
}
async function pollStatus() {
if (!currentJobId) return;
const res = await api.imports.status(currentJobId);
if (res.data?.status === 'waiting_confirmation') {
scanResults = res.data;
showResultModal = true;
toast.success('Escaneo completado. Revisa los resultados.');
isUploading = false;
} else if (res.data?.status === 'failed') {
toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido'));
isUploading = false;
currentJobId = null;
scanResults = null;
commitResults = null;
showResultModal = false;
} else if (res.data?.status === 'warning') {
// Caso cuando no se insertaron registros pero hay información de rechazo
commitResults = res.data;
showResultModal = true;
const inserted = res.data?.inserted || 0;
const skippedInvalid = res.data?.skipped_invalid || 0;
const skippedFk = res.data?.skipped_missing_fk || 0;
const totalSkipped = skippedInvalid + skippedFk;
if (inserted === 0) {
toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`);
} else {
toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`);
}
isUploading = false;
} else if (res.data?.status === 'finished') {
commitResults = res.data;
showResultModal = true;
const inserted = res.data?.inserted || 0;
const skippedInvalid = res.data?.skipped_invalid || 0;
const skippedFk = res.data?.skipped_missing_fk || 0;
const skippedDetails = res.data?.skipped_details || [];
if (inserted > 0) {
toast.success(`Importación completada: ${inserted} registros insertados`);
if (skippedInvalid > 0 || skippedFk > 0) {
const totalSkipped = skippedInvalid + skippedFk;
toast.warning(`${totalSkipped} registros fueron rechazados`);
}
} else {
toast.error('No se insertaron registros. Revisa los errores a continuación.');
}
isUploading = false;
} else {
// Continue polling
setTimeout(pollStatus, 2000);
}
}
</script>
<div class="flex flex-col h-[calc(100vh-4rem)] -m-4 overflow-hidden">
<!-- Scrollable Content Area -->
<div class="flex-1 overflow-y-auto p-4 md:p-8 space-y-4">
<div class="flex items-center gap-4">
<h1 class="text-lg font-semibold md:text-2xl">Importación Masiva de Datos (CSV)</h1>
</div>
<Tabs.Root bind:value={activeTab} class="w-full">
<Tabs.List class="grid w-full grid-cols-2 md:grid-cols-4 lg:w-auto">
<Tabs.Trigger value="catalogos">Catálogos</Tabs.Trigger>
<Tabs.Trigger value="transportes">Transportes</Tabs.Trigger>
<Tabs.Trigger value="importacion">Importación</Tabs.Trigger>
<Tabs.Trigger value="exportacion">Exportación</Tabs.Trigger>
</Tabs.List>
<div class="mt-6">
<Tabs.Content value="catalogos" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Catálogos Generales</h2>
</div>
<UploadLauncherGrid items={catalogosConfig} onUpload={handleUpload} />
</Tabs.Content>
<Tabs.Content value="transportes" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Logística y Transporte</h2>
</div>
<UploadLauncherGrid items={transportesConfig} onUpload={handleUpload} />
</Tabs.Content>
<Tabs.Content value="importacion" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Operaciones de Importación</h2>
</div>
<UploadLauncherGrid items={importacionConfig} onUpload={handleUpload} />
</Tabs.Content>
<Tabs.Content value="exportacion" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Operaciones de Exportación</h2>
</div>
<UploadLauncherGrid items={exportacionConfig} onUpload={handleUpload} />
</Tabs.Content>
</div>
</Tabs.Root>
<div class="h-4"></div>
</div>
<!-- Fixed Footer Area -->
{#if allSettings[activeTab]}
<div class="flex-none z-20">
<ConfigFooter {activeTab} bind:settings={allSettings[activeTab]} />
</div>
{/if}
</div>
<ProcessingResultModal
bind:open={showResultModal}
{scanResults}
{commitResults}
{isUploading}
onConfirm={async () => {
if (currentJobId && activeModelTarget) {
try {
isUploading = true;
const res = await api.imports.commit(currentJobId, activeModelTarget);
if (res.data?.commit_job_id) {
currentJobId = res.data.commit_job_id;
pollStatus();
}
} catch (err) {
toast.error('Error al iniciar la importación');
isUploading = false;
}
}
}}
onCancel={() => {
currentJobId = null;
scanResults = null;
commitResults = null;
showResultModal = false;
}}
onClose={() => {
currentJobId = null;
scanResults = null;
commitResults = null;
showResultModal = false;
}}
/>
<script lang="ts">
import * as Tabs from '$lib/components/ui/tabs/index.js';
import UploadLauncherGrid from '$lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte';
import ConfigFooter from '$lib/components/dashboard/csv-upload/ConfigFooter.svelte';
import ProcessingResultModal from '$lib/components/dashboard/csv-upload/ProcessingResultModal.svelte';
import {
catalogosConfig,
transportesConfig,
importacionConfig,
exportacionConfig,
tabSettings,
type CsvUploadItem
} from '$lib/config/csv-upload';
import { api } from '$lib/api';
import { toast } from 'svelte-sonner';
import { companyStore } from '$lib/stores/company.svelte';
// We no longer need modal state
let activeTab = $state('catalogos');
let isUploading = $state(false);
let currentJobId = $state<string | null>(null);
let activeModelTarget = $state<string | null>(null);
let scanResults = $state<any>(null);
let commitResults = $state<any>(null);
let showResultModal = $state(false);
// Cuando es true, usamos API de importación de Agentes Aduanales (customs_brokers/imports)
let useCustomsBrokerImport = $state(false);
// Cuando es true, usamos API de importación de Clientes y Proveedores (clients_and_providers/imports)
let useClientProviderImport = $state(false);
// Cuando es true, usamos API de importación de Tipos de Cambio (exchange_rate/imports)
let useExchangeRateImport = $state(false);
// Cuando es true, usamos API de importación de Fracción Americana (us_tariff_fractions/imports)
let useAmericanFractionImport = $state(false);
// Cuando es true, usamos API de importación de Pedimentos (pedimentos/imports)
let usePedimentosImport = $state(false);
// Cuando es true, usamos API de importación de Clases de Materiales (classes/imports)
let useMaterialClassesImport = $state(false);
// Cuando es true, usamos API de importación de Vehículos / Transportes (vehicles/imports)
let useVehicleImport = $state(false);
// Cuando es true, usamos API de importación de Conductores (drivers/imports)
let useDriverImport = $state(false);
// Cuando es true, usamos API de importación de Trailers y Cajas (trailers/imports)
let useTrailerImport = $state(false);
// Initialize settings for all tabs upfront to avoid reactivity loops
let allSettings = $state<Record<string, any>>(() => {
const initial: Record<string, any> = {};
for (const tab in tabSettings) {
initial[tab] = {};
tabSettings[tab].forEach((f) => {
initial[tab][f.name] = f.defaultValue;
});
}
return initial;
});
async function handleUpload(file: File, config: CsvUploadItem) {
console.log('handleUpload started', { file, config });
isUploading = true;
activeModelTarget = config.modelTarget || null;
scanResults = null;
useCustomsBrokerImport = config.id === 'customs_brokers';
useClientProviderImport = config.id === 'clients_providers';
useExchangeRateImport = config.id === 'exchange_rates';
useAmericanFractionImport = config.id === 'american_fractions';
usePedimentosImport = config.id === 'pedimentos';
useMaterialClassesImport = config.id === 'material_classes';
useVehicleImport = config.id === 'transports';
useDriverImport = config.id === 'drivers';
useTrailerImport = config.id === 'trailers';
const companyId = companyStore.activeCompany?.id || 1;
if (useCustomsBrokerImport) {
try {
const res = await api.customsBrokerImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useClientProviderImport) {
try {
const res = await api.clientProviderImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useExchangeRateImport) {
try {
const res = await api.exchangeRateImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useAmericanFractionImport) {
try {
const res = await api.americanFractionImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (usePedimentosImport) {
try {
const res = await api.pedimentosImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useMaterialClassesImport) {
try {
const res = await api.materialClassImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useVehicleImport) {
try {
const res = await api.vehicleImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useDriverImport) {
try {
const res = await api.driverImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
if (useTrailerImport) {
try {
const res = await api.trailerImports.upload(file, companyId);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
return;
}
const currentSettings = allSettings[activeTab] || {};
const footerConfig = { ...currentSettings };
if (activeTab === 'importacion') {
footerConfig.invoice_type = config.id?.startsWith('imp_def_') ? 'DEF' : 'TEM';
}
const opType = activeTab === 'exportacion' ? 'exp' : 'imp';
try {
const res = await api.imports.upload(
file,
config.modelTarget || '',
footerConfig,
companyId,
opType,
config.id
);
if (res.data?.job_id) {
currentJobId = res.data.job_id;
pollStatus();
} else {
toast.error(res.error || 'Error al subir el archivo');
isUploading = false;
}
} catch (e) {
console.error('Upload exception', e);
toast.error('Error inesperado al subir el archivo');
isUploading = false;
}
}
async function pollStatus() {
if (!currentJobId) return;
try {
const res = useCustomsBrokerImport
? await api.customsBrokerImports.status(currentJobId)
: useClientProviderImport
? await api.clientProviderImports.status(currentJobId)
: useExchangeRateImport
? await api.exchangeRateImports.status(currentJobId)
: useAmericanFractionImport
? await api.americanFractionImports.status(currentJobId)
: usePedimentosImport
? await api.pedimentosImports.status(currentJobId)
: useMaterialClassesImport
? await api.materialClassImports.status(currentJobId)
: useVehicleImport
? await api.vehicleImports.status(currentJobId)
: useDriverImport
? await api.driverImports.status(currentJobId)
: useTrailerImport
? await api.trailerImports.status(currentJobId)
: await api.imports.status(currentJobId);
console.log('Poll response', res);
if (res.error && !res.data) {
toast.error(res.error || 'Error al consultar el estado');
isUploading = false;
currentJobId = null;
return;
}
if (res.data?.status === 'waiting_confirmation') {
scanResults = res.data;
showResultModal = true;
toast.success('Escaneo completado. Revisa los resultados.');
isUploading = false;
} else if (res.data?.status === 'failed' || res.data?.status === 'FAILURE') {
const errRaw = res.data.error;
const errText =
typeof errRaw === 'string'
? errRaw.includes('finished') && errRaw.includes('inserted')
? 'La importación pudo completarse. Revisa el listado de registros.'
: errRaw
: (errRaw?.message ?? 'Error desconocido');
toast.error('Error en el procesamiento: ' + errText);
isUploading = false;
currentJobId = null;
scanResults = null;
commitResults = null;
showResultModal = false;
} else if (res.data?.status === 'warning') {
// Caso cuando no se insertaron registros pero hay información de rechazo
commitResults = res.data;
showResultModal = true;
const inserted = res.data?.inserted || 0;
const skippedInvalid = res.data?.skipped_invalid || 0;
const skippedFk = res.data?.skipped_missing_fk || 0;
const skippedDup = res.data?.skipped_duplicate || 0;
const totalSkipped = skippedInvalid + skippedFk + skippedDup;
if (inserted === 0) {
toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`);
} else {
toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`);
}
isUploading = false;
} else if (res.data?.status === 'finished') {
commitResults = res.data;
showResultModal = true;
const inserted = res.data?.inserted || 0;
const skippedInvalid = res.data?.skipped_invalid || 0;
const skippedFk = res.data?.skipped_missing_fk || 0;
const skippedDup = res.data?.skipped_duplicate || 0;
const skippedDetails = res.data?.skipped_details || [];
if (inserted > 0) {
toast.success(`Importación completada: ${inserted} registros insertados`);
if (skippedInvalid > 0 || skippedFk > 0 || skippedDup > 0) {
const totalSkipped = skippedInvalid + skippedFk + skippedDup;
toast.warning(`${totalSkipped} registros fueron rechazados`);
}
} else {
toast.error('No se insertaron registros. Revisa los errores a continuación.');
}
isUploading = false;
} else {
// Continue polling
console.log('Status not final, polling again in 2s...', res.data?.status);
setTimeout(pollStatus, 2000);
}
} catch (e) {
console.error('Poll exception', e);
// Retry on network error? Or fail?
// For now, let's keep retrying a few times or hard fail.
// Let's just log and retry.
setTimeout(pollStatus, 2000);
}
}
</script>
<div class="flex flex-col h-[calc(100vh-4rem)] -m-4 overflow-hidden">
<!-- Scrollable Content Area -->
<div class="flex-1 overflow-y-auto p-4 md:p-8 space-y-4">
<div class="flex items-center gap-4">
<h1 class="text-lg font-semibold md:text-2xl">Importación Masiva de Datos (CSV)</h1>
</div>
<Tabs.Root bind:value={activeTab} class="w-full">
<Tabs.List class="grid w-full grid-cols-2 md:grid-cols-4 lg:w-auto">
<Tabs.Trigger value="catalogos">Catálogos</Tabs.Trigger>
<Tabs.Trigger value="transportes">Transportes</Tabs.Trigger>
<Tabs.Trigger value="importacion">Importación</Tabs.Trigger>
<Tabs.Trigger value="exportacion">Exportación</Tabs.Trigger>
</Tabs.List>
<div class="mt-6">
<Tabs.Content value="catalogos" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Catálogos Generales</h2>
</div>
<UploadLauncherGrid items={catalogosConfig} onUpload={handleUpload} />
</Tabs.Content>
<Tabs.Content value="transportes" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Logística y Transporte</h2>
</div>
<UploadLauncherGrid items={transportesConfig} onUpload={handleUpload} />
</Tabs.Content>
<Tabs.Content value="importacion" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Operaciones de Importación</h2>
</div>
<UploadLauncherGrid items={importacionConfig} onUpload={handleUpload} />
</Tabs.Content>
<Tabs.Content value="exportacion" class="space-y-4">
<div class="flex items-center justify-between">
<h2 class="text-lg font-medium tracking-tight">Operaciones de Exportación</h2>
</div>
<UploadLauncherGrid items={exportacionConfig} onUpload={handleUpload} />
</Tabs.Content>
</div>
</Tabs.Root>
<div class="h-4"></div>
</div>
<!-- Fixed Footer Area -->
{#if allSettings[activeTab]}
<div class="flex-none z-20">
<ConfigFooter {activeTab} bind:settings={allSettings[activeTab]} />
</div>
{/if}
</div>
{#if scanResults || commitResults}
<ProcessingResultModal
bind:open={showResultModal}
{scanResults}
{commitResults}
{isUploading}
onConfirm={async () => {
if (!currentJobId) return;
try {
isUploading = true;
const res = useCustomsBrokerImport
? await api.customsBrokerImports.commit(currentJobId)
: useClientProviderImport
? await api.clientProviderImports.commit(currentJobId)
: useExchangeRateImport
? await api.exchangeRateImports.commit(currentJobId)
: useAmericanFractionImport
? await api.americanFractionImports.commit(currentJobId)
: usePedimentosImport
? await api.pedimentosImports.commit(currentJobId)
: useMaterialClassesImport
? await api.materialClassImports.commit(currentJobId)
: useVehicleImport
? await api.vehicleImports.commit(currentJobId)
: useDriverImport
? await api.driverImports.commit(currentJobId)
: useTrailerImport
? await api.trailerImports.commit(currentJobId)
: await api.imports.commit(currentJobId, activeModelTarget || '');
if (res.data?.commit_job_id) {
currentJobId = res.data.commit_job_id;
pollStatus();
}
} catch (err) {
toast.error('Error al iniciar la importación');
isUploading = false;
}
}}
onCancel={() => {
currentJobId = null;
scanResults = null;
commitResults = null;
showResultModal = false;
}}
onClose={() => {
currentJobId = null;
scanResults = null;
commitResults = null;
showResultModal = false;
}}
/>
{/if}

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -19,7 +19,7 @@ wait_for_tcp() {
echo "Esperando a que $service esté disponible en ${host}:${port}..."
while [ $attempt -le $max_attempts ]; do
if python -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
if python3 -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
echo "$service está listo y accesible"
return 0
fi

View File

@@ -12,11 +12,6 @@
# 6. Relación usuario-tenant en tabla user_tenants
# 7. Actualización del tenant_id del usuario con el valor real
# 8. Licencia Enterprise para el tenant (ilimitada, 1 año de vigencia)
# 9. [OPCIONAL] Datos iniciales de ejemplo si se pasa --seed-data
#
# Uso:
# ./init_first_time.sh # Solo configuración básica
# ./init_first_time.sh --seed-data # Configuración + datos de ejemplo
#
# Requisitos:
# - Keycloak corriendo en http://localhost:8080
@@ -24,6 +19,8 @@
# - Base de datos anexo76_core creada
# - jq instalado (para procesamiento JSON)
#
# Puertos: Keycloak en 18080/19000, frontend en 15173, API en 18000.
#
# Nota: El atributo tenant_id se crea con valor inicial "1" y luego
# se actualiza con el ID real del tenant creado en PostgreSQL.
###############################################################################
@@ -33,22 +30,6 @@ set -euo pipefail # Modo strict: exit on error, undefined vars, pipe failures
# Trap para cleanup en caso de error
trap 'echo -e "\n${RED}✗ Error en línea $LINENO. Script abortado.${NC}" >&2' ERR
# Parsear argumentos
SEED_DATA=false
while [[ $# -gt 0 ]]; do
case $1 in
--seed-data)
SEED_DATA=true
shift
;;
*)
echo "Uso: $0 [--seed-data]"
echo " --seed-data: Carga datos de ejemplo en las tablas"
exit 1
;;
esac
done
# Colores para output
RED='\033[0;31m'
GREEN='\033[0;32m'
@@ -144,172 +125,7 @@ create_tenant_mapper() {
fi
}
###############################################################################
# Funciones de seed data
###############################################################################
# Insertar datos de ejemplo para customs_brokers
seed_customs_brokers() {
echo " → Insertando customs brokers..."
exec_pg_sql "
INSERT INTO a76.customs_brokers (tenant_id, company_id, type, broker_key, name, address, postal_code, city, state, phone, email, country, tax_id, license, company, contact, created_at, updated_at)
VALUES
(${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB001', 'Agente Aduanal García', 'Av. Reforma 123', '01000', 'Ciudad de México', 'CDMX', '5555555555', 'garcia@aduanas.com', 'MEX', 'GAAR800101ABC', '1234', 'García y Asociados', 'Juan García', now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB002', 'Agente Aduanal López', 'Blvd. Díaz Ordaz 456', '22000', 'Tijuana', 'BC', '6641234567', 'lopez@customs.com', 'MEX', 'LOPL750505XYZ', '2345', 'López Customs', 'María López', now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB003', 'Agente Aduanal Martínez', 'Calle Industria 789', '45000', 'Guadalajara', 'JAL', '3339876543', 'martinez@broker.com', 'MEX', 'MARM850315DEF', '3456', 'Martínez Brokerage', 'Pedro Martínez', now(), now())
ON CONFLICT (broker_key, tenant_id, company_id) DO NOTHING;
" >/dev/null 2>&1
}
# Insertar datos de ejemplo para clients_and_providers
seed_clients_and_providers() {
echo " → Insertando clientes y proveedores..."
exec_pg_sql "
INSERT INTO a76.clients_and_providers (tenant_id, company_id, type_nat_foreign, name, short_name, rfc, client_or_provider, web_key, is_active, created_at, updated_at)
VALUES
(${TENANT_ID}, ${COMPANY_ID}, 'N', 'Proveedor Tecnológico SA de CV', 'PROVTECH', 'PTE901201ABC', 'BOTH', 'PROV001', true, now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'N', 'Cliente Industrial del Norte SA', 'CINORTE', 'CIN850615XYZ', 'BOTH', 'CLI001', true, now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'E', 'Global Supplies Inc', 'GLOBSUP', 'GSI123456789', 'BOTH', 'BOTH001', true, now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'N', 'Manufacturas del Bajío SA', 'MANBAJIO', 'MDB920310DEF', 'BOTH', 'CLI002', true, now(), now())
RETURNING id;
" >/dev/null 2>&1
}
# Insertar datos de ejemplo para packages
seed_packages() {
echo " → Insertando tipos de paquete..."
exec_pg_sql "
INSERT INTO a76.packages (tenant_id, company_id, key, description_es, description_en, weight_unit, plurals, plural_in, code_ace, code_aamex, created_at, updated_at)
VALUES
(${TENANT_ID}, ${COMPANY_ID}, 'PK01', 'Caja de Cartón', 'Cardboard Box', 0.5, 'CAJS', 'BOXS', 'CB01', 'CAJA001', now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'PK02', 'Pallet de Madera', 'Wooden Pallet', 15.0, 'PLTS', 'PLTS', 'WP01', 'PALL001', now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'PK03', 'Tambor Metálico', 'Metal Drum', 10.0, 'TMBS', 'DRMS', 'MD01', 'TAMB001', now(), now()),
(${TENANT_ID}, ${COMPANY_ID}, 'PK04', 'Contenedor', 'Container', 2000.0, 'CONT', 'CONT', 'CT01', 'CONT001', now(), now())
ON CONFLICT (tenant_id, company_id, key) DO NOTHING;
" >/dev/null 2>&1
}
# Insertar datos de ejemplo para classes
seed_classes() {
echo " → Insertando clases..."
exec_pg_sql "INSERT INTO a76.classes (tenant_id, company_id, class_code, description_es, description_en, material_key, unit_of_measure, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'CLS001', 'Componentes Electrónicos', 'Electronic Components', 'MP', 'PZA', '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS002', 'Partes Automotrices', 'Automotive Parts', 'MP', 'KGS', '8708.29.99', '8708.29.9900', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS003', 'Textiles y Telas', 'Textiles and Fabrics', 'MP', 'MT', '5407.20.01', '5407.20.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS004', 'Equipo de Computación', 'Computer Equipment', 'MP', 'PZA', '8471.30.01', '8471.30.0100', now(), now()) ON CONFLICT (tenant_id, company_id, class_code) DO NOTHING;" >/dev/null 2>&1
}
# Insertar datos de ejemplo para parts
seed_parts() {
echo " → Insertando partes/componentes..."
# Obtener un client_id para asociar las partes
local client_id
client_id=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} LIMIT 1;" | xargs)
if [ -n "$client_id" ]; then
exec_pg_sql "INSERT INTO a76.parts (tenant_id, company_id, client_id, part_number, description_spanish, description_english, part_class, currency_key, unit_of_measure, unit_cost, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-001', 'Microcontrolador ARM Cortex-M4', 'ARM Cortex-M4 Microcontroller', 'CLS001', 'USD', 'PZA', 15.50, '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-002', 'Filtro de Aceite Automotriz', 'Automotive Oil Filter', 'CLS002', 'USD', 'PZA', 8.75, '8421.23.01', '8421.23.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-003', 'Tela de Algodón para Tapicería', 'Cotton Upholstery Fabric', 'CLS003', 'USD', 'MT', 12.00, '5208.31.01', '5208.31.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-004', 'Disco Duro SSD 500GB', '500GB SSD Hard Drive', 'CLS004', 'USD', 'PZA', 65.00, '8471.70.01', '8471.70.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-005', 'Sensor de Temperatura Digital', 'Digital Temperature Sensor', 'CLS001', 'USD', 'PZA', 5.25, '9025.19.01', '9025.19.0100', now(), now()) ON CONFLICT (tenant_id, company_id, part_number) DO NOTHING;" >/dev/null 2>&1
fi
}
# Insertar datos de ejemplo para pedimentos
seed_pedimentos() {
echo " → Insertando pedimentos..."
# Primero obtener IDs de clientes
local client_ids
client_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('CLIENT', 'BOTH') LIMIT 2;")
local client_id_1=$(echo "$client_ids" | sed -n '1p' | xargs)
local client_id_2=$(echo "$client_ids" | sed -n '2p' | xargs)
if [ -n "$client_id_1" ]; then
exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001234', ${client_id_1}, 'imp', 'normal', 'V1', 'ITE', 'draft', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001235', ${client_id_1}, 'exp', 'normal', 'V1', 'ETE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$client_id_2" ]; then
exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001236', ${client_id_2}, 'imp', 'consolidated', 'V1', 'ITE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1
fi
}
# Insertar datos de ejemplo para invoices
seed_invoices() {
echo " → Insertando facturas..."
# Obtener IDs de clientes/proveedores
local provider_ids
provider_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('PROVIDER', 'BOTH') LIMIT 2;")
local provider_id_1=$(echo "$provider_ids" | sed -n '1p' | xargs)
local provider_id_2=$(echo "$provider_ids" | sed -n '2p' | xargs)
# Insertar facturas en invoice_header (mínimo requerido)
exec_pg_sql "INSERT INTO a76.invoice_header (tenant_id, company_id, system, operation_type, invoice_type, invoice_number, invoice_date, is_updated, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'DEF', 'INV-2024-001', '2024-01-10', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'exp', 'EXDEF', 'INV-2024-002', '2024-02-15', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'TEM', 'INV-2024-003', '2024-03-05', false, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
# Obtener IDs de las facturas recién creadas
local invoice_ids
invoice_ids=$(exec_pg_sql "SELECT id FROM a76.invoice_header WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND invoice_number IN ('INV-2024-001', 'INV-2024-002', 'INV-2024-003') ORDER BY invoice_number;")
local invoice_id_1=$(echo "$invoice_ids" | sed -n '1p' | xargs)
local invoice_id_2=$(echo "$invoice_ids" | sed -n '2p' | xargs)
local invoice_id_3=$(echo "$invoice_ids" | sed -n '3p' | xargs)
# Insertar datos financieros para las facturas
if [ -n "$invoice_id_1" ]; then
exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'foreign', 17.50, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$invoice_id_2" ]; then
exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'foreign', 17.45, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$invoice_id_3" ]; then
exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'local', 1.00, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
# Insertar datos de compliance mexicano con proveedores
if [ -n "$invoice_id_1" ] && [ -n "$provider_id_1" ]; then
exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$invoice_id_2" ] && [ -n "$provider_id_1" ]; then
exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$invoice_id_3" ] && [ -n "$provider_id_2" ]; then
exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, ${provider_id_2}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
# Insertar datos de logística con incoterm
if [ -n "$invoice_id_1" ]; then
exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'none', 'kgs', 'FOB', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$invoice_id_2" ]; then
exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'none', 'kgs', 'CIF', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
if [ -n "$invoice_id_3" ]; then
exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'none', 'kgs', 'EXW', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
fi
}
# Ejecutar todas las funciones de seed
execute_seed_data() {
echo -e "\n${YELLOW}[SEED] Cargando datos de ejemplo...${NC}"
seed_customs_brokers
seed_clients_and_providers
seed_packages
seed_classes
seed_parts
seed_pedimentos
seed_invoices
echo -e "${GREEN}✓ Datos de ejemplo cargados exitosamente${NC}"
echo -e "${YELLOW} • 3 Agentes aduanales${NC}"
echo -e "${YELLOW} • 4 Clientes/Proveedores${NC}"
echo -e "${YELLOW} • 4 Tipos de paquete${NC}"
echo -e "${YELLOW} • 4 Clases${NC}"
echo -e "${YELLOW} • 5 Parts/Componentes${NC}"
echo -e "${YELLOW} • 3 Pedimentos${NC}"
echo -e "${YELLOW} • 3 Facturas${NC}"
}
# Variables de configuración
# Variables de configuración (puertos con prefijo 1 hardcodeados)
KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080/kcauth}"
KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}"
KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}"
@@ -328,7 +144,7 @@ DEMO_EMAIL="demo@aduanasoft.com"
DEMO_FIRSTNAME="Demo"
DEMO_LASTNAME="User"
TENANT_NAME="Aduanasoft"
TENANT_NAME="Aduanasoft"A
TENANT_SLUG="aduanasoft"
COMPANY_NAME="Aduanasoft S.A. de C.V."
COMPANY_RFC="ADS010101AAA"
@@ -797,12 +613,14 @@ else
echo -e "${YELLOW}⚠ Company ya existe para este tenant${NC}"
fi
# Obtener información de la company
COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;")
# Obtener ID de la company (necesario para user_tenants)
COMPANY_ID=$(exec_pg_sql "SELECT id FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;" | xargs)
echo -e "${GREEN}✓ Company: ${COMPANY_INFO}${NC}"
echo -e "${GREEN}✓ Company ID: ${COMPANY_ID}${NC}"
if [ -z "$COMPANY_ID" ]; then
echo -e "${RED}✗ Error: No se pudo obtener el ID de la company${NC}"
exit 1
fi
COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;")
echo -e "${GREEN}✓ Company ID: ${COMPANY_ID} | ${COMPANY_INFO}${NC}"
# Agregar tenant_id al usuario demo en Keycloak
echo -e "\n${YELLOW}Asignando tenant_id al usuario demo...${NC}"
@@ -818,7 +636,7 @@ curl -s -X PUT "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/users/${USER_ID}"
echo -e "${GREEN}✓ Atributo tenant_id asignado al usuario${NC}"
# Agregar relación usuario-tenant en la base de datos
# Agregar relación usuario-tenant en la base de datos (usar company_id real)
echo -e "\n${YELLOW}Creando relación usuario-tenant en la base de datos...${NC}"
exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null
@@ -880,13 +698,6 @@ else
fi
fi
###############################################################################
# 9. Cargar datos de ejemplo (opcional)
###############################################################################
if [ "$SEED_DATA" = true ]; then
execute_seed_data
fi
###############################################################################
# Resumen final
###############################################################################
@@ -925,18 +736,6 @@ echo -e " ${GREEN}✓${NC} Plan: Enterprise (ilimitado)"
echo -e " ${GREEN}${NC} Status: Activa"
echo -e " ${GREEN}${NC} Features: API, Reportes Avanzados, Integraciones, Soporte Dedicado"
echo -e " ${GREEN}${NC} Vigencia: 1 año"
if [ "$SEED_DATA" = true ]; then
echo -e "\n${YELLOW}Datos de ejemplo:${NC}"
echo -e " ${GREEN}${NC} Agentes aduanales: 3"
echo -e " ${GREEN}${NC} Clientes/Proveedores: 4"
echo -e " ${GREEN}${NC} Tipos de paquete: 4"
echo -e " ${GREEN}${NC} Clases: 4"
echo -e " ${GREEN}${NC} Parts/Componentes: 5"
echo -e " ${GREEN}${NC} Pedimentos: 3"
echo -e " ${GREEN}${NC} Facturas: 3"
fi
echo -e "\n${YELLOW}Puedes acceder al sistema en:${NC}"
echo -e " ${GREEN}http://localhost:5173${NC}"
echo -e "\n${GREEN}════════════════════════════════════════════════════════${NC}\n"