Merge branch 'feature/csv-cajas-trailers' into feature/faltantes-csv
This commit is contained in:
6
.gitignore
vendored
6
.gitignore
vendored
@@ -24,9 +24,11 @@ wheels/
|
||||
*.egg
|
||||
.pnpm-store/
|
||||
|
||||
# Environment
|
||||
# Environment (no subir: cada quien puede usar puertos distintos vía .env)
|
||||
.env
|
||||
.env.local
|
||||
backend/.env
|
||||
frontend/.env
|
||||
backend/SCRIPTS/
|
||||
# IDEs
|
||||
.vscode/
|
||||
@@ -67,3 +69,5 @@ postgres-data/
|
||||
backend/uploads/
|
||||
docker-compose.yml
|
||||
.mypy_cache/
|
||||
|
||||
backend/celerybeat-schedule
|
||||
0
backend/api/__init__.py
Normal file
0
backend/api/__init__.py
Normal file
0
backend/api/v1/__init__.py
Normal file
0
backend/api/v1/__init__.py
Normal file
0
backend/api/v1/modules/__init__.py
Normal file
0
backend/api/v1/modules/__init__.py
Normal file
0
backend/api/v1/modules/a76/__init__.py
Normal file
0
backend/api/v1/modules/a76/__init__.py
Normal file
@@ -2,6 +2,20 @@
|
||||
Módulo de Class
|
||||
"""
|
||||
|
||||
from .routes import router
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["router"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""
|
||||
Lazy export to avoid circular imports during Celery init.
|
||||
|
||||
This package is imported when models are loaded (e.g. a76.items.models),
|
||||
so importing FastAPI routes at import-time can break Celery startup.
|
||||
"""
|
||||
if name == "router":
|
||||
from .routes import router
|
||||
|
||||
return router
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
1
backend/api/v1/modules/a76/classes/imports/__init__.py
Normal file
1
backend/api/v1/modules/a76/classes/imports/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# CSV import for Clases de Materiales (upload → scan → status → commit).
|
||||
151
backend/api/v1/modules/a76/classes/imports/routes.py
Normal file
151
backend/api/v1/modules/a76/classes/imports/routes.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Rutas de importación CSV para Clases de Materiales.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CLS_IMPORT_FILE_PREFIX,
|
||||
CLS_IMPORT_META_PREFIX,
|
||||
CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "material_classes",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
21
backend/api/v1/modules/a76/classes/imports/schemas.py
Normal file
21
backend/api/v1/modules/a76/classes/imports/schemas.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
528
backend/api/v1/modules/a76/classes/imports/tasks.py
Normal file
528
backend/api/v1/modules/a76/classes/imports/tasks.py
Normal file
@@ -0,0 +1,528 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
if not class_code:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
|
||||
if len(class_code) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
material_key = (row.get("CLAVEMAT") or "").strip()
|
||||
if material_key:
|
||||
if len(material_key) > 10:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
|
||||
if valid_material_keys is not None and material_key not in valid_material_keys:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom:
|
||||
if len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
if valid_uom_codes is not None and uom not in valid_uom_codes:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 20:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
sub_key = (row.get("CLAVESUB") or "").strip()
|
||||
if sub_key and len(sub_key) > 5:
|
||||
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
|
||||
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if iva_exempt and len(iva_exempt) > 4:
|
||||
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
rev_fisica = row.get("REVFISICA")
|
||||
if rev_fisica is not None and rev_fisica != "":
|
||||
try:
|
||||
v = int(rev_fisica)
|
||||
if v < -32768 or v > 32767:
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Classes import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Classes import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code: Dict[str, Class] = {}
|
||||
for c in (
|
||||
session.query(Class)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_code[c.class_code] = c
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
if existing:
|
||||
existing.description_es = desc_es
|
||||
existing.description_en = desc_en
|
||||
existing.material_key = material_key
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.sub_key = sub_key
|
||||
existing.physical_review = physical_review
|
||||
existing.iva_exempt_fraction = iva_exempt_fraction
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
class_code=class_code,
|
||||
description_es=desc_es,
|
||||
description_en=desc_en,
|
||||
material_key=material_key,
|
||||
unit_of_measure=unit_of_measure,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
sub_key=sub_key,
|
||||
physical_review=physical_review,
|
||||
iva_exempt_fraction=iva_exempt_fraction,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Classes import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -12,10 +12,14 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t
|
||||
|
||||
from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse
|
||||
from .service import ClassService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create a new router for custom endpoints
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / classes / csv_import"])
|
||||
|
||||
# Add consolidated catalog endpoints FIRST (before generic CRUD routes)
|
||||
# This ensures they have priority over the generic /{id} route
|
||||
@router.get(
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# CSV import flow for Clientes y Proveedores (Client Providers).
|
||||
# Replicates the same two-phase flow as customs_brokers/imports: upload → scan → waiting_confirmation → commit.
|
||||
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Rutas de importación CSV para Clientes y Proveedores.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CP_IMPORT_FILE_PREFIX,
|
||||
CP_IMPORT_META_PREFIX,
|
||||
CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "client_providers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("CP import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientOrProviderEnum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active(val: Optional[str]) -> bool:
|
||||
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
|
||||
rfc = (row.get("RFC") or "").strip()
|
||||
if not rfc:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
|
||||
if len(rfc) > 30:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
|
||||
name = (row.get("NOMBRE") or "").strip()
|
||||
if len(name) > 256:
|
||||
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
if short_name and len(short_name) > 10:
|
||||
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
curp = (row.get("CURP") or "").strip()
|
||||
if curp and len(curp) > 19:
|
||||
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CP import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
|
||||
Upsert por (tenant_id, company_id, rfc).
|
||||
"""
|
||||
logger.info(f"CP import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CP import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
rfc = _str_or_none(row_norm.get("RFC"), 30)
|
||||
if not rfc:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
|
||||
if client_or_provider is None:
|
||||
client_or_provider = ClientOrProviderEnum.BOTH
|
||||
|
||||
is_active = _parse_active(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
|
||||
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
|
||||
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
|
||||
existing.client_or_provider = client_or_provider
|
||||
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
rfc=rfc,
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 256),
|
||||
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
|
||||
curp=_str_or_none(row_norm.get("CURP"), 19),
|
||||
client_or_provider=client_or_provider,
|
||||
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
|
||||
# Opcional: crear dirección si hay email/teléfono/dirección
|
||||
email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
|
||||
if email or phone or address_str:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_str,
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
phone=phone,
|
||||
email=email,
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
|
||||
)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CP import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CP import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"client_providers": [
|
||||
{"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]},
|
||||
{"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
|
||||
{"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]},
|
||||
{"canonical": "CURP", "aliases": []},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]},
|
||||
{"canonical": "ESTADO", "aliases": []},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]},
|
||||
{"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "INCOTERM", "aliases": []},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla client_providers."""
|
||||
cols = TEMPLATE_COLUMNS.get("client_providers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -20,10 +20,14 @@ from .dto import (
|
||||
)
|
||||
from .service import ClientProviderService
|
||||
from .models import ClientProvider
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create main router to add custom endpoints
|
||||
router = APIRouter(prefix="/clients-providers")
|
||||
|
||||
# CSV import (mismo flujo que customs_brokers/imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["clients_and_providers / csv_import"])
|
||||
|
||||
|
||||
@router.get("/", response_model=ClientProviderPaginatedResponseDTO)
|
||||
async def get_clients_and_providers(
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# CSV import flow for Agentes Aduanales (Customs Brokers).
|
||||
# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit.
|
||||
161
backend/api/v1/modules/a76/customs_brokers/imports/routes.py
Normal file
161
backend/api/v1/modules/a76/customs_brokers/imports/routes.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Rutas de importación CSV para Agentes Aduanales.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CB_IMPORT_FILE_PREFIX,
|
||||
CB_IMPORT_META_PREFIX,
|
||||
CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "customs_brokers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("CB import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
447
backend/api/v1/modules/a76/customs_brokers/imports/tasks.py
Normal file
447
backend/api/v1/modules/a76/customs_brokers/imports/tasks.py
Normal file
@@ -0,0 +1,447 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con a76.imports)
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > 5:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CB import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Guardar números de línea con error en Redis para insert_valid_rows
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
|
||||
"""
|
||||
logger.info(f"CB import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CB import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
clave = (row_norm.get("CLAVE") or "").strip()[:5]
|
||||
if not clave:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
|
||||
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
|
||||
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
|
||||
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
|
||||
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
|
||||
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
|
||||
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
|
||||
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
|
||||
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
existing.license = lic[:4] if lic and lic.isdigit() else None
|
||||
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
|
||||
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
license_val = lic[:4] if lic and lic.isdigit() else None
|
||||
new_broker = CustomsBroker(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
broker_key=clave,
|
||||
type=_str_or_none(row_norm.get("TIPO"), 9),
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 80),
|
||||
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
|
||||
fax=_str_or_none(row_norm.get("FAX"), 30),
|
||||
email=_str_or_none(row_norm.get("EMAIL"), 100),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
tax_id=_str_or_none(row_norm.get("RFC"), 30),
|
||||
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
license=license_val,
|
||||
company=_str_or_none(row_norm.get("EMPRESA"), 200),
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
|
||||
)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CB import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Limpieza
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CB import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"customs_brokers": [
|
||||
{"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]},
|
||||
{"canonical": "TIPO"},
|
||||
{"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD"},
|
||||
{"canonical": "ESTADO"},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]},
|
||||
{"canonical": "FAX"},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]},
|
||||
{"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]},
|
||||
{"canonical": "EMPRESA", "aliases": ["COMPANY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla customs_brokers."""
|
||||
cols = TEMPLATE_COLUMNS.get("customs_brokers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -7,9 +7,13 @@ from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from . import dto, services
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (mismo flujo que a76.imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"])
|
||||
|
||||
customs_broker_crud = TenantCRUDRoutes(
|
||||
service=services.CustomsBrokerService,
|
||||
create_schema=dto.CustomsBrokerCreateDTO,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Exchange rate CSV import module
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Tipos de Cambio.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
ER_IMPORT_FILE_PREFIX,
|
||||
ER_IMPORT_META_PREFIX,
|
||||
ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "exchange_rates",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("ER import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,465 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if _parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
|
||||
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
|
||||
if local_raw and len(local_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
|
||||
if foreign_raw and len(foreign_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"ER import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
|
||||
"""
|
||||
logger.info(f"ER import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"ER import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
parsed_date = _parse_date(row_norm.get("FECHA"))
|
||||
if not parsed_date:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
|
||||
continue
|
||||
|
||||
try:
|
||||
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
|
||||
value_decimal = Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
|
||||
continue
|
||||
|
||||
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
|
||||
if local_currency:
|
||||
local_currency = local_currency.upper()
|
||||
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
|
||||
if foreign_currency:
|
||||
foreign_currency = foreign_currency.upper()
|
||||
|
||||
key_date = parsed_date.date()
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
|
||||
if existing:
|
||||
existing.value = value_decimal
|
||||
existing.local_currency = local_currency or None
|
||||
existing.foreign_currency = foreign_currency or None
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
date=parsed_date,
|
||||
value=value_decimal,
|
||||
local_currency=local_currency,
|
||||
foreign_currency=foreign_currency,
|
||||
)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"ER import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"ER import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Tipos de Cambio (EstructuraCatTiposCambio.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"exchange_rates": [
|
||||
{"canonical": "FECHA", "aliases": ["FECHA APLICABLE", "DATE", "FECHA TIPO CAMBIO"]},
|
||||
{"canonical": "VALOR", "aliases": ["TIPO_DE_CAMBIO", "TIPO CAMBIO", "TIPO DE CAMBIO", "VALUE", "RATE"]},
|
||||
{"canonical": "MONEDA_LOCAL", "aliases": ["MONEDA LOCAL", "LOCAL_CURRENCY", "MONEDA BASE"]},
|
||||
{"canonical": "MONEDA_EXTRANJERA", "aliases": ["MONEDA EXTRANJERA", "FOREIGN_CURRENCY", "MONEDA DESTINO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla exchange_rates."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "exchange_rates") -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -34,6 +34,10 @@ crud_router = route_handler.router
|
||||
from fastapi import APIRouter
|
||||
custom_router = APIRouter(prefix="/exchange-rate", tags=[])
|
||||
|
||||
# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered
|
||||
from .imports.routes import router as imports_router
|
||||
custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"])
|
||||
|
||||
@custom_router.get("/test-ping")
|
||||
async def test_ping():
|
||||
return {"message": "pong"}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for US Tariff Fractions (Fracción Americana): upload → scan → commit
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Fracción Americana (US Tariff Fractions).
|
||||
Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
FA_IMPORT_FILE_PREFIX,
|
||||
FA_IMPORT_META_PREFIX,
|
||||
FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "us_tariff_fractions",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("FA import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,474 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
|
||||
FA_IMPORT_META_PREFIX = "fa_import_meta:"
|
||||
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
|
||||
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:16] if len(s) > 16 else s
|
||||
|
||||
|
||||
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
|
||||
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
code_norm = _normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
if len(code_norm) > 16:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
|
||||
|
||||
prefix_raw = (row.get("PREFIJO") or "").strip()
|
||||
if prefix_raw and len(prefix_raw) > 10:
|
||||
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
|
||||
if um_raw and len(um_raw) > 10:
|
||||
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
|
||||
if tipo_raw and len(tipo_raw) > 10:
|
||||
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
adv_pct = row.get("ADVALOREM_PCT")
|
||||
if adv_pct is not None and str(adv_pct).strip():
|
||||
try:
|
||||
v = float(str(adv_pct).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
|
||||
|
||||
adv_dlls = row.get("ADVALOREM_DLLS")
|
||||
if adv_dlls is not None and str(adv_dlls).strip():
|
||||
try:
|
||||
v = float(str(adv_dlls).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"FA import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(str(val).strip().replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
|
||||
"""
|
||||
logger.info(f"FA import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"FA import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
|
||||
continue
|
||||
|
||||
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
|
||||
description = _str_or_none(row_norm.get("DESCRIPCION"))
|
||||
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
|
||||
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
|
||||
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.prefix = prefix
|
||||
existing.type_code = type_code
|
||||
existing.ad_valorem = ad_valorem
|
||||
existing.fixed_cost = fixed_cost
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.description = description
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_row = USTariffFraction(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
code=code,
|
||||
prefix=prefix,
|
||||
type_code=type_code,
|
||||
ad_valorem=ad_valorem,
|
||||
fixed_cost=fixed_cost,
|
||||
unit_of_measure=unit_of_measure,
|
||||
description=description,
|
||||
)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"FA import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path_clean = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path_clean):
|
||||
os.remove(meta_path_clean)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"FA import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Fracción Americana (EstructuraCatFraccAme.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"us_tariff_fractions": [
|
||||
{"canonical": "FRACCION_ARANCELARIA", "aliases": ["FRACCION ARANCELARIA", "CODE", "FRACCION"]},
|
||||
{"canonical": "PREFIJO"},
|
||||
{"canonical": "UNIDAD_DE_MEDIDA", "aliases": ["UNIDAD DE MEDIDA", "UMT"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "TIPO_DE_ADVALOREM", "aliases": ["TIPO DE ADVALOREM", "TIPO"]},
|
||||
{"canonical": "ADVALOREM_PCT", "aliases": ["ADVALOREM %", "ADVALOREM"]},
|
||||
{"canonical": "ADVALOREM_DLLS", "aliases": ["ADVALOREM DLLS", "ADVALOREM DLL"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "us_tariff_fractions") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla us_tariff_fractions."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(
|
||||
row: Dict[str, Any], normalize_header_fn, template_id: str = "us_tariff_fractions"
|
||||
) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -17,23 +17,28 @@ from .dto import (
|
||||
)
|
||||
from .service import USTariffFractionService
|
||||
|
||||
# Create router using TenantCRUDRoutes factory for basic CRUD operations
|
||||
# Create router using TenantCRUDRoutes factory for basic CRUD operations (prefix="" so we mount under main_router)
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=USTariffFractionService,
|
||||
create_schema=USTariffFractionCreateDTO,
|
||||
update_schema=USTariffFractionUpdateDTO,
|
||||
response_schema=USTariffFractionResponseDTO,
|
||||
prefix="/us-tariff-fractions",
|
||||
prefix="",
|
||||
tags=["a76 / general catalogs / us tariff fractions"],
|
||||
resource_name="US Tariff Fraction",
|
||||
id_name="id",
|
||||
enable_list=False, # We implement our custom list endpoint
|
||||
)
|
||||
|
||||
router = crud_router.router
|
||||
# Master router with prefix so all routes live under /us-tariff-fractions
|
||||
from .imports.routes import router as imports_router
|
||||
main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"])
|
||||
main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"])
|
||||
main_router.include_router(crud_router.router)
|
||||
|
||||
# Custom list endpoint with search filter
|
||||
@router.get(
|
||||
|
||||
# Custom list endpoint with search filter (under /us-tariff-fractions/)
|
||||
@main_router.get(
|
||||
"/",
|
||||
response_model=Dict[str, Any],
|
||||
summary="List US Tariff Fractions",
|
||||
@@ -65,3 +70,6 @@ async def list_us_tariff_fractions(
|
||||
"page_size": page_size,
|
||||
"pages": (total + page_size - 1) // page_size,
|
||||
}
|
||||
|
||||
|
||||
router = main_router
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
import base64
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
@@ -12,24 +13,39 @@ from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .tasks import scan_file, insert_valid_rows
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
IMPORT_FILE_KEY_PREFIX,
|
||||
IMPORT_META_KEY_PREFIX,
|
||||
IMPORT_REDIS_TTL,
|
||||
)
|
||||
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
"""Redis client (same broker as Celery so worker can read)."""
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
@router.post("/upload/{model_target}", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
model_target: Literal["invoice_header", "invoice_details"],
|
||||
file: UploadFile = File(...),
|
||||
footer_config: Optional[str] = Form(None), # JSON string with settings
|
||||
company_id: int = Query(..., description="Company ID"), # Required for context
|
||||
template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas
|
||||
company_id: int = Query(..., description="Company ID"), # Required for context
|
||||
operation_type: Optional[str] = Query("imp"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Step 1: Upload CSV, save to temp, trigger scan task.
|
||||
Si se envía template_id, solo se leen las columnas de esa plantilla.
|
||||
"""
|
||||
# 1. Validate Access & Get Tenant
|
||||
try:
|
||||
@@ -40,40 +56,51 @@ async def upload_import_file(
|
||||
|
||||
if not file.filename.endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Only .csv files allowed")
|
||||
|
||||
|
||||
job_id = str(uuid4())
|
||||
|
||||
# Ensure directory exists (Safety check)
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"operation_type": operation_type,
|
||||
"template_id": template_id,
|
||||
}
|
||||
|
||||
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
|
||||
try:
|
||||
# Save CSV
|
||||
contents = await file.read()
|
||||
redis_client = _get_redis()
|
||||
redis_client.set(
|
||||
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
redis_client.set(
|
||||
f"{IMPORT_META_KEY_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
|
||||
|
||||
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(contents)
|
||||
|
||||
# Save Metadata (Context)
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"operation_type": operation_type,
|
||||
}
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"File save error: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
logger.warning(f"Local file save failed (worker will use Redis): {e}")
|
||||
|
||||
# Trigger Celery Task (Async)
|
||||
# Use our job_id as the Celery task_id for easier tracking
|
||||
scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id)
|
||||
# Trigger Celery Task (Async). Worker loads file from Redis.
|
||||
scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
@@ -84,24 +111,56 @@ async def upload_import_file(
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Poll this endpoint to get % progress or final report.
|
||||
Poll to get progress or final report. Always returns an object with "status".
|
||||
"""
|
||||
# In a real app, query Redis or DB.
|
||||
# For MVP, we might mock or use Celery AsyncResult if backend shares Redis.
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == 'PENDING':
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
elif task_result.state == 'PROGRESS':
|
||||
if task_result.state == "PROGRESS":
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": task_result.info.get('current', 0),
|
||||
"total": task_result.info.get('total', 0)
|
||||
"status": "processing",
|
||||
"progress": (task_result.info or {}).get("current", 0),
|
||||
"total": (task_result.info or {}).get("total", 0),
|
||||
}
|
||||
elif task_result.state == 'SUCCESS':
|
||||
return task_result.result # Should return the report
|
||||
else:
|
||||
return {"status": task_result.state, "error": str(task_result.info)}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
|
||||
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb:
|
||||
logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg and len(lines) > 1:
|
||||
err_msg = lines[-2] + " " + (lines[-1] or "")
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg:
|
||||
result = getattr(task_result, "result", None)
|
||||
info = getattr(task_result, "info", None)
|
||||
if result is not None and not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif isinstance(result, dict) and (result.get("error") or result.get("message")):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
if not err_msg and isinstance(info, str):
|
||||
err_msg = info
|
||||
elif not err_msg and isinstance(info, dict) and "error" in info:
|
||||
err_msg = str(info["error"])
|
||||
if not err_msg:
|
||||
err_msg = "Task failed"
|
||||
return {"status": "failed", "error": err_msg}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
118
backend/api/v1/modules/a76/imports/template_config.py
Normal file
118
backend/api/v1/modules/a76/imports/template_config.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean.
|
||||
La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV).
|
||||
# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que
|
||||
# espera la lógica de validación e insert (tasks.py).
|
||||
# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA).
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
# --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) ---
|
||||
"imp_temp_header": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
|
||||
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
|
||||
{"canonical": "FECHA EMISION"},
|
||||
{"canonical": "CLAVE PROVEEDOR"},
|
||||
{"canonical": "CLAVE VENDIDO A"},
|
||||
{"canonical": "CLAVE ENVIADO A"},
|
||||
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
|
||||
{"canonical": "ADUANA DE CRUCE"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
{"canonical": "CLAVE INCOTERM"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "TIPO DE CAMBIO"},
|
||||
{"canonical": "TIPO PESO"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "REMESA"},
|
||||
{"canonical": "AGENTE ADUANAL"},
|
||||
{"canonical": "FLETES"},
|
||||
{"canonical": "VALOR SEGUROS"},
|
||||
{"canonical": "SEGUROS"},
|
||||
{"canonical": "EMBALAJES"},
|
||||
{"canonical": "OTROS INCREMENTABLES"},
|
||||
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
|
||||
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
{"canonical": "FACTURA ALTERNA"},
|
||||
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
|
||||
{"canonical": "OBSERVACIONES E"},
|
||||
{"canonical": "OBSERVACIONES I"},
|
||||
{"canonical": "E DOCUMENT"},
|
||||
{"canonical": "NUM OPERACION"},
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "PRECINTO"},
|
||||
],
|
||||
# --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura ---
|
||||
"imp_def_header": None, # se resuelve igual que imp_temp_header
|
||||
# --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura ---
|
||||
"exp_def_header": None,
|
||||
# --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) ---
|
||||
"imp_temp_details": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
|
||||
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
|
||||
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
|
||||
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
|
||||
{"canonical": "CANTIDAD"},
|
||||
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
|
||||
{"canonical": "FRACCION"},
|
||||
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
],
|
||||
# --- Partidas: Impo Def y Expo - misma estructura ---
|
||||
"imp_def_details": None,
|
||||
"exp_def_details": None,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]:
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if cols is not None:
|
||||
return cols
|
||||
if template_id in ("imp_def_header", "exp_def_header"):
|
||||
return TEMPLATE_COLUMNS.get("imp_temp_header")
|
||||
if template_id in ("imp_def_details", "exp_def_details"):
|
||||
return TEMPLATE_COLUMNS.get("imp_temp_details")
|
||||
return None
|
||||
|
||||
|
||||
def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]:
|
||||
"""
|
||||
Construye un diccionario: normalized_header -> canonical_name.
|
||||
normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos).
|
||||
"""
|
||||
cols = _resolve_template_columns(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]:
|
||||
"""
|
||||
A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict
|
||||
solo con las columnas de la plantilla, usando nombres canónicos.
|
||||
Así la plantilla se respeta: solo entran columnas definidas en la plantilla.
|
||||
"""
|
||||
lookup = build_normalized_lookup(template_id, normalize_header_fn)
|
||||
if not lookup:
|
||||
# Sin template definido: comportamiento legacy (normalizar todo)
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
from api.v1.modules.public.reference_data.transport_types.models import TransportType
|
||||
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen
|
||||
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode
|
||||
|
||||
@@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel):
|
||||
operation_type: Optional[OperationType] = Field(
|
||||
..., description="Operation type: imp/exp/sm/ctm"
|
||||
)
|
||||
|
||||
@field_validator("operation_type", mode="before")
|
||||
@classmethod
|
||||
def normalize_operation_type(cls, v):
|
||||
"""Accept DB string (e.g. 'IMP') and coerce to enum value ('imp')."""
|
||||
if isinstance(v, str):
|
||||
return v.lower() if v else v
|
||||
return v
|
||||
invoice_type: Optional[str] = Field(
|
||||
None, max_length=5, description="Invoice type key"
|
||||
)
|
||||
@@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel):
|
||||
class InvoiceFinancialsBase(BaseModel):
|
||||
"""Base fields for Financials"""
|
||||
|
||||
currency: Currency = Field(None, max_length=7, description="Currency code")
|
||||
currency: Optional[Currency] = Field(None, max_length=7, description="Currency code")
|
||||
currency_type: Optional[str] = Field("USD", description="Currency type")
|
||||
exchange_rate: Decimal = Field(0.00, description="Exchange rate")
|
||||
exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate")
|
||||
exchange_rate_mm: Optional[Decimal] = Field(
|
||||
None, description="Exchange rate currency to currency"
|
||||
)
|
||||
|
||||
@@ -67,11 +67,13 @@ class InvoiceService:
|
||||
|
||||
# Apply filters if provided
|
||||
if filters:
|
||||
if filters.get("status"):
|
||||
if filters.get("status") is not None:
|
||||
query = query.filter(models.InvoiceHeader.is_updated == filters["status"])
|
||||
if filters.get("operation_type"):
|
||||
ot = filters["operation_type"]
|
||||
ot_val = ot.value if hasattr(ot, "value") else ot
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.operation_type == filters["operation_type"]
|
||||
models.InvoiceHeader.operation_type == ot_val
|
||||
)
|
||||
if filters.get("invoice_type"):
|
||||
query = query.filter(
|
||||
@@ -89,10 +91,9 @@ class InvoiceService:
|
||||
f"%{filters['pedimento']}%"
|
||||
)
|
||||
)
|
||||
if (
|
||||
not filters.get("invoice_type")
|
||||
and filters.get("operation_type") == "exp"
|
||||
):
|
||||
ot_exp = filters.get("operation_type")
|
||||
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
|
||||
if not filters.get("invoice_type") and ot_exp_val == "exp":
|
||||
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
|
||||
|
||||
if filters.get("manifest_number"):
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Pedimentos (upload → scan → commit)
|
||||
160
backend/api/v1/modules/a76/pedmientos/imports/routes.py
Normal file
160
backend/api/v1/modules/a76/pedmientos/imports/routes.py
Normal file
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Rutas de importación CSV para Pedimentos.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
PED_IMPORT_FILE_PREFIX,
|
||||
PED_IMPORT_META_PREFIX,
|
||||
PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "pedimentos",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"ped_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"ped_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""
|
||||
Polling: estado del escaneo o del commit.
|
||||
"""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
"""
|
||||
Fase 2: Usuario confirma; se encola la inserción de filas válidas.
|
||||
"""
|
||||
task = insert_valid_rows.delay(job_id)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
25
backend/api/v1/modules/a76/pedmientos/imports/schemas.py
Normal file
25
backend/api/v1/modules/a76/pedmientos/imports/schemas.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
pass # no body needed for single model
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: Optional[str] = None
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
563
backend/api/v1/modules/a76/pedmientos/imports/tasks.py
Normal file
563
backend/api/v1/modules/a76/pedmientos/imports/tasks.py
Normal file
@@ -0,0 +1,563 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Pedimentos.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con otros imports)
|
||||
PED_IMPORT_FILE_PREFIX = "ped_import_file:"
|
||||
PED_IMPORT_META_PREFIX = "ped_import_meta:"
|
||||
PED_IMPORT_ERROR_LINES_PREFIX = "ped_import_error_lines:"
|
||||
PED_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"ped_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{PED_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{PED_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_pedimento(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_client_ids: Set[int],
|
||||
valid_regimes: Set[str],
|
||||
valid_pedimento_codes: Set[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Pedimento. Retorna error dict o None."""
|
||||
# Required: AÑO, ADUANA, PATENTE, NUMERO, CLIENTE_ID, CODIGO_PEDIMENTO, REGIMEN
|
||||
year = (row.get("AÑO") or "").strip()
|
||||
if not year:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Requerido"}
|
||||
if len(year) > 2:
|
||||
return {"line": line_num, "col": "AÑO", "msg": "Máximo 2 caracteres"}
|
||||
|
||||
customs_office = (row.get("ADUANA") or "").strip()
|
||||
if not customs_office:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Requerido"}
|
||||
if len(customs_office) > 3:
|
||||
return {"line": line_num, "col": "ADUANA", "msg": "Máximo 3 caracteres"}
|
||||
|
||||
license_val = (row.get("PATENTE") or "").strip()
|
||||
if not license_val:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Requerido"}
|
||||
if len(license_val) > 4:
|
||||
return {"line": line_num, "col": "PATENTE", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()
|
||||
if not pedimento_number:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Requerido"}
|
||||
if len(pedimento_number) > 7:
|
||||
return {"line": line_num, "col": "NUMERO", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
if not client_id_str:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Requerido"}
|
||||
try:
|
||||
client_id = int(client_id_str)
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Debe ser número entero"}
|
||||
if client_id not in valid_client_ids:
|
||||
return {"line": line_num, "col": "CLIENTE_ID", "msg": "Cliente no existe en catálogo"}
|
||||
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()
|
||||
if not pedimento_code:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Requerido"}
|
||||
if len(pedimento_code) > 2:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Máximo 2 caracteres"}
|
||||
if pedimento_code not in valid_pedimento_codes:
|
||||
return {"line": line_num, "col": "CODIGO_PEDIMENTO", "msg": "Código no existe en catálogo"}
|
||||
|
||||
regime = (row.get("REGIMEN") or "").strip()
|
||||
if not regime:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Requerido"}
|
||||
if len(regime) > 3:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Máximo 3 caracteres"}
|
||||
if regime not in valid_regimes:
|
||||
return {"line": line_num, "col": "REGIMEN", "msg": "Régimen no existe en catálogo"}
|
||||
|
||||
# Optional numeric/string fields - validate format if present
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status and len(status) > 30:
|
||||
return {"line": line_num, "col": "ESTATUS", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
for col, max_len in [("VALOR_USD", 17), ("PRECIO_PAGADO", 17), ("PESO_BRUTO", 19), ("TIPO_CAMBIO", 9)]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
continue
|
||||
try:
|
||||
Decimal(val.replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return {"line": line_num, "col": col, "msg": "Valor numérico inválido"}
|
||||
|
||||
operation_type = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if operation_type and operation_type not in ("imp", "exp", ""):
|
||||
return {"line": line_num, "col": "TIPO_OPERACION", "msg": "Debe ser imp o exp"}
|
||||
|
||||
pedimento_type = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if pedimento_type and pedimento_type not in ("normal", "consolidated", "complementary", "automobile", ""):
|
||||
return {"line": line_num, "col": "TIPO_PEDIMENTO", "msg": "Tipo no válido (normal, consolidated, complementary, automobile)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).strip().replace(",", "."))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_pedimentos_create(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict for PedimentosCreate from normalized CSV row (canonical names)."""
|
||||
year = (row.get("AÑO") or "").strip()[:2]
|
||||
customs_office = (row.get("ADUANA") or "").strip()[:3]
|
||||
license_val = (row.get("PATENTE") or "").strip()[:4]
|
||||
pedimento_number = (row.get("NUMERO") or "").strip()[:7]
|
||||
client_id_str = (row.get("CLIENTE_ID") or "").strip()
|
||||
client_id = int(client_id_str) if client_id_str else None
|
||||
pedimento_code = (row.get("CODIGO_PEDIMENTO") or "").strip()[:2]
|
||||
regime = (row.get("REGIMEN") or "").strip()[:3]
|
||||
|
||||
data = {
|
||||
"year": year,
|
||||
"customs_office": customs_office,
|
||||
"license": license_val,
|
||||
"pedimento_number": pedimento_number,
|
||||
"client_id": client_id,
|
||||
"pedimento_code": pedimento_code,
|
||||
"regime": regime,
|
||||
}
|
||||
|
||||
op = (row.get("TIPO_OPERACION") or "").strip().lower()
|
||||
if op in ("imp", "exp"):
|
||||
data["operation_type"] = op
|
||||
|
||||
ptype = (row.get("TIPO_PEDIMENTO") or "").strip().lower()
|
||||
if ptype in ("normal", "consolidated", "complementary", "automobile"):
|
||||
data["pedimento_type"] = ptype
|
||||
|
||||
status = (row.get("ESTATUS") or "").strip()
|
||||
if status:
|
||||
data["status"] = status[:30]
|
||||
|
||||
data["usd_value"] = _parse_decimal(row.get("VALOR_USD"))
|
||||
data["paid_price"] = _parse_decimal(row.get("PRECIO_PAGADO"))
|
||||
data["gross_weight"] = _parse_decimal(row.get("PESO_BRUTO"))
|
||||
data["exchange_rate"] = _parse_decimal(row.get("TIPO_CAMBIO"))
|
||||
|
||||
obs = (row.get("OBSERVACIONES") or "").strip()
|
||||
if obs:
|
||||
data["observations"] = obs
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Load valid FK sets for validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": f"No se pudo cargar catálogos: {e}"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar Pedimentos vía PedimentosService.create.
|
||||
"""
|
||||
logger.info(f"Pedimentos import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"ped_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"ped_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{PED_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Pedimentos import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
# Reload FK sets for commit-time validation
|
||||
valid_client_ids: Set[int] = set()
|
||||
valid_regimes: Set[str] = set()
|
||||
valid_pedimento_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
|
||||
for cp in session.query(ClientProvider).filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
).all():
|
||||
valid_client_ids.add(cp.id)
|
||||
for r in session.query(RegimenPedimento).all():
|
||||
valid_regimes.add(r.code)
|
||||
for pc in session.query(PedimentoCode).all():
|
||||
valid_pedimento_codes.add(pc.code)
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: failed to load FK sets: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimentos import PedimentosCreate
|
||||
from api.v1.modules.a76.pedmientos.dtos.pedimento_dates import PedimentoDatesCreate
|
||||
from api.v1.modules.a76.pedmientos.services.pedimentos import PedimentosService
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_missing_fk = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, "pedimentos")
|
||||
err = _validate_row_pedimento(
|
||||
row_norm, i, valid_client_ids, valid_regimes, valid_pedimento_codes
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
data = _row_to_pedimentos_create(row_norm)
|
||||
# Service expects at least pedimento_dates with entry_date/end_date for DB NOT NULL
|
||||
if "pedimento_dates" not in data or data.get("pedimento_dates") is None:
|
||||
data["pedimento_dates"] = PedimentoDatesCreate(
|
||||
entry_date=datetime.now(),
|
||||
end_date=datetime.now(),
|
||||
)
|
||||
create_data = PedimentosCreate(**data)
|
||||
PedimentosService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except ValueError as ve:
|
||||
if "Ya existe" in str(ve) or "duplicate" in str(ve).lower():
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
else:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(ve)})
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import line {i}: {e}")
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": str(e)})
|
||||
|
||||
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Pedimentos import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Pedimentos (EstructuraCatPedimentos.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"pedimentos": [
|
||||
{"canonical": "AÑO", "aliases": ["YEAR", "ANIO"]},
|
||||
{"canonical": "ADUANA", "aliases": ["CUSTOMS_OFFICE", "CUSTOMS OFFICE"]},
|
||||
{"canonical": "PATENTE", "aliases": ["LICENCIA", "LICENSE", "LIC"]},
|
||||
{"canonical": "NUMERO", "aliases": ["PEDIMENTO_NUMBER", "PEDIMENTO NUMBER", "NUMERO PEDIMENTO"]},
|
||||
{"canonical": "CLIENTE_ID", "aliases": ["CLIENT_ID", "CLIENTE", "ID CLIENTE"]},
|
||||
{"canonical": "TIPO_OPERACION", "aliases": ["OPERATION_TYPE", "OPERACION"]},
|
||||
{"canonical": "TIPO_PEDIMENTO", "aliases": ["PEDIMENTO_TYPE", "TIPO"]},
|
||||
{"canonical": "CODIGO_PEDIMENTO", "aliases": ["PEDIMENTO_CODE", "CODIGO", "CLAVE PEDIMENTO"]},
|
||||
{"canonical": "REGIMEN", "aliases": ["REGIME"]},
|
||||
{"canonical": "ESTATUS", "aliases": ["STATUS", "ESTADO"]},
|
||||
{"canonical": "VALOR_USD", "aliases": ["USD_VALUE", "VALOR USD", "USD"]},
|
||||
{"canonical": "PRECIO_PAGADO", "aliases": ["PAID_PRICE", "PRECIO PAGADO"]},
|
||||
{"canonical": "PESO_BRUTO", "aliases": ["GROSS_WEIGHT", "PESO BRUTO", "PESO"]},
|
||||
{"canonical": "TIPO_CAMBIO", "aliases": ["EXCHANGE_RATE", "TIPO CAMBIO", "CAMBIO"]},
|
||||
{"canonical": "OBSERVACIONES", "aliases": ["OBSERVATIONS", "OBS", "NOTAS"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla pedimentos."""
|
||||
cols = TEMPLATE_COLUMNS.get(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn, template_id: str = "pedimentos") -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn, template_id)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -31,6 +31,7 @@ from .routes.pedimento_rectification_origin import (
|
||||
from .routes.pedimento_transport_means import router as pedimento_transport_means_router
|
||||
from .routes.pedimento_validation import router as pedimento_validation_router
|
||||
from .routes.pedimentos import router as pedimentos_router
|
||||
from .imports.routes import router as pedimentos_imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -117,3 +118,8 @@ router.include_router(
|
||||
router.include_router(
|
||||
pedimentos_router, prefix="/pedimentos", tags=["a76 / pedimentos"]
|
||||
)
|
||||
router.include_router(
|
||||
pedimentos_imports_router,
|
||||
prefix="/pedimentos/imports",
|
||||
tags=["a76 / pedimentos / csv_import"],
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ from .general_catalogs.router import router as general_catalogs_router
|
||||
from .invoices.routes import router as invoices_router
|
||||
from .items.routes import router as items_router
|
||||
from .classes import router as classes_router
|
||||
from .classes import router as classes_router
|
||||
|
||||
from .clients_and_providers import router as client_and_provider_router
|
||||
from .imports.routes import router as imports_router
|
||||
from .invoice_settings.routes import router as invoice_settings_router
|
||||
|
||||
@@ -6,28 +6,28 @@ from pydantic import BaseModel
|
||||
class DriverBaseDTO(BaseModel):
|
||||
transporter_key: str
|
||||
line: int
|
||||
driver_name: Optional[str]
|
||||
license_number: Optional[str]
|
||||
express_line_id: Optional[str]
|
||||
ace_id: Optional[str]
|
||||
birth_date: Optional[int]
|
||||
gender: Optional[str]
|
||||
birth_country: Optional[str]
|
||||
hazardous_material_auth: Optional[str]
|
||||
hazardous_material_state: Optional[str]
|
||||
first_name: Optional[str]
|
||||
last_name: Optional[str]
|
||||
id_key1: Optional[str]
|
||||
id_number1: Optional[str]
|
||||
id_state1: Optional[str]
|
||||
id_country1: Optional[str]
|
||||
id_key2: Optional[str]
|
||||
id_number2: Optional[str]
|
||||
id_state2: Optional[str]
|
||||
id_country2: Optional[str]
|
||||
badge_number: Optional[str]
|
||||
class_type: Optional[str]
|
||||
unique_badge_number: Optional[str]
|
||||
driver_name: Optional[str] = None
|
||||
license_number: Optional[str] = None
|
||||
express_line_id: Optional[str] = None
|
||||
ace_id: Optional[str] = None
|
||||
birth_date: Optional[int] = None
|
||||
gender: Optional[str] = None
|
||||
birth_country: Optional[str] = None
|
||||
hazardous_material_auth: Optional[str] = None
|
||||
hazardous_material_state: Optional[str] = None
|
||||
first_name: Optional[str] = None
|
||||
last_name: Optional[str] = None
|
||||
id_key1: Optional[str] = None
|
||||
id_number1: Optional[str] = None
|
||||
id_state1: Optional[str] = None
|
||||
id_country1: Optional[str] = None
|
||||
id_key2: Optional[str] = None
|
||||
id_number2: Optional[str] = None
|
||||
id_state2: Optional[str] = None
|
||||
id_country2: Optional[str] = None
|
||||
badge_number: Optional[str] = None
|
||||
class_type: Optional[str] = None
|
||||
unique_badge_number: Optional[str] = None
|
||||
company_id: int
|
||||
tenant_id: int
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Drivers / Conductores (upload -> scan -> status -> commit).
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Rutas de importacion CSV para Conductores.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
DRV_IMPORT_FILE_PREFIX,
|
||||
DRV_IMPORT_META_PREFIX,
|
||||
DRV_IMPORT_STATUS_PREFIX,
|
||||
DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "drivers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Drivers import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not write processing status: {e}")
|
||||
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Drivers import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Insercion iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
updated: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,640 @@
|
||||
"""
|
||||
Tareas Celery para importacion CSV de Conductores.
|
||||
Flujo: scan_file (validacion) -> insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
|
||||
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
|
||||
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
|
||||
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
|
||||
DRV_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"drv_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{DRV_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
_MAX = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
"badge_number": 20,
|
||||
"class_type": 1,
|
||||
"unique_badge_number": 100,
|
||||
}
|
||||
|
||||
|
||||
def _parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_driver(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
transporter_key = (row.get("TRANSPORTISTA") or "").strip()
|
||||
if not transporter_key:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": "Requerido"}
|
||||
if len(transporter_key) > _MAX["transporter_key"]:
|
||||
return {"line": line_num, "col": "TRANSPORTISTA", "msg": f"Maximo {_MAX['transporter_key']} caracteres"}
|
||||
|
||||
line_val = _parse_int(row.get("LINEA"))
|
||||
if line_val is None:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser numerico"}
|
||||
if line_val <= 0:
|
||||
return {"line": line_num, "col": "LINEA", "msg": "Debe ser mayor a 0"}
|
||||
|
||||
driver_name = (row.get("CLAVE CONDUCTOR") or "").strip()
|
||||
if driver_name and len(driver_name) > _MAX["driver_name"]:
|
||||
return {"line": line_num, "col": "CLAVE CONDUCTOR", "msg": f"Maximo {_MAX['driver_name']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("LICENCIA", _MAX["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", _MAX["express_line_id"]),
|
||||
("IDENTIFICACION ACE", _MAX["ace_id"]),
|
||||
("SEXO", _MAX["gender"]),
|
||||
("PAIS NACIMIENTO", _MAX["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", _MAX["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", _MAX["hazardous_material_state"]),
|
||||
("NOMBRE(S)", _MAX["first_name"]),
|
||||
("APELLIDO PATERNO", _MAX["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", _MAX["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", _MAX["id_number1"]),
|
||||
("ESTADO", _MAX["id_state1"]),
|
||||
("PAIS", _MAX["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", _MAX["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", _MAX["id_number2"]),
|
||||
("ESTADO 2", _MAX["id_state2"]),
|
||||
("PAIS 2", _MAX["id_country2"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
|
||||
fecha = row.get("FECHA NACIMIENTO")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_birth_date(fecha) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA NACIMIENTO",
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _row_to_driver_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
transporter_key = _str_or_none(row.get("TRANSPORTISTA"), _MAX["transporter_key"])
|
||||
line = _parse_int(row.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
data = {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row.get("CLAVE CONDUCTOR"), _MAX["driver_name"]),
|
||||
"license_number": _str_or_none(row.get("LICENCIA"), _MAX["license_number"]),
|
||||
"express_line_id": _str_or_none(row.get("PERMISO LINEA EXPRESS"), _MAX["express_line_id"]),
|
||||
"ace_id": _str_or_none(row.get("IDENTIFICACION ACE"), _MAX["ace_id"]),
|
||||
"birth_date": _parse_birth_date(row.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row.get("SEXO"), _MAX["gender"]),
|
||||
"birth_country": _str_or_none(row.get("PAIS NACIMIENTO"), _MAX["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row.get("TRANSPORTA MAT. PELIGROSO?"), _MAX["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row.get("PERMISO MAT. PELIGROSO"), _MAX["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row.get("NOMBRE(S)"), _MAX["first_name"]),
|
||||
"last_name": _str_or_none(row.get("APELLIDO PATERNO"), _MAX["last_name"]),
|
||||
"id_key1": _str_or_none(row.get("FORMA IDENTIFICACION 1"), _MAX["id_key1"]),
|
||||
"id_number1": _str_or_none(row.get("NUM. IDENTIFICACION 1"), _MAX["id_number1"]),
|
||||
"id_state1": _str_or_none(row.get("ESTADO"), _MAX["id_state1"]),
|
||||
"id_country1": _str_or_none(row.get("PAIS"), _MAX["id_country1"]),
|
||||
"id_key2": _str_or_none(row.get("FORMA IDENTIFICACION 2"), _MAX["id_key2"]),
|
||||
"id_number2": _str_or_none(row.get("NUM. IDENTIFICACION 2"), _MAX["id_number2"]),
|
||||
"id_state2": _str_or_none(row.get("ESTADO 2"), _MAX["id_state2"]),
|
||||
"id_country2": _str_or_none(row.get("PAIS 2"), _MAX["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Drivers import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"drv_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"drv_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{DRV_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.drivers.services import DriverService
|
||||
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_driver(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"invoice": (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_driver_dto(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("transporter_key") or data.get("line") is None:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
key = f"{data['transporter_key']}:{data['line']}"
|
||||
if key in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[key] = i
|
||||
|
||||
existing = DriverService.get_driver_by_key_and_line(
|
||||
session, data["transporter_key"], data["line"], str(company_id), tenant_id
|
||||
)
|
||||
try:
|
||||
if existing:
|
||||
update_fields = {k: v for k, v in data.items() if k not in ("transporter_key", "line", "company_id", "tenant_id")}
|
||||
for field, value in update_fields.items():
|
||||
setattr(existing, field, value)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = DriverCreateDTO(**data)
|
||||
DriverService.create_driver(session, create_data)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "driver_key": key, "invoice": key, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Drivers import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Drivers import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros validos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Drivers import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Configuracion de plantilla CSV para Conductores (EstructuraCatConductor.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"drivers": [
|
||||
{"canonical": "TRANSPORTISTA", "aliases": ["TRANSPORTISTA CLAVE", "CLAVE TRANSPORTISTA", "TRANSPORTER"]},
|
||||
{"canonical": "LINEA", "aliases": ["LINE", "LINEA CONDUCTOR"]},
|
||||
{"canonical": "CLAVE CONDUCTOR", "aliases": ["CONDUCTOR", "DRIVER", "NOMBRE CONDUCTOR"]},
|
||||
{"canonical": "LICENCIA", "aliases": ["LICENSE", "LICENCE"]},
|
||||
{"canonical": "PERMISO LINEA EXPRESS", "aliases": ["LINEA EXPRESS", "LINEA EXPRESS ID", "EXPRESS LINE"]},
|
||||
{"canonical": "IDENTIFICACION ACE", "aliases": ["ACE", "ACE ID", "IDENTIFICACION ACE ID"]},
|
||||
{"canonical": "FECHA NACIMIENTO", "aliases": ["FECHA NAC", "BIRTH DATE", "BIRTHDATE"]},
|
||||
{"canonical": "SEXO", "aliases": ["GENERO", "GENDER"]},
|
||||
{"canonical": "PAIS NACIMIENTO", "aliases": ["PAIS Nacimiento", "BIRTH COUNTRY"]},
|
||||
{"canonical": "TRANSPORTA MAT. PELIGROSO?", "aliases": ["MATERIAL PELIGROSO", "HAZMAT", "HAZARDOUS MATERIAL"]},
|
||||
{"canonical": "PERMISO MAT. PELIGROSO", "aliases": ["PERMISO MAT PELIGROSO", "HAZMAT PERMIT"]},
|
||||
{"canonical": "NOMBRE(S)", "aliases": ["NOMBRE", "NOMBRES", "FIRST NAME"]},
|
||||
{"canonical": "APELLIDO PATERNO", "aliases": ["APELLIDO", "LAST NAME", "APELLIDO P"]},
|
||||
{"canonical": "FORMA IDENTIFICACION 1", "aliases": ["FORMA IDENTIFICACION", "ID TIPO 1", "ID KEY 1"]},
|
||||
{"canonical": "NUM. IDENTIFICACION 1", "aliases": ["NUM IDENTIFICACION 1", "ID NUMERO 1", "ID NUMBER 1"]},
|
||||
{"canonical": "ESTADO", "aliases": ["ESTADO 1", "STATE 1"]},
|
||||
{"canonical": "PAIS", "aliases": ["PAIS 1", "COUNTRY 1"]},
|
||||
{"canonical": "FORMA IDENTIFICACION 2", "aliases": ["ID TIPO 2", "ID KEY 2"]},
|
||||
{"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]},
|
||||
{"canonical": "ESTADO 2", "aliases": ["STATE 2"]},
|
||||
{"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("drivers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -8,9 +8,13 @@ from sqlalchemy.orm import Session
|
||||
from .dto import DriverCreateDTO, DriverResponseDTO
|
||||
from .models import Driver
|
||||
from .services import DriverService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
router = APIRouter(prefix="/drivers")
|
||||
|
||||
# CSV import (upload -> scan -> status -> commit)
|
||||
router.include_router(imports_router, prefix="/imports", tags=["a76 / drivers / csv_import"])
|
||||
|
||||
|
||||
@router.get("/", response_model=Dict[str, Any])
|
||||
async def list_drivers(
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Trailers CSV import: upload -> scan -> status -> commit
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Rutas de importación CSV para Trailers y Cajas.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
TRL_IMPORT_FILE_PREFIX,
|
||||
TRL_IMPORT_META_PREFIX,
|
||||
TRL_IMPORT_STATUS_PREFIX,
|
||||
TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "trailers",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Trailers import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not write processing status: {e}")
|
||||
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Trailers import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
updated: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,535 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Trailers y Cajas.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Upsert por trailer_number usando TrailerService.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRL_IMPORT_FILE_PREFIX = "trl_import_file:"
|
||||
TRL_IMPORT_META_PREFIX = "trl_import_meta:"
|
||||
TRL_IMPORT_ERROR_LINES_PREFIX = "trl_import_error_lines:"
|
||||
TRL_IMPORT_STATUS_PREFIX = "trl_import_status:"
|
||||
TRL_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"trl_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{TRL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Trailer model (a76.trailer)
|
||||
_MAX = {
|
||||
"trailer_number": 20,
|
||||
"ace_trailer_number": 10,
|
||||
"trailer_type_key": 2,
|
||||
"seal": 15,
|
||||
"entity_code": 1,
|
||||
"plate_number": 17,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"container_key": 3,
|
||||
}
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
if count == 1:
|
||||
unique.append(name)
|
||||
else:
|
||||
unique.append(f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _validate_row_trailer(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Trailer. Retorna error dict o None."""
|
||||
trailer_number = (row.get("NUMERO TRAILER") or "").strip()
|
||||
if not trailer_number:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": "Requerido"}
|
||||
if len(trailer_number) > _MAX["trailer_number"]:
|
||||
return {"line": line_num, "col": "NUMERO TRAILER", "msg": f"Máximo {_MAX['trailer_number']} caracteres"}
|
||||
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_trailer_number"]),
|
||||
("TIPO TRAILER", _MAX["trailer_type_key"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("CODIGO ENTIDAD", _MAX["entity_code"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("CLAVE CONTENEDOR", _MAX["container_key"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_trailer_dto(row: Dict[str, Any], tenant_id: int, company_id: int) -> Dict[str, Any]:
|
||||
"""Build dict for TrailerCreateDTO / TrailerUpdateDTO from normalized row."""
|
||||
trailer_number = _str_or_none(row.get("NUMERO TRAILER"), _MAX["trailer_number"])
|
||||
if not trailer_number:
|
||||
return {}
|
||||
return {
|
||||
"trailer_number": trailer_number,
|
||||
"ace_trailer_number": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_trailer_number"]),
|
||||
"trailer_type_key": _str_or_none(row.get("TIPO TRAILER"), _MAX["trailer_type_key"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO ENTIDAD"), _MAX["entity_code"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"container_key": _str_or_none(row.get("CLAVE CONTENEDOR"), _MAX["container_key"]),
|
||||
}
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store error lines: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Trailers import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"trl_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"trl_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{TRL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.trailers.services import TrailerService
|
||||
from api.v1.modules.a76.transportation.trailers.dto import TrailerCreateDTO, TrailerUpdateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_trailer(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"invoice": (row_norm.get("NUMERO TRAILER") or "").strip()[:20] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_trailer_dto(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("trailer_number"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
tn = data["trailer_number"]
|
||||
if tn in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"trailer_number": tn,
|
||||
"invoice": tn,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[tn] = i
|
||||
|
||||
existing = TrailerService.get_by_id(session, tn, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = TrailerUpdateDTO(**{k: v for k, v in data.items() if k != "trailer_number"})
|
||||
TrailerService.update(session, tn, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = TrailerCreateDTO(**data)
|
||||
TrailerService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "trailer_number": tn, "invoice": tn, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Trailers import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Trailers import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Trailers import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Trailers / Cajas (EstructuraCatTrailers.xls).
|
||||
Mapeo: NUMERO TRAILER → trailer_number, CLAVE ACE → ace_trailer_number, etc.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"trailers": [
|
||||
{"canonical": "NUMERO TRAILER", "aliases": ["CLAVE TRAILER", "TRAILER NUMBER", "TRAILER", "NUMERO"]},
|
||||
{"canonical": "CLAVE ACE", "aliases": ["ACE", "NUMERO ACE", "ACE TRAILER"]},
|
||||
{"canonical": "TIPO TRAILER", "aliases": ["TIPO", "TRAILER TYPE", "TIPO CAJA"]},
|
||||
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
|
||||
{"canonical": "CODIGO ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE", "CODIGO DE ENTIDAD"]},
|
||||
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE NUMBER", "PLATE"]},
|
||||
{"canonical": "ESTADO", "aliases": ["STATE"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
|
||||
{"canonical": "CLAVE CONTENEDOR", "aliases": ["CONTAINER", "CONTAINER KEY", "CONTENEDOR"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("trailers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -1,11 +1,19 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import TrailerCreateDTO, TrailerResponseDTO, TrailerUpdateDTO
|
||||
from .services import TrailerService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create router using TenantCRUDRoutes factory
|
||||
# Note: trailer_number is a string (not int) and is used as the primary key
|
||||
router = TenantCRUDRoutes(
|
||||
# Main router: trailers CRUD + CSV imports
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/trailers/imports", tags=["a76 / trailers / csv_import"])
|
||||
|
||||
# CRUD routes
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=TrailerService,
|
||||
create_schema=TrailerCreateDTO,
|
||||
update_schema=TrailerUpdateDTO,
|
||||
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
|
||||
prefix="/trailers",
|
||||
tags=[],
|
||||
resource_name="Trailer",
|
||||
id_name="trailer_number", # Using trailer_number instead of numeric ID
|
||||
id_type=str, # Specify that the ID is a string
|
||||
enable_list=True, # Enable GET /trailers with pagination
|
||||
enable_filters=True, # Enable filtering by plate_number and trailer_type_key
|
||||
id_name="trailer_number",
|
||||
id_type=str,
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
default_page_size=50,
|
||||
max_page_size=100,
|
||||
).router
|
||||
router.include_router(crud_router)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# CSV import for Vehicles / Transportes (upload → scan → status → commit).
|
||||
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
Rutas de importación CSV para Vehículos (Transportes).
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
VEHL_IMPORT_FILE_PREFIX,
|
||||
VEHL_IMPORT_META_PREFIX,
|
||||
VEHL_IMPORT_STATUS_PREFIX,
|
||||
VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: access validation failed: {e}")
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "vehicles",
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
# Fallback sin worker: ejecutar scan en un hilo y guardar resultado en Redis
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Vehicles import: background scan failed: {e}")
|
||||
|
||||
threading.Thread(target=run_scan_background, daemon=True).start()
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
# Si el resultado del scan está en Redis (worker o hilo de fallback), usarlo
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not read status from Redis: {e}")
|
||||
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = (task_result.info or {})
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg and result is not None:
|
||||
if not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif result.get("error") or result.get("message"):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str):
|
||||
# Escribir "processing" en Redis para que el polling muestre estado hasta que termine el commit
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps({"status": "processing", "message": "Insertando..."}).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not write processing status: {e}")
|
||||
|
||||
# Ejecutar commit en hilo (mismo fallback que el scan: no depende del worker de Celery)
|
||||
def run_commit_background():
|
||||
try:
|
||||
run_commit_sync(job_id)
|
||||
except Exception as e:
|
||||
logger.exception(f"Vehicles import: background commit failed: {e}")
|
||||
|
||||
threading.Thread(target=run_commit_background, daemon=True).start()
|
||||
|
||||
# Mismo job_id para que el front siga haciendo polling y reciba el resultado de Redis
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Inserción iniciada.",
|
||||
"commit_job_id": job_id,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
skipped_invalid: Optional[int] = 0
|
||||
skipped_missing_fk: Optional[int] = 0
|
||||
skipped_duplicate: Optional[int] = 0
|
||||
skipped_details: Optional[list] = None
|
||||
@@ -0,0 +1,607 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Vehículos (Transportes).
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Respeta la lógica manual: vehicle_key requerido, resto opcional; upsert por vehicle_key.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VEHL_IMPORT_FILE_PREFIX = "veh_import_file:"
|
||||
VEHL_IMPORT_META_PREFIX = "veh_import_meta:"
|
||||
VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:"
|
||||
VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:"
|
||||
VEHL_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"veh_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{VEHL_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
# Max lengths from Vehicle model (a76.vehicle)
|
||||
_MAX = {
|
||||
"vehicle_key": 14,
|
||||
"ace_vehicle_key": 10,
|
||||
"transporter_key": 23,
|
||||
"transport_identifier": 30,
|
||||
"transport_type": 2,
|
||||
"entity_code": 1,
|
||||
"transponder_number": 16,
|
||||
"dot_number": 8,
|
||||
"plate_number": 17,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"seal": 49,
|
||||
"insurance_company_name": 30,
|
||||
"insurance_number": 20,
|
||||
"series": 30,
|
||||
}
|
||||
|
||||
|
||||
def _parse_decimal(val: Any) -> Optional[Decimal]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
try:
|
||||
s = str(val).strip().replace(",", "")
|
||||
return Decimal(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_insurance_date(val: Any) -> Optional[int]:
|
||||
"""Parse FECHA DE ASEGURADORA to integer yyyymmdd. Tolerates DD/MM/YYYY, YYYY-MM-DD, or YYYYMMDD."""
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
# Already integer-like
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
# Try DD/MM/YYYY or similar
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2: # c=year
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2: # a=year
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_vehicle(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Vehículo. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > _MAX["vehicle_key"]:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": f"Máximo {_MAX['vehicle_key']} caracteres"}
|
||||
|
||||
# Optional fields: max lengths only
|
||||
for col, max_len in [
|
||||
("CLAVE ACE", _MAX["ace_vehicle_key"]),
|
||||
("CLAVE TRANSPORTE", _MAX["transporter_key"]),
|
||||
("VIN", _MAX["series"]),
|
||||
("TIPO TRANSPORTE", _MAX["transport_type"]),
|
||||
("CODIGO DE ENTIDAD", _MAX["entity_code"]),
|
||||
("TRANSPONDEDOR", _MAX["transponder_number"]),
|
||||
("NUMERO DOT", _MAX["dot_number"]),
|
||||
("PLACAS", _MAX["plate_number"]),
|
||||
("CIUDAD", _MAX["city"]),
|
||||
("ESTADO", _MAX["state"]),
|
||||
("PAIS", _MAX["country"]),
|
||||
("PRECINTO", _MAX["seal"]),
|
||||
("EMPRESA ASEGURADORA", _MAX["insurance_company_name"]),
|
||||
("NUM. ASEGURADORA", _MAX["insurance_number"]),
|
||||
]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
|
||||
# MONTO ASEGURADO: must be numeric if present
|
||||
monto = row.get("MONTO ASEGURADO") or row.get("MONTO")
|
||||
if monto is not None and str(monto).strip():
|
||||
if _parse_decimal(monto) is None:
|
||||
return {"line": line_num, "col": "MONTO ASEGURADO", "msg": "Debe ser numérico"}
|
||||
|
||||
# FECHA DE ASEGURADORA: optional; if present try parse (do not fail row if invalid, set None)
|
||||
# Plan says: "en caso de formato inválido, marcar error pero no rechazar toda la fila" -> we can either
|
||||
# reject or set None. We reject invalid date to keep data quality.
|
||||
fecha = row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")
|
||||
if fecha is not None and str(fecha).strip():
|
||||
if _parse_insurance_date(fecha) is None:
|
||||
return {"line": line_num, "col": "FECHA DE ASEGURADORA", "msg": "Formato de fecha inválido (use YYYYMMDD o DD/MM/YYYY)"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _do_scan(
|
||||
job_id: str,
|
||||
progress_callback: Optional[Any] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de escaneo (sin Celery). Usado por la tarea scan_file y por run_scan_sync.
|
||||
progress_callback(current, total, error_count) opcional.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store error lines in Redis: {e}")
|
||||
|
||||
result = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el escaneo en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de upload en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting scan for job {job_id}")
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": current, "total": total, "errors": errors},
|
||||
)
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store scan status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _row_to_vehicle_dto(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build dict suitable for VehicleCreateDTO / VehicleUpdateDTO from normalized row."""
|
||||
vehicle_key = _str_or_none(row.get("CLAVE"), _MAX["vehicle_key"])
|
||||
if not vehicle_key:
|
||||
return {}
|
||||
data = {
|
||||
"vehicle_key": vehicle_key,
|
||||
"ace_vehicle_key": _str_or_none(row.get("CLAVE ACE"), _MAX["ace_vehicle_key"]),
|
||||
"transporter_key": _str_or_none(row.get("CLAVE TRANSPORTE"), _MAX["transporter_key"]),
|
||||
"series": _str_or_none(row.get("VIN"), _MAX["series"]),
|
||||
"transport_type": _str_or_none(row.get("TIPO TRANSPORTE"), _MAX["transport_type"]),
|
||||
"entity_code": _str_or_none(row.get("CODIGO DE ENTIDAD"), _MAX["entity_code"]),
|
||||
"transponder_number": _str_or_none(row.get("TRANSPONDEDOR"), _MAX["transponder_number"]),
|
||||
"dot_number": _str_or_none(row.get("NUMERO DOT"), _MAX["dot_number"]),
|
||||
"plate_number": _str_or_none(row.get("PLACAS"), _MAX["plate_number"]),
|
||||
"city": _str_or_none(row.get("CIUDAD"), _MAX["city"]),
|
||||
"state": _str_or_none(row.get("ESTADO"), _MAX["state"]),
|
||||
"country": _str_or_none(row.get("PAIS"), _MAX["country"]),
|
||||
"seal": _str_or_none(row.get("PRECINTO"), _MAX["seal"]),
|
||||
"insurance_company_name": _str_or_none(row.get("EMPRESA ASEGURADORA"), _MAX["insurance_company_name"]),
|
||||
"insurance_number": _str_or_none(row.get("NUM. ASEGURADORA"), _MAX["insurance_number"]),
|
||||
"insurance_amount": _parse_decimal(row.get("MONTO ASEGURADO") or row.get("MONTO")),
|
||||
"insurance_date": _parse_insurance_date(row.get("FECHA DE ASEGURADORA") or row.get("FECHA ASEGURADORA")),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Lógica de commit (inserción/actualización). Usado por la tarea insert_valid_rows y por run_commit_sync.
|
||||
"""
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"veh_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"veh_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{VEHL_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.transportation.vehicles.services import VehicleService
|
||||
from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_vehicle(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"vehicle_key": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"invoice": (row_norm.get("CLAVE") or "").strip()[:14] or "-",
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
data = _row_to_vehicle_dto(row_norm)
|
||||
if not data or not data.get("vehicle_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
vk = data["vehicle_key"]
|
||||
if vk in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)"}
|
||||
)
|
||||
continue
|
||||
seen_keys_in_file[vk] = i
|
||||
|
||||
existing = VehicleService.get_by_id(session, vk, tenant_id, company_id)
|
||||
try:
|
||||
if existing:
|
||||
update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"})
|
||||
VehicleService.update(session, vk, tenant_id, update_data, company_id)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = VehicleCreateDTO(**data)
|
||||
VehicleService.create(session, create_data, tenant_id, company_id)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err)}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Vehicles import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Vehicles import cleanup failed: {cleanup_err}")
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Ejecuta el commit en el proceso actual y guarda el resultado en Redis.
|
||||
Usado desde el endpoint de commit en un hilo cuando no hay worker de Celery.
|
||||
"""
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, create/update via VehicleService.
|
||||
"""
|
||||
logger.info(f"Vehicles import: starting commit for job {job_id}")
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: failed to store commit status in Redis: {e}")
|
||||
return result
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Vehículos / Transportes (EstructuraCatTransportes.xls).
|
||||
Mapeo: CLAVE → vehicle_key, CLAVE ACE → ace_vehicle_key, etc.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"vehicles": [
|
||||
{"canonical": "CLAVE", "aliases": ["CLAVE VEHICULO", "VEHICLE KEY", "KEY"]},
|
||||
{"canonical": "CLAVE ACE", "aliases": ["ACE", "CLAVEACE"]},
|
||||
{"canonical": "CLAVE TRANSPORTE", "aliases": ["TRANSPORTE", "TRANSPORTISTA"]},
|
||||
{"canonical": "VIN", "aliases": ["SERIE", "SERIES", "NUMERO SERIE"]},
|
||||
{"canonical": "TIPO TRANSPORTE", "aliases": ["TIPO", "TRANSPORT TYPE"]},
|
||||
{"canonical": "CODIGO DE ENTIDAD", "aliases": ["ENTIDAD", "ENTITY CODE"]},
|
||||
{"canonical": "TRANSPONDEDOR", "aliases": ["TRANSPONDER", "TRANSPONDER NUMBER"]},
|
||||
{"canonical": "NUMERO DOT", "aliases": ["DOT", "NUM DOT"]},
|
||||
{"canonical": "PLACAS", "aliases": ["PLACA", "PLATE", "PLATE NUMBER"]},
|
||||
{"canonical": "CIUDAD", "aliases": ["CITY"]},
|
||||
{"canonical": "ESTADO", "aliases": ["STATE"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "Pais"]},
|
||||
{"canonical": "PRECINTO", "aliases": ["SEAL"]},
|
||||
{"canonical": "EMPRESA ASEGURADORA", "aliases": ["ASEGURADORA", "INSURANCE COMPANY"]},
|
||||
{"canonical": "NUM. ASEGURADORA", "aliases": ["NUM ASEGURADORA", "POLIZA", "INSURANCE NUMBER"]},
|
||||
{"canonical": "MONTO ASEGURADO", "aliases": ["MONTO", "INSURANCE AMOUNT"]},
|
||||
{"canonical": "FECHA DE ASEGURADORA", "aliases": ["FECHA ASEGURADORA", "INSURANCE DATE"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("vehicles")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -1,11 +1,19 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
|
||||
from .dto import VehicleCreateDTO, VehicleResponseDTO, VehicleUpdateDTO
|
||||
from .services import VehicleService
|
||||
from .imports.routes import router as imports_router
|
||||
|
||||
# Create router using TenantCRUDRoutes factory
|
||||
# Note: vehicle_key is a string (not int) and is used as the primary key
|
||||
router = TenantCRUDRoutes(
|
||||
# Main router: vehicles CRUD + CSV imports
|
||||
router = APIRouter()
|
||||
|
||||
# CSV import (upload → scan → status → commit)
|
||||
router.include_router(imports_router, prefix="/vehicles/imports", tags=["a76 / vehicles / csv_import"])
|
||||
|
||||
# CRUD routes
|
||||
crud_router = TenantCRUDRoutes(
|
||||
service=VehicleService,
|
||||
create_schema=VehicleCreateDTO,
|
||||
update_schema=VehicleUpdateDTO,
|
||||
@@ -13,10 +21,11 @@ router = TenantCRUDRoutes(
|
||||
prefix="/vehicles",
|
||||
tags=[],
|
||||
resource_name="Vehicle",
|
||||
id_name="vehicle_key", # Using vehicle_key instead of numeric ID
|
||||
id_type=str, # Specify that the ID is a string
|
||||
enable_list=True, # Enable GET /vehicles with pagination
|
||||
enable_filters=True, # Enable filtering by plate_number and transport_type
|
||||
id_name="vehicle_key",
|
||||
id_type=str,
|
||||
enable_list=True,
|
||||
enable_filters=True,
|
||||
default_page_size=50,
|
||||
max_page_size=100,
|
||||
).router
|
||||
router.include_router(crud_router)
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import os
|
||||
from celery import Celery
|
||||
|
||||
# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen (mapper)
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
|
||||
CodePedimentoRegimen,
|
||||
)
|
||||
|
||||
# Import models in correct order for SQLAlchemy relationship resolution
|
||||
# CRITICAL: FaLineItem must be imported BEFORE LineItem
|
||||
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem # noqa: F401
|
||||
@@ -26,6 +33,13 @@ celery_app.conf.update(
|
||||
"api.v1.modules.a76.reports.movements.invoices.tasks",
|
||||
"api.v1.modules.a76.reports.exportacion.descargo.task",
|
||||
"api.v1.modules.a76.imports.tasks",
|
||||
"api.v1.modules.a76.customs_brokers.imports.tasks",
|
||||
"api.v1.modules.a76.clients_and_providers.imports.tasks",
|
||||
"api.v1.modules.a76.pedmientos.imports.tasks",
|
||||
"api.v1.modules.a76.general_catalogs.exchange_rate.imports.tasks",
|
||||
"api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.tasks",
|
||||
"api.v1.modules.a76.classes.imports.tasks",
|
||||
"api.v1.modules.a76.transportation.vehicles.imports.tasks",
|
||||
"api.v1.modules.a76.reports.exportacion.transmission.MAINX30.task",
|
||||
"api.v1.modules.a76.reports.importacion.transmission.temporal.MAINX30.task",
|
||||
"api.v1.modules.a76.reports.importacion.transmission.definitive.MAINX30.task",
|
||||
|
||||
@@ -11,11 +11,23 @@ from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
|
||||
|
||||
from .config import settings
|
||||
from .exceptions import BaseAPIException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cors_headers(request: Request) -> Dict[str, str]:
|
||||
"""CORS headers for error responses so browser does not block on 4xx/5xx."""
|
||||
origin = request.headers.get("origin")
|
||||
if not origin or origin not in settings.cors_origins_list:
|
||||
return {}
|
||||
return {
|
||||
"Access-Control-Allow-Origin": origin,
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
}
|
||||
|
||||
|
||||
async def base_exception_handler(
|
||||
request: Request,
|
||||
exc: BaseAPIException,
|
||||
@@ -36,10 +48,13 @@ async def base_exception_handler(
|
||||
if hasattr(exc, "errors") and exc.errors:
|
||||
logger.warning(f"Validation errors details: {exc.errors}")
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content=jsonable_encoder(exc.to_dict()),
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def validation_exception_handler(
|
||||
@@ -65,7 +80,7 @@ async def validation_exception_handler(
|
||||
extra={"errors": errors},
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
content={
|
||||
"error": "VALIDATION_ERROR",
|
||||
@@ -74,6 +89,9 @@ async def validation_exception_handler(
|
||||
"errors": errors,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def integrity_error_handler(
|
||||
@@ -105,7 +123,7 @@ async def integrity_error_handler(
|
||||
else:
|
||||
error_message = "Error de integridad en la base de datos"
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
content={
|
||||
"error": "DATABASE_INTEGRITY_ERROR",
|
||||
@@ -113,6 +131,9 @@ async def integrity_error_handler(
|
||||
"status_code": status.HTTP_409_CONFLICT,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def sqlalchemy_error_handler(
|
||||
@@ -131,7 +152,7 @@ async def sqlalchemy_error_handler(
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
content={
|
||||
"error": "DATABASE_ERROR",
|
||||
@@ -139,6 +160,9 @@ async def sqlalchemy_error_handler(
|
||||
"status_code": status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
async def http_exception_handler(
|
||||
@@ -174,7 +198,7 @@ async def general_exception_handler(
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
content={
|
||||
"error": "INTERNAL_SERVER_ERROR",
|
||||
@@ -182,6 +206,9 @@ async def general_exception_handler(
|
||||
"status_code": status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
},
|
||||
)
|
||||
for k, v in _cors_headers(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
def register_exception_handlers(app) -> None:
|
||||
|
||||
@@ -17,8 +17,13 @@ from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
|
||||
# Orden: PedimentoCode y RegimenPedimento antes de CodePedimentoRegimen para que
|
||||
# SQLAlchemy resuelva los nombres en relationship() al configurar el mapper
|
||||
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
|
||||
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
|
||||
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
|
||||
CodePedimentoRegimen,
|
||||
)
|
||||
from api.v1.modules.public.reference_data.sectors.models import Sector
|
||||
from api.v1.modules.public.reference_data.states.models import State
|
||||
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode
|
||||
@@ -134,6 +139,20 @@ logger = logging.getLogger(__name__)
|
||||
register_exception_handlers(app)
|
||||
|
||||
|
||||
def _cors_headers_for_request(request: Request):
|
||||
"""Return CORS headers if request Origin is allowed (so error responses don't get blocked by browser)."""
|
||||
origin = request.headers.get("origin")
|
||||
if not origin:
|
||||
return {}
|
||||
allowed = settings.cors_origins_list
|
||||
if origin in allowed:
|
||||
return {
|
||||
"Access-Control-Allow-Origin": origin,
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
# Add validation error handler
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
@@ -141,12 +160,29 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
|
||||
f"Validation error for {request.method} {request.url.path}: {exc.errors()}"
|
||||
)
|
||||
logger.error(f"Request body: {await request.body()}")
|
||||
return JSONResponse(
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
content={"detail": exc.errors(), "body": exc.body},
|
||||
)
|
||||
for k, v in _cors_headers_for_request(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
|
||||
# Add HTTP exception handler
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
logger.error(
|
||||
f"HTTP {exc.status_code} for {request.method} {request.url.path}: {exc.detail}"
|
||||
)
|
||||
response = JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={"detail": exc.detail},
|
||||
)
|
||||
for k, v in _cors_headers_for_request(request).items():
|
||||
response.headers[k] = v
|
||||
return response
|
||||
|
||||
def run_migrations():
|
||||
subprocess.run(["alembic", "upgrade", "head"], check=True)
|
||||
|
||||
|
||||
@@ -64,5 +64,6 @@
|
||||
"lucide-svelte": "^0.553.0",
|
||||
"marked": "^12.0.0",
|
||||
"svelte-sonner": "^1.0.7"
|
||||
}
|
||||
}
|
||||
},
|
||||
"packageManager": "pnpm@10.30.3+sha512.c961d1e0a2d8e354ecaa5166b822516668b7f44cb5bd95122d590dd81922f606f5473b6d23ec4a5be05e7fcd18e8488d47d978bbe981872f1145d06e9a740017"
|
||||
}
|
||||
|
||||
@@ -341,6 +341,173 @@ export const api = {
|
||||
validate: (tenantId: number) => api.get(`/v1/licenses/validate/${tenantId}/`)
|
||||
},
|
||||
|
||||
imports: {
|
||||
upload: (
|
||||
file: File,
|
||||
modelTarget: string,
|
||||
footerConfig: any,
|
||||
companyId: number,
|
||||
operationType: string,
|
||||
templateId?: string
|
||||
) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
if (footerConfig) {
|
||||
formData.append('footer_config', JSON.stringify(footerConfig));
|
||||
}
|
||||
if (templateId) {
|
||||
formData.append('template_id', templateId);
|
||||
}
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
company_id: String(companyId),
|
||||
operation_type: operationType || 'imp'
|
||||
}).toString();
|
||||
|
||||
return fetchApi(`/v1/a76/imports/upload/${modelTarget}?${queryParams}`, {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/imports/${jobId}/status`),
|
||||
commit: (jobId: string, modelTarget: string) =>
|
||||
api.post(`/v1/a76/imports/${jobId}/commit`, { model_target: modelTarget })
|
||||
},
|
||||
|
||||
// CSV import for Agentes Aduanales (flujo propio en customs_brokers/imports)
|
||||
customsBrokerImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/customs-brokers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/customs-brokers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/customs-brokers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Clientes y Proveedores (flujo propio en clients_and_providers/imports)
|
||||
clientProviderImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/clients-providers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/clients-providers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/clients-providers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Tipos de Cambio (flujo en general_catalogs/exchange_rate/imports)
|
||||
exchangeRateImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/exchange-rate/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/exchange-rate/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/exchange-rate/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Fracción Americana (us_tariff_fractions/imports)
|
||||
americanFractionImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/us-tariff-fractions/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/us-tariff-fractions/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/us-tariff-fractions/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Pedimentos (pedimentos/imports)
|
||||
pedimentosImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/pedimentos/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/pedimentos/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/pedimentos/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Clases de Materiales (classes/imports)
|
||||
materialClassImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/classes/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/classes/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/classes/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Vehículos / Transportes (transportation/vehicles/imports)
|
||||
vehicleImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/transportation/vehicles/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/transportation/vehicles/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/transportation/vehicles/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Conductores (drivers/imports)
|
||||
driverImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/drivers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/drivers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) => api.post(`/v1/a76/drivers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// CSV import for Trailers y Cajas (transportation/trailers/imports)
|
||||
trailerImports: {
|
||||
upload: (file: File, companyId: number) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
return fetchApi(
|
||||
`/v1/a76/transportation/trailers/imports/upload?company_id=${companyId}`,
|
||||
{ method: 'POST', body: formData }
|
||||
);
|
||||
},
|
||||
status: (jobId: string) => api.get(`/v1/a76/transportation/trailers/imports/${jobId}/status`),
|
||||
commit: (jobId: string) =>
|
||||
api.post(`/v1/a76/transportation/trailers/imports/${jobId}/commit`, {})
|
||||
},
|
||||
|
||||
// Generic request for custom needs (like file uploads)
|
||||
request: <T = any>(endpoint: string, options: RequestInit = {}) => fetchApi<T>(endpoint, options)
|
||||
};
|
||||
|
||||
@@ -143,17 +143,53 @@
|
||||
|
||||
{#if scanResults.error_count > 0}
|
||||
<div
|
||||
class="rounded-md bg-destructive/5 border border-destructive/10 p-4 flex items-start gap-3"
|
||||
class="rounded-md bg-destructive/5 border border-destructive/10 p-4 flex items-start gap-3 mb-4"
|
||||
>
|
||||
<XCircle class="w-5 h-5 text-destructive mt-0.5 shrink-0" />
|
||||
<div class="text-sm text-destructive-foreground/90">
|
||||
<p class="font-semibold mb-1">Se detectaron problemas en el archivo</p>
|
||||
<p>
|
||||
Las filas con errores serán omitidas automáticamente. Solo se importarán los
|
||||
registros válidos.
|
||||
Corrija los datos indicados abajo en su CSV y vuelva a subir, o confirme para
|
||||
importar solo las filas válidas (las erróneas se omitirán).
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{#if scanResults.errors && scanResults.errors.length > 0}
|
||||
<div class="border rounded-lg overflow-hidden shadow-sm">
|
||||
<div class="bg-muted/50 px-4 py-2 border-b flex justify-between items-center">
|
||||
<h5 class="text-xs font-bold text-foreground uppercase tracking-wide">
|
||||
Detalle de errores (para corregir en el CSV)
|
||||
</h5>
|
||||
<span
|
||||
class="text-[10px] bg-secondary text-secondary-foreground px-2 py-0.5 rounded-full border"
|
||||
>
|
||||
{scanResults.errors.length} error(es)
|
||||
</span>
|
||||
</div>
|
||||
<div class="max-h-60 overflow-y-auto bg-card relative">
|
||||
<table class="w-full text-xs text-left">
|
||||
<thead
|
||||
class="text-muted-foreground font-medium bg-muted/30 sticky top-0 z-10 shadow-sm backdrop-blur-sm"
|
||||
>
|
||||
<tr>
|
||||
<th class="px-4 py-2 w-16">Línea</th>
|
||||
<th class="px-4 py-2 w-40">Columna</th>
|
||||
<th class="px-4 py-2">Mensaje</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y">
|
||||
{#each scanResults.errors as err}
|
||||
<tr class="hover:bg-muted/30 transition-colors">
|
||||
<td class="px-4 py-2 font-mono text-muted-foreground">{err.line}</td>
|
||||
<td class="px-4 py-2 font-mono font-medium text-foreground">{err.col || '-'}</td>
|
||||
<td class="px-4 py-2 text-destructive">{err.msg || '-'}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
{:else}
|
||||
<div class="rounded-md bg-primary/5 border border-primary/10 p-4 flex items-start gap-3">
|
||||
<CheckCircle2 class="w-5 h-5 text-primary mt-0.5 shrink-0" />
|
||||
|
||||
@@ -123,7 +123,7 @@
|
||||
ondragover={(e) => handleDragOver(e, item.disabled)}
|
||||
ondrop={(e) => handleDrop(e, item)}
|
||||
oncontextmenu={(e) => handleContextMenu(e, item)}
|
||||
roles="button"
|
||||
role="button"
|
||||
tabindex={item.disabled ? -1 : 0}
|
||||
onclick={() => handleClick(item.id, item.disabled)}
|
||||
onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)}
|
||||
|
||||
@@ -8,6 +8,20 @@ export type { CustomsBroker };
|
||||
|
||||
export function createColumns(onSuccess?: () => void): ColumnDef<CustomsBroker>[] {
|
||||
return [
|
||||
{
|
||||
accessorKey: "id",
|
||||
header: "ID",
|
||||
cell: ({ row }) => {
|
||||
const idSnippet = createRawSnippet<[{ id: number }]>((getId) => {
|
||||
const { id } = getId();
|
||||
return {
|
||||
render: () =>
|
||||
`<code class="relative rounded bg-muted px-[0.3rem] py-[0.2rem] font-mono text-sm">${id}</code>`
|
||||
};
|
||||
});
|
||||
return renderSnippet(idSnippet, { id: row.original.id });
|
||||
}
|
||||
},
|
||||
{
|
||||
accessorKey: "broker_key",
|
||||
header: "Clave",
|
||||
|
||||
@@ -249,7 +249,7 @@ export const importacionConfig: CsvUploadItem[] = [
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
modelTarget: 'invoice_header',
|
||||
templateUrl: '/csv/EstructuraEncFacImpoDef.xls'
|
||||
},
|
||||
{
|
||||
@@ -257,7 +257,7 @@ export const importacionConfig: CsvUploadItem[] = [
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Impo. Def.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
modelTarget: 'invoice_details',
|
||||
templateUrl: '/csv/EstructuraParFacImpoDefAF.xls'
|
||||
},
|
||||
{
|
||||
@@ -274,7 +274,7 @@ export const importacionConfig: CsvUploadItem[] = [
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
modelTarget: 'invoice_header',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
@@ -282,7 +282,7 @@ export const importacionConfig: CsvUploadItem[] = [
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Compras Mex.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
modelTarget: 'invoice_details',
|
||||
disabled: true,
|
||||
},
|
||||
{
|
||||
@@ -302,7 +302,7 @@ export const exportacionConfig: CsvUploadItem[] = [
|
||||
title: 'Encabezado',
|
||||
icon: FileText,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'InvoiceHeader',
|
||||
modelTarget: 'invoice_header',
|
||||
templateUrl: '/csv/EstructuraEncFacExpoCamReg.xls'
|
||||
},
|
||||
{
|
||||
@@ -310,7 +310,7 @@ export const exportacionConfig: CsvUploadItem[] = [
|
||||
title: 'Partidas',
|
||||
icon: Package,
|
||||
group: 'Expo. Def./Cam. Reg.',
|
||||
modelTarget: 'InvoiceSalesDetails',
|
||||
modelTarget: 'invoice_details',
|
||||
templateUrl: '/csv/EstructuraParExpoCamReg.xls'
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1,209 +1,474 @@
|
||||
<script lang="ts">
|
||||
import * as Tabs from '$lib/components/ui/tabs/index.js';
|
||||
import UploadLauncherGrid from '$lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte';
|
||||
import ConfigFooter from '$lib/components/dashboard/csv-upload/ConfigFooter.svelte';
|
||||
import ProcessingResultModal from '$lib/components/dashboard/csv-upload/ProcessingResultModal.svelte';
|
||||
import {
|
||||
catalogosConfig,
|
||||
transportesConfig,
|
||||
importacionConfig,
|
||||
exportacionConfig,
|
||||
tabSettings,
|
||||
type CsvUploadItem
|
||||
} from '$lib/config/csv-upload';
|
||||
import { api } from '$lib/api';
|
||||
import { toast } from 'svelte-sonner';
|
||||
import { companyStore } from '$lib/stores/company.svelte';
|
||||
|
||||
// We no longer need modal state
|
||||
let activeTab = $state('catalogos');
|
||||
|
||||
let isUploading = $state(false);
|
||||
let currentJobId = $state<string | null>(null);
|
||||
let activeModelTarget = $state<string | null>(null);
|
||||
let scanResults = $state<any>(null);
|
||||
let commitResults = $state<any>(null);
|
||||
let showResultModal = $state(false);
|
||||
|
||||
// Initialize settings for all tabs upfront to avoid reactivity loops
|
||||
let allSettings = $state<Record<string, any>>(() => {
|
||||
const initial: Record<string, any> = {};
|
||||
for (const tab in tabSettings) {
|
||||
initial[tab] = {};
|
||||
tabSettings[tab].forEach((f) => {
|
||||
initial[tab][f.name] = f.defaultValue;
|
||||
});
|
||||
}
|
||||
return initial;
|
||||
});
|
||||
|
||||
async function handleUpload(file: File, config: CsvUploadItem) {
|
||||
isUploading = true;
|
||||
activeModelTarget = config.modelTarget || null;
|
||||
scanResults = null;
|
||||
const currentSettings = allSettings[activeTab] || {};
|
||||
const companyId = companyStore.activeCompany?.id || 1;
|
||||
const opType = activeTab === 'exportacion' ? 'exp' : 'imp';
|
||||
|
||||
const res = await api.imports.upload(
|
||||
file,
|
||||
config.modelTarget || '',
|
||||
currentSettings,
|
||||
companyId,
|
||||
opType
|
||||
);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error('Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollStatus() {
|
||||
if (!currentJobId) return;
|
||||
|
||||
const res = await api.imports.status(currentJobId);
|
||||
if (res.data?.status === 'waiting_confirmation') {
|
||||
scanResults = res.data;
|
||||
showResultModal = true;
|
||||
toast.success('Escaneo completado. Revisa los resultados.');
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'failed') {
|
||||
toast.error('Error en el procesamiento: ' + (res.data.error || 'Error desconocido'));
|
||||
isUploading = false;
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
} else if (res.data?.status === 'warning') {
|
||||
// Caso cuando no se insertaron registros pero hay información de rechazo
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const totalSkipped = skippedInvalid + skippedFk;
|
||||
|
||||
if (inserted === 0) {
|
||||
toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`);
|
||||
} else {
|
||||
toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`);
|
||||
}
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'finished') {
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const skippedDetails = res.data?.skipped_details || [];
|
||||
|
||||
if (inserted > 0) {
|
||||
toast.success(`Importación completada: ${inserted} registros insertados`);
|
||||
if (skippedInvalid > 0 || skippedFk > 0) {
|
||||
const totalSkipped = skippedInvalid + skippedFk;
|
||||
toast.warning(`${totalSkipped} registros fueron rechazados`);
|
||||
}
|
||||
} else {
|
||||
toast.error('No se insertaron registros. Revisa los errores a continuación.');
|
||||
}
|
||||
isUploading = false;
|
||||
} else {
|
||||
// Continue polling
|
||||
setTimeout(pollStatus, 2000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col h-[calc(100vh-4rem)] -m-4 overflow-hidden">
|
||||
<!-- Scrollable Content Area -->
|
||||
<div class="flex-1 overflow-y-auto p-4 md:p-8 space-y-4">
|
||||
<div class="flex items-center gap-4">
|
||||
<h1 class="text-lg font-semibold md:text-2xl">Importación Masiva de Datos (CSV)</h1>
|
||||
</div>
|
||||
|
||||
<Tabs.Root bind:value={activeTab} class="w-full">
|
||||
<Tabs.List class="grid w-full grid-cols-2 md:grid-cols-4 lg:w-auto">
|
||||
<Tabs.Trigger value="catalogos">Catálogos</Tabs.Trigger>
|
||||
<Tabs.Trigger value="transportes">Transportes</Tabs.Trigger>
|
||||
<Tabs.Trigger value="importacion">Importación</Tabs.Trigger>
|
||||
<Tabs.Trigger value="exportacion">Exportación</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
|
||||
<div class="mt-6">
|
||||
<Tabs.Content value="catalogos" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Catálogos Generales</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={catalogosConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="transportes" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Logística y Transporte</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={transportesConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="importacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Importación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={importacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="exportacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Exportación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={exportacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
</div>
|
||||
</Tabs.Root>
|
||||
|
||||
<div class="h-4"></div>
|
||||
</div>
|
||||
|
||||
<!-- Fixed Footer Area -->
|
||||
{#if allSettings[activeTab]}
|
||||
<div class="flex-none z-20">
|
||||
<ConfigFooter {activeTab} bind:settings={allSettings[activeTab]} />
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<ProcessingResultModal
|
||||
bind:open={showResultModal}
|
||||
{scanResults}
|
||||
{commitResults}
|
||||
{isUploading}
|
||||
onConfirm={async () => {
|
||||
if (currentJobId && activeModelTarget) {
|
||||
try {
|
||||
isUploading = true;
|
||||
const res = await api.imports.commit(currentJobId, activeModelTarget);
|
||||
if (res.data?.commit_job_id) {
|
||||
currentJobId = res.data.commit_job_id;
|
||||
pollStatus();
|
||||
}
|
||||
} catch (err) {
|
||||
toast.error('Error al iniciar la importación');
|
||||
isUploading = false;
|
||||
}
|
||||
}
|
||||
}}
|
||||
onCancel={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
onClose={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
/>
|
||||
<script lang="ts">
|
||||
import * as Tabs from '$lib/components/ui/tabs/index.js';
|
||||
import UploadLauncherGrid from '$lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte';
|
||||
import ConfigFooter from '$lib/components/dashboard/csv-upload/ConfigFooter.svelte';
|
||||
import ProcessingResultModal from '$lib/components/dashboard/csv-upload/ProcessingResultModal.svelte';
|
||||
import {
|
||||
catalogosConfig,
|
||||
transportesConfig,
|
||||
importacionConfig,
|
||||
exportacionConfig,
|
||||
tabSettings,
|
||||
type CsvUploadItem
|
||||
} from '$lib/config/csv-upload';
|
||||
import { api } from '$lib/api';
|
||||
import { toast } from 'svelte-sonner';
|
||||
import { companyStore } from '$lib/stores/company.svelte';
|
||||
|
||||
// We no longer need modal state
|
||||
let activeTab = $state('catalogos');
|
||||
|
||||
let isUploading = $state(false);
|
||||
let currentJobId = $state<string | null>(null);
|
||||
let activeModelTarget = $state<string | null>(null);
|
||||
let scanResults = $state<any>(null);
|
||||
let commitResults = $state<any>(null);
|
||||
let showResultModal = $state(false);
|
||||
// Cuando es true, usamos API de importación de Agentes Aduanales (customs_brokers/imports)
|
||||
let useCustomsBrokerImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Clientes y Proveedores (clients_and_providers/imports)
|
||||
let useClientProviderImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Tipos de Cambio (exchange_rate/imports)
|
||||
let useExchangeRateImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Fracción Americana (us_tariff_fractions/imports)
|
||||
let useAmericanFractionImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Pedimentos (pedimentos/imports)
|
||||
let usePedimentosImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Clases de Materiales (classes/imports)
|
||||
let useMaterialClassesImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Vehículos / Transportes (vehicles/imports)
|
||||
let useVehicleImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Conductores (drivers/imports)
|
||||
let useDriverImport = $state(false);
|
||||
// Cuando es true, usamos API de importación de Trailers y Cajas (trailers/imports)
|
||||
let useTrailerImport = $state(false);
|
||||
|
||||
// Initialize settings for all tabs upfront to avoid reactivity loops
|
||||
let allSettings = $state<Record<string, any>>(() => {
|
||||
const initial: Record<string, any> = {};
|
||||
for (const tab in tabSettings) {
|
||||
initial[tab] = {};
|
||||
tabSettings[tab].forEach((f) => {
|
||||
initial[tab][f.name] = f.defaultValue;
|
||||
});
|
||||
}
|
||||
return initial;
|
||||
});
|
||||
|
||||
async function handleUpload(file: File, config: CsvUploadItem) {
|
||||
console.log('handleUpload started', { file, config });
|
||||
isUploading = true;
|
||||
activeModelTarget = config.modelTarget || null;
|
||||
scanResults = null;
|
||||
useCustomsBrokerImport = config.id === 'customs_brokers';
|
||||
useClientProviderImport = config.id === 'clients_providers';
|
||||
useExchangeRateImport = config.id === 'exchange_rates';
|
||||
useAmericanFractionImport = config.id === 'american_fractions';
|
||||
usePedimentosImport = config.id === 'pedimentos';
|
||||
useMaterialClassesImport = config.id === 'material_classes';
|
||||
useVehicleImport = config.id === 'transports';
|
||||
useDriverImport = config.id === 'drivers';
|
||||
useTrailerImport = config.id === 'trailers';
|
||||
|
||||
const companyId = companyStore.activeCompany?.id || 1;
|
||||
|
||||
if (useCustomsBrokerImport) {
|
||||
try {
|
||||
const res = await api.customsBrokerImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useClientProviderImport) {
|
||||
try {
|
||||
const res = await api.clientProviderImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useExchangeRateImport) {
|
||||
try {
|
||||
const res = await api.exchangeRateImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useAmericanFractionImport) {
|
||||
try {
|
||||
const res = await api.americanFractionImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (usePedimentosImport) {
|
||||
try {
|
||||
const res = await api.pedimentosImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useMaterialClassesImport) {
|
||||
try {
|
||||
const res = await api.materialClassImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useVehicleImport) {
|
||||
try {
|
||||
const res = await api.vehicleImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useDriverImport) {
|
||||
try {
|
||||
const res = await api.driverImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (useTrailerImport) {
|
||||
try {
|
||||
const res = await api.trailerImports.upload(file, companyId);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const currentSettings = allSettings[activeTab] || {};
|
||||
const footerConfig = { ...currentSettings };
|
||||
if (activeTab === 'importacion') {
|
||||
footerConfig.invoice_type = config.id?.startsWith('imp_def_') ? 'DEF' : 'TEM';
|
||||
}
|
||||
const opType = activeTab === 'exportacion' ? 'exp' : 'imp';
|
||||
|
||||
try {
|
||||
const res = await api.imports.upload(
|
||||
file,
|
||||
config.modelTarget || '',
|
||||
footerConfig,
|
||||
companyId,
|
||||
opType,
|
||||
config.id
|
||||
);
|
||||
if (res.data?.job_id) {
|
||||
currentJobId = res.data.job_id;
|
||||
pollStatus();
|
||||
} else {
|
||||
toast.error(res.error || 'Error al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Upload exception', e);
|
||||
toast.error('Error inesperado al subir el archivo');
|
||||
isUploading = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollStatus() {
|
||||
if (!currentJobId) return;
|
||||
|
||||
try {
|
||||
const res = useCustomsBrokerImport
|
||||
? await api.customsBrokerImports.status(currentJobId)
|
||||
: useClientProviderImport
|
||||
? await api.clientProviderImports.status(currentJobId)
|
||||
: useExchangeRateImport
|
||||
? await api.exchangeRateImports.status(currentJobId)
|
||||
: useAmericanFractionImport
|
||||
? await api.americanFractionImports.status(currentJobId)
|
||||
: usePedimentosImport
|
||||
? await api.pedimentosImports.status(currentJobId)
|
||||
: useMaterialClassesImport
|
||||
? await api.materialClassImports.status(currentJobId)
|
||||
: useVehicleImport
|
||||
? await api.vehicleImports.status(currentJobId)
|
||||
: useDriverImport
|
||||
? await api.driverImports.status(currentJobId)
|
||||
: useTrailerImport
|
||||
? await api.trailerImports.status(currentJobId)
|
||||
: await api.imports.status(currentJobId);
|
||||
console.log('Poll response', res);
|
||||
if (res.error && !res.data) {
|
||||
toast.error(res.error || 'Error al consultar el estado');
|
||||
isUploading = false;
|
||||
currentJobId = null;
|
||||
return;
|
||||
}
|
||||
if (res.data?.status === 'waiting_confirmation') {
|
||||
scanResults = res.data;
|
||||
showResultModal = true;
|
||||
toast.success('Escaneo completado. Revisa los resultados.');
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'failed' || res.data?.status === 'FAILURE') {
|
||||
const errRaw = res.data.error;
|
||||
const errText =
|
||||
typeof errRaw === 'string'
|
||||
? errRaw.includes('finished') && errRaw.includes('inserted')
|
||||
? 'La importación pudo completarse. Revisa el listado de registros.'
|
||||
: errRaw
|
||||
: (errRaw?.message ?? 'Error desconocido');
|
||||
toast.error('Error en el procesamiento: ' + errText);
|
||||
isUploading = false;
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
} else if (res.data?.status === 'warning') {
|
||||
// Caso cuando no se insertaron registros pero hay información de rechazo
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const skippedDup = res.data?.skipped_duplicate || 0;
|
||||
const totalSkipped = skippedInvalid + skippedFk + skippedDup;
|
||||
|
||||
if (inserted === 0) {
|
||||
toast.error(`No se insertaron registros. ${totalSkipped} fueron rechazados.`);
|
||||
} else {
|
||||
toast.warning(`Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`);
|
||||
}
|
||||
isUploading = false;
|
||||
} else if (res.data?.status === 'finished') {
|
||||
commitResults = res.data;
|
||||
showResultModal = true;
|
||||
const inserted = res.data?.inserted || 0;
|
||||
const skippedInvalid = res.data?.skipped_invalid || 0;
|
||||
const skippedFk = res.data?.skipped_missing_fk || 0;
|
||||
const skippedDup = res.data?.skipped_duplicate || 0;
|
||||
const skippedDetails = res.data?.skipped_details || [];
|
||||
|
||||
if (inserted > 0) {
|
||||
toast.success(`Importación completada: ${inserted} registros insertados`);
|
||||
if (skippedInvalid > 0 || skippedFk > 0 || skippedDup > 0) {
|
||||
const totalSkipped = skippedInvalid + skippedFk + skippedDup;
|
||||
toast.warning(`${totalSkipped} registros fueron rechazados`);
|
||||
}
|
||||
} else {
|
||||
toast.error('No se insertaron registros. Revisa los errores a continuación.');
|
||||
}
|
||||
isUploading = false;
|
||||
} else {
|
||||
// Continue polling
|
||||
console.log('Status not final, polling again in 2s...', res.data?.status);
|
||||
setTimeout(pollStatus, 2000);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Poll exception', e);
|
||||
// Retry on network error? Or fail?
|
||||
// For now, let's keep retrying a few times or hard fail.
|
||||
// Let's just log and retry.
|
||||
setTimeout(pollStatus, 2000);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col h-[calc(100vh-4rem)] -m-4 overflow-hidden">
|
||||
<!-- Scrollable Content Area -->
|
||||
<div class="flex-1 overflow-y-auto p-4 md:p-8 space-y-4">
|
||||
<div class="flex items-center gap-4">
|
||||
<h1 class="text-lg font-semibold md:text-2xl">Importación Masiva de Datos (CSV)</h1>
|
||||
</div>
|
||||
|
||||
<Tabs.Root bind:value={activeTab} class="w-full">
|
||||
<Tabs.List class="grid w-full grid-cols-2 md:grid-cols-4 lg:w-auto">
|
||||
<Tabs.Trigger value="catalogos">Catálogos</Tabs.Trigger>
|
||||
<Tabs.Trigger value="transportes">Transportes</Tabs.Trigger>
|
||||
<Tabs.Trigger value="importacion">Importación</Tabs.Trigger>
|
||||
<Tabs.Trigger value="exportacion">Exportación</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
|
||||
<div class="mt-6">
|
||||
<Tabs.Content value="catalogos" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Catálogos Generales</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={catalogosConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="transportes" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Logística y Transporte</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={transportesConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="importacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Importación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={importacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="exportacion" class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h2 class="text-lg font-medium tracking-tight">Operaciones de Exportación</h2>
|
||||
</div>
|
||||
<UploadLauncherGrid items={exportacionConfig} onUpload={handleUpload} />
|
||||
</Tabs.Content>
|
||||
</div>
|
||||
</Tabs.Root>
|
||||
|
||||
<div class="h-4"></div>
|
||||
</div>
|
||||
|
||||
<!-- Fixed Footer Area -->
|
||||
{#if allSettings[activeTab]}
|
||||
<div class="flex-none z-20">
|
||||
<ConfigFooter {activeTab} bind:settings={allSettings[activeTab]} />
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if scanResults || commitResults}
|
||||
<ProcessingResultModal
|
||||
bind:open={showResultModal}
|
||||
{scanResults}
|
||||
{commitResults}
|
||||
{isUploading}
|
||||
onConfirm={async () => {
|
||||
if (!currentJobId) return;
|
||||
try {
|
||||
isUploading = true;
|
||||
const res = useCustomsBrokerImport
|
||||
? await api.customsBrokerImports.commit(currentJobId)
|
||||
: useClientProviderImport
|
||||
? await api.clientProviderImports.commit(currentJobId)
|
||||
: useExchangeRateImport
|
||||
? await api.exchangeRateImports.commit(currentJobId)
|
||||
: useAmericanFractionImport
|
||||
? await api.americanFractionImports.commit(currentJobId)
|
||||
: usePedimentosImport
|
||||
? await api.pedimentosImports.commit(currentJobId)
|
||||
: useMaterialClassesImport
|
||||
? await api.materialClassImports.commit(currentJobId)
|
||||
: useVehicleImport
|
||||
? await api.vehicleImports.commit(currentJobId)
|
||||
: useDriverImport
|
||||
? await api.driverImports.commit(currentJobId)
|
||||
: useTrailerImport
|
||||
? await api.trailerImports.commit(currentJobId)
|
||||
: await api.imports.commit(currentJobId, activeModelTarget || '');
|
||||
if (res.data?.commit_job_id) {
|
||||
currentJobId = res.data.commit_job_id;
|
||||
pollStatus();
|
||||
}
|
||||
} catch (err) {
|
||||
toast.error('Error al iniciar la importación');
|
||||
isUploading = false;
|
||||
}
|
||||
}}
|
||||
onCancel={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
onClose={() => {
|
||||
currentJobId = null;
|
||||
scanResults = null;
|
||||
commitResults = null;
|
||||
showResultModal = false;
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
BIN
frontend/static/csv/EstructuraBOMS.xlsx
Normal file
BIN
frontend/static/csv/EstructuraBOMS.xlsx
Normal file
Binary file not shown.
BIN
frontend/static/csv/EstructuraCatNumerosParte.xlsx
Normal file
BIN
frontend/static/csv/EstructuraCatNumerosParte.xlsx
Normal file
Binary file not shown.
BIN
frontend/static/csv/EstructuraCatTransportistas.xlsx
Normal file
BIN
frontend/static/csv/EstructuraCatTransportistas.xlsx
Normal file
Binary file not shown.
@@ -19,7 +19,7 @@ wait_for_tcp() {
|
||||
echo "Esperando a que $service esté disponible en ${host}:${port}..."
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if python -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||
if python3 -c "import socket; s = socket.socket(); s.settimeout(3); s.connect(('$host', $port)); s.close()" 2>/dev/null; then
|
||||
echo "✓ $service está listo y accesible"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -12,11 +12,6 @@
|
||||
# 6. Relación usuario-tenant en tabla user_tenants
|
||||
# 7. Actualización del tenant_id del usuario con el valor real
|
||||
# 8. Licencia Enterprise para el tenant (ilimitada, 1 año de vigencia)
|
||||
# 9. [OPCIONAL] Datos iniciales de ejemplo si se pasa --seed-data
|
||||
#
|
||||
# Uso:
|
||||
# ./init_first_time.sh # Solo configuración básica
|
||||
# ./init_first_time.sh --seed-data # Configuración + datos de ejemplo
|
||||
#
|
||||
# Requisitos:
|
||||
# - Keycloak corriendo en http://localhost:8080
|
||||
@@ -24,6 +19,8 @@
|
||||
# - Base de datos anexo76_core creada
|
||||
# - jq instalado (para procesamiento JSON)
|
||||
#
|
||||
# Puertos: Keycloak en 18080/19000, frontend en 15173, API en 18000.
|
||||
#
|
||||
# Nota: El atributo tenant_id se crea con valor inicial "1" y luego
|
||||
# se actualiza con el ID real del tenant creado en PostgreSQL.
|
||||
###############################################################################
|
||||
@@ -33,22 +30,6 @@ set -euo pipefail # Modo strict: exit on error, undefined vars, pipe failures
|
||||
# Trap para cleanup en caso de error
|
||||
trap 'echo -e "\n${RED}✗ Error en línea $LINENO. Script abortado.${NC}" >&2' ERR
|
||||
|
||||
# Parsear argumentos
|
||||
SEED_DATA=false
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--seed-data)
|
||||
SEED_DATA=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Uso: $0 [--seed-data]"
|
||||
echo " --seed-data: Carga datos de ejemplo en las tablas"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Colores para output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
@@ -144,172 +125,7 @@ create_tenant_mapper() {
|
||||
fi
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Funciones de seed data
|
||||
###############################################################################
|
||||
|
||||
# Insertar datos de ejemplo para customs_brokers
|
||||
seed_customs_brokers() {
|
||||
echo " → Insertando customs brokers..."
|
||||
exec_pg_sql "
|
||||
INSERT INTO a76.customs_brokers (tenant_id, company_id, type, broker_key, name, address, postal_code, city, state, phone, email, country, tax_id, license, company, contact, created_at, updated_at)
|
||||
VALUES
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB001', 'Agente Aduanal García', 'Av. Reforma 123', '01000', 'Ciudad de México', 'CDMX', '5555555555', 'garcia@aduanas.com', 'MEX', 'GAAR800101ABC', '1234', 'García y Asociados', 'Juan García', now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB002', 'Agente Aduanal López', 'Blvd. Díaz Ordaz 456', '22000', 'Tijuana', 'BC', '6641234567', 'lopez@customs.com', 'MEX', 'LOPL750505XYZ', '2345', 'López Customs', 'María López', now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'persona', 'CB003', 'Agente Aduanal Martínez', 'Calle Industria 789', '45000', 'Guadalajara', 'JAL', '3339876543', 'martinez@broker.com', 'MEX', 'MARM850315DEF', '3456', 'Martínez Brokerage', 'Pedro Martínez', now(), now())
|
||||
ON CONFLICT (broker_key, tenant_id, company_id) DO NOTHING;
|
||||
" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Insertar datos de ejemplo para clients_and_providers
|
||||
seed_clients_and_providers() {
|
||||
echo " → Insertando clientes y proveedores..."
|
||||
exec_pg_sql "
|
||||
INSERT INTO a76.clients_and_providers (tenant_id, company_id, type_nat_foreign, name, short_name, rfc, client_or_provider, web_key, is_active, created_at, updated_at)
|
||||
VALUES
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'N', 'Proveedor Tecnológico SA de CV', 'PROVTECH', 'PTE901201ABC', 'BOTH', 'PROV001', true, now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'N', 'Cliente Industrial del Norte SA', 'CINORTE', 'CIN850615XYZ', 'BOTH', 'CLI001', true, now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'E', 'Global Supplies Inc', 'GLOBSUP', 'GSI123456789', 'BOTH', 'BOTH001', true, now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'N', 'Manufacturas del Bajío SA', 'MANBAJIO', 'MDB920310DEF', 'BOTH', 'CLI002', true, now(), now())
|
||||
RETURNING id;
|
||||
" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Insertar datos de ejemplo para packages
|
||||
seed_packages() {
|
||||
echo " → Insertando tipos de paquete..."
|
||||
exec_pg_sql "
|
||||
INSERT INTO a76.packages (tenant_id, company_id, key, description_es, description_en, weight_unit, plurals, plural_in, code_ace, code_aamex, created_at, updated_at)
|
||||
VALUES
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'PK01', 'Caja de Cartón', 'Cardboard Box', 0.5, 'CAJS', 'BOXS', 'CB01', 'CAJA001', now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'PK02', 'Pallet de Madera', 'Wooden Pallet', 15.0, 'PLTS', 'PLTS', 'WP01', 'PALL001', now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'PK03', 'Tambor Metálico', 'Metal Drum', 10.0, 'TMBS', 'DRMS', 'MD01', 'TAMB001', now(), now()),
|
||||
(${TENANT_ID}, ${COMPANY_ID}, 'PK04', 'Contenedor', 'Container', 2000.0, 'CONT', 'CONT', 'CT01', 'CONT001', now(), now())
|
||||
ON CONFLICT (tenant_id, company_id, key) DO NOTHING;
|
||||
" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Insertar datos de ejemplo para classes
|
||||
seed_classes() {
|
||||
echo " → Insertando clases..."
|
||||
exec_pg_sql "INSERT INTO a76.classes (tenant_id, company_id, class_code, description_es, description_en, material_key, unit_of_measure, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'CLS001', 'Componentes Electrónicos', 'Electronic Components', 'MP', 'PZA', '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS002', 'Partes Automotrices', 'Automotive Parts', 'MP', 'KGS', '8708.29.99', '8708.29.9900', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS003', 'Textiles y Telas', 'Textiles and Fabrics', 'MP', 'MT', '5407.20.01', '5407.20.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'CLS004', 'Equipo de Computación', 'Computer Equipment', 'MP', 'PZA', '8471.30.01', '8471.30.0100', now(), now()) ON CONFLICT (tenant_id, company_id, class_code) DO NOTHING;" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Insertar datos de ejemplo para parts
|
||||
seed_parts() {
|
||||
echo " → Insertando partes/componentes..."
|
||||
|
||||
# Obtener un client_id para asociar las partes
|
||||
local client_id
|
||||
client_id=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} LIMIT 1;" | xargs)
|
||||
|
||||
if [ -n "$client_id" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.parts (tenant_id, company_id, client_id, part_number, description_spanish, description_english, part_class, currency_key, unit_of_measure, unit_cost, fraction, us_fraction, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-001', 'Microcontrolador ARM Cortex-M4', 'ARM Cortex-M4 Microcontroller', 'CLS001', 'USD', 'PZA', 15.50, '8542.31.01', '8542.31.0000', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-002', 'Filtro de Aceite Automotriz', 'Automotive Oil Filter', 'CLS002', 'USD', 'PZA', 8.75, '8421.23.01', '8421.23.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-003', 'Tela de Algodón para Tapicería', 'Cotton Upholstery Fabric', 'CLS003', 'USD', 'MT', 12.00, '5208.31.01', '5208.31.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-004', 'Disco Duro SSD 500GB', '500GB SSD Hard Drive', 'CLS004', 'USD', 'PZA', 65.00, '8471.70.01', '8471.70.0100', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, ${client_id}, 'PART-005', 'Sensor de Temperatura Digital', 'Digital Temperature Sensor', 'CLS001', 'USD', 'PZA', 5.25, '9025.19.01', '9025.19.0100', now(), now()) ON CONFLICT (tenant_id, company_id, part_number) DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
|
||||
# Insertar datos de ejemplo para pedimentos
|
||||
seed_pedimentos() {
|
||||
echo " → Insertando pedimentos..."
|
||||
|
||||
# Primero obtener IDs de clientes
|
||||
local client_ids
|
||||
client_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('CLIENT', 'BOTH') LIMIT 2;")
|
||||
local client_id_1=$(echo "$client_ids" | sed -n '1p' | xargs)
|
||||
local client_id_2=$(echo "$client_ids" | sed -n '2p' | xargs)
|
||||
|
||||
if [ -n "$client_id_1" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001234', ${client_id_1}, 'imp', 'normal', 'V1', 'ITE', 'draft', now(), now()), (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001235', ${client_id_1}, 'exp', 'normal', 'V1', 'ETE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$client_id_2" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.pedimentos (tenant_id, company_id, year, customs_office, license, pedimento_number, client_id, operation_type, pedimento_type, pedimento_code, regime, status, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, '24', '47', '3807', '8001236', ${client_id_2}, 'imp', 'consolidated', 'V1', 'ITE', 'draft', now(), now()) ON CONFLICT (tenant_id, company_id, year, customs_office, license, pedimento_number) DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
|
||||
# Insertar datos de ejemplo para invoices
|
||||
seed_invoices() {
|
||||
echo " → Insertando facturas..."
|
||||
|
||||
# Obtener IDs de clientes/proveedores
|
||||
local provider_ids
|
||||
provider_ids=$(exec_pg_sql "SELECT id FROM a76.clients_and_providers WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND client_or_provider IN ('PROVIDER', 'BOTH') LIMIT 2;")
|
||||
local provider_id_1=$(echo "$provider_ids" | sed -n '1p' | xargs)
|
||||
local provider_id_2=$(echo "$provider_ids" | sed -n '2p' | xargs)
|
||||
|
||||
# Insertar facturas en invoice_header (mínimo requerido)
|
||||
exec_pg_sql "INSERT INTO a76.invoice_header (tenant_id, company_id, system, operation_type, invoice_type, invoice_number, invoice_date, is_updated, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'DEF', 'INV-2024-001', '2024-01-10', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'exp', 'EXDEF', 'INV-2024-002', '2024-02-15', false, now(), now()), (${TENANT_ID}, ${COMPANY_ID}, 'fixed_asset', 'imp', 'TEM', 'INV-2024-003', '2024-03-05', false, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
|
||||
# Obtener IDs de las facturas recién creadas
|
||||
local invoice_ids
|
||||
invoice_ids=$(exec_pg_sql "SELECT id FROM a76.invoice_header WHERE tenant_id = ${TENANT_ID} AND company_id = ${COMPANY_ID} AND invoice_number IN ('INV-2024-001', 'INV-2024-002', 'INV-2024-003') ORDER BY invoice_number;")
|
||||
local invoice_id_1=$(echo "$invoice_ids" | sed -n '1p' | xargs)
|
||||
local invoice_id_2=$(echo "$invoice_ids" | sed -n '2p' | xargs)
|
||||
local invoice_id_3=$(echo "$invoice_ids" | sed -n '3p' | xargs)
|
||||
|
||||
# Insertar datos financieros para las facturas
|
||||
if [ -n "$invoice_id_1" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'foreign', 17.50, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$invoice_id_2" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'foreign', 17.45, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$invoice_id_3" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_financials (tenant_id, company_id, invoice_id, currency, exchange_rate, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'local', 1.00, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
# Insertar datos de compliance mexicano con proveedores
|
||||
if [ -n "$invoice_id_1" ] && [ -n "$provider_id_1" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$invoice_id_2" ] && [ -n "$provider_id_1" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, ${provider_id_1}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$invoice_id_3" ] && [ -n "$provider_id_2" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_compliance_mx (tenant_id, company_id, invoice_id, provider_id, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, ${provider_id_2}, now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
# Insertar datos de logística con incoterm
|
||||
if [ -n "$invoice_id_1" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_1}, 'none', 'kgs', 'FOB', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$invoice_id_2" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_2}, 'none', 'kgs', 'CIF', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ -n "$invoice_id_3" ]; then
|
||||
exec_pg_sql "INSERT INTO a76.invoice_logistics (tenant_id, company_id, invoice_id, transport_type, weight_type, incoterm, created_at, updated_at) VALUES (${TENANT_ID}, ${COMPANY_ID}, ${invoice_id_3}, 'none', 'kgs', 'EXW', now(), now()) ON CONFLICT DO NOTHING;" >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
|
||||
# Ejecutar todas las funciones de seed
|
||||
execute_seed_data() {
|
||||
echo -e "\n${YELLOW}[SEED] Cargando datos de ejemplo...${NC}"
|
||||
|
||||
seed_customs_brokers
|
||||
seed_clients_and_providers
|
||||
seed_packages
|
||||
seed_classes
|
||||
seed_parts
|
||||
seed_pedimentos
|
||||
seed_invoices
|
||||
|
||||
echo -e "${GREEN}✓ Datos de ejemplo cargados exitosamente${NC}"
|
||||
echo -e "${YELLOW} • 3 Agentes aduanales${NC}"
|
||||
echo -e "${YELLOW} • 4 Clientes/Proveedores${NC}"
|
||||
echo -e "${YELLOW} • 4 Tipos de paquete${NC}"
|
||||
echo -e "${YELLOW} • 4 Clases${NC}"
|
||||
echo -e "${YELLOW} • 5 Parts/Componentes${NC}"
|
||||
echo -e "${YELLOW} • 3 Pedimentos${NC}"
|
||||
echo -e "${YELLOW} • 3 Facturas${NC}"
|
||||
}
|
||||
|
||||
# Variables de configuración
|
||||
# Variables de configuración (puertos con prefijo 1 hardcodeados)
|
||||
KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080/kcauth}"
|
||||
KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}"
|
||||
KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}"
|
||||
@@ -328,7 +144,7 @@ DEMO_EMAIL="demo@aduanasoft.com"
|
||||
DEMO_FIRSTNAME="Demo"
|
||||
DEMO_LASTNAME="User"
|
||||
|
||||
TENANT_NAME="Aduanasoft"
|
||||
TENANT_NAME="Aduanasoft"A
|
||||
TENANT_SLUG="aduanasoft"
|
||||
COMPANY_NAME="Aduanasoft S.A. de C.V."
|
||||
COMPANY_RFC="ADS010101AAA"
|
||||
@@ -797,12 +613,14 @@ else
|
||||
echo -e "${YELLOW}⚠ Company ya existe para este tenant${NC}"
|
||||
fi
|
||||
|
||||
# Obtener información de la company
|
||||
COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;")
|
||||
# Obtener ID de la company (necesario para user_tenants)
|
||||
COMPANY_ID=$(exec_pg_sql "SELECT id FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;" | xargs)
|
||||
|
||||
echo -e "${GREEN}✓ Company: ${COMPANY_INFO}${NC}"
|
||||
echo -e "${GREEN}✓ Company ID: ${COMPANY_ID}${NC}"
|
||||
if [ -z "$COMPANY_ID" ]; then
|
||||
echo -e "${RED}✗ Error: No se pudo obtener el ID de la company${NC}"
|
||||
exit 1
|
||||
fi
|
||||
COMPANY_INFO=$(exec_pg_sql "SELECT id, name FROM a76.company WHERE tenant_id = ${TENANT_ID} LIMIT 1;")
|
||||
echo -e "${GREEN}✓ Company ID: ${COMPANY_ID} | ${COMPANY_INFO}${NC}"
|
||||
|
||||
# Agregar tenant_id al usuario demo en Keycloak
|
||||
echo -e "\n${YELLOW}Asignando tenant_id al usuario demo...${NC}"
|
||||
@@ -818,7 +636,7 @@ curl -s -X PUT "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/users/${USER_ID}"
|
||||
|
||||
echo -e "${GREEN}✓ Atributo tenant_id asignado al usuario${NC}"
|
||||
|
||||
# Agregar relación usuario-tenant en la base de datos
|
||||
# Agregar relación usuario-tenant en la base de datos (usar company_id real)
|
||||
echo -e "\n${YELLOW}Creando relación usuario-tenant en la base de datos...${NC}"
|
||||
|
||||
exec_pg_sql "INSERT INTO core.user_tenants (keycloak_user_id, tenant_id, company_id, role, is_active, created_at, updated_at) VALUES ('${USER_ID}', ${TENANT_ID}, 1, 'admin', true, now(), now()) ON CONFLICT (keycloak_user_id, tenant_id, company_id) DO UPDATE SET is_active = true, updated_at = CURRENT_TIMESTAMP;" >/dev/null
|
||||
@@ -880,13 +698,6 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# 9. Cargar datos de ejemplo (opcional)
|
||||
###############################################################################
|
||||
if [ "$SEED_DATA" = true ]; then
|
||||
execute_seed_data
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# Resumen final
|
||||
###############################################################################
|
||||
@@ -925,18 +736,6 @@ echo -e " ${GREEN}✓${NC} Plan: Enterprise (ilimitado)"
|
||||
echo -e " ${GREEN}✓${NC} Status: Activa"
|
||||
echo -e " ${GREEN}✓${NC} Features: API, Reportes Avanzados, Integraciones, Soporte Dedicado"
|
||||
echo -e " ${GREEN}✓${NC} Vigencia: 1 año"
|
||||
|
||||
if [ "$SEED_DATA" = true ]; then
|
||||
echo -e "\n${YELLOW}Datos de ejemplo:${NC}"
|
||||
echo -e " ${GREEN}✓${NC} Agentes aduanales: 3"
|
||||
echo -e " ${GREEN}✓${NC} Clientes/Proveedores: 4"
|
||||
echo -e " ${GREEN}✓${NC} Tipos de paquete: 4"
|
||||
echo -e " ${GREEN}✓${NC} Clases: 4"
|
||||
echo -e " ${GREEN}✓${NC} Parts/Componentes: 5"
|
||||
echo -e " ${GREEN}✓${NC} Pedimentos: 3"
|
||||
echo -e " ${GREEN}✓${NC} Facturas: 3"
|
||||
fi
|
||||
|
||||
echo -e "\n${YELLOW}Puedes acceder al sistema en:${NC}"
|
||||
echo -e " ${GREEN}http://localhost:5173${NC}"
|
||||
echo -e "\n${GREEN}════════════════════════════════════════════════════════${NC}\n"
|
||||
|
||||
Reference in New Issue
Block a user