Merge pull request 'feature/validaciones-clarion-csv-transportistas' (#191) from feature/validaciones-clarion-csv-transportistas into development
Reviewed-on: ADUANASOFT/anexo76#191
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -30,6 +30,7 @@ wheels/
|
||||
backend/.env
|
||||
frontend/.env
|
||||
backend/SCRIPTS/
|
||||
.cursor/
|
||||
|
||||
# IDEs
|
||||
.vscode/
|
||||
@@ -68,6 +69,7 @@ htmlcov/
|
||||
*.dockerignore
|
||||
postgres-data/
|
||||
backend/uploads/
|
||||
backend/layouts/imports/
|
||||
docker-compose.yml
|
||||
.mypy_cache/
|
||||
|
||||
|
||||
19
README.md
19
README.md
@@ -42,14 +42,24 @@ anexo76/
|
||||
│ ├── .env.example # Variables de entorno de ejemplo
|
||||
│ ├── core/ # Módulos core
|
||||
│ │ ├── config.py # Configuración centralizada
|
||||
│ │ ├── paths.py # Rutas base y layout_path() para importación CSV
|
||||
│ │ ├── database.py # Configuración de BD multi-tenant
|
||||
│ │ ├── security.py # Autenticación y autorización
|
||||
│ │ └── middleware.py # Middlewares personalizados
|
||||
│ ├── layouts/ # Directorio de datos: temp y errors de importación CSV (creado al arranque)
|
||||
│ └── api/
|
||||
│ └── v1/
|
||||
│ ├── router.py # Router principal API v1
|
||||
│ └── modules/ # Módulos de negocio
|
||||
│ ├── auth/ # Autenticación
|
||||
│ └── a76/
|
||||
│ ├── layouts_csv/ # Lógica centralizada de cargas por CSV (routes, tasks, validaciones por proceso)
|
||||
│ │ ├── facturas/
|
||||
│ │ ├── customs_brokers/
|
||||
│ │ ├── clients_and_providers/
|
||||
│ │ └── ... # Una carpeta por proceso de importación
|
||||
│ ├── csv_templates/
|
||||
│ └── ...
|
||||
│ ├── auth/
|
||||
│ ├── tenants/ # Gestión de tenants
|
||||
│ ├── licenses/ # Control de licencias
|
||||
│ └── ...
|
||||
@@ -136,6 +146,11 @@ python -c "from core.database import init_db; init_db()"
|
||||
|
||||
```
|
||||
docker build -t dev.aduanasoft.com/anexo76/backend:latest -f ./backend/Dockerfile ./backend
|
||||
```
|
||||
|
||||
**Importación CSV y workers Celery**: La lógica de cargas por CSV está en `api/v1/modules/a76/layouts_csv/` (una carpeta por proceso). Los workers Celery deben cargar los módulos `api.v1.modules.a76.layouts_csv.<proceso>.tasks`. El directorio de trabajo del worker debe ser la raíz del backend para que `layout_path("imports", "temp")` y `layout_path("imports", "errors")` apunten a los mismos directorios que la API. Si API y worker comparten volumen, los archivos temporales y de errores se escriben en `backend/layouts/imports/temp` y `backend/layouts/imports/errors`.
|
||||
|
||||
```
|
||||
docker build \
|
||||
--build-arg VITE_API_URL=https://anexo76-dev.aduanasoft.com/api/ \
|
||||
--build-arg VITE_KEYCLOAK_URL=https://anexo76-dev.aduanasoft.com/kcauth/ \
|
||||
@@ -165,6 +180,8 @@ pip install -r requirements.txt
|
||||
uvicorn main:app --reload
|
||||
```
|
||||
|
||||
Los directorios `backend/layouts/imports/temp` y `backend/layouts/imports/errors` se crean automáticamente al arrancar el backend para la importación CSV.
|
||||
|
||||
### Frontend
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,430 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y devuelve resultado; el mapeo a tabla se añadirá cuando exista.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"bom_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{BOM_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila BOM según columnas del template. FK opcional a a76.parts."""
|
||||
parent = (row.get("NUMPARTE_PADRE") or "").strip()
|
||||
if not parent:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Requerido"}
|
||||
if len(parent) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
component = (row.get("NUMPARTE_COMPONENTE") or "").strip()
|
||||
if not component:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Requerido"}
|
||||
if len(component) > 70:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Máximo 70 caracteres"}
|
||||
|
||||
qty = row.get("CANTIDAD")
|
||||
if qty is None or qty == "":
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Requerido"}
|
||||
try:
|
||||
val = Decimal(str(qty))
|
||||
if val < 0:
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": "CANTIDAD", "msg": "Debe ser número"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom and len(uom) > 10:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
version_bom = row.get("VERSION_BOM")
|
||||
if not _optional_number(version_bom):
|
||||
return {"line": line_num, "col": "VERSION_BOM", "msg": "Debe ser número"}
|
||||
|
||||
version_bill = row.get("VERSION_BILL")
|
||||
if not _optional_number(version_bill):
|
||||
return {"line": line_num, "col": "VERSION_BILL", "msg": "Debe ser número"}
|
||||
|
||||
# Solo exigir que padre/componente existan en catálogo si hay partes cargadas (evita rechazar todo cuando el catálogo está vacío o en pruebas)
|
||||
if valid_part_numbers is not None and len(valid_part_numbers) > 0:
|
||||
if parent not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_PADRE", "msg": "Parte padre no existe en catálogo"}
|
||||
if component not in valid_part_numbers:
|
||||
return {"line": line_num, "col": "NUMPARTE_COMPONENTE", "msg": "Parte componente no existe en catálogo"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"BOMs import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts for FK validation: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"BOMs import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"bom_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"bom_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{BOM_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"BOMs import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: could not load parts: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
valid_count = 0
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
# Cuando exista la tabla de destino, aquí se hará insert/update.
|
||||
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = f"WIP: {valid_count} filas válidas. La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"BOMs import cleanup failed: {cleanup_err}")
|
||||
|
||||
return response
|
||||
@@ -5,7 +5,7 @@ Mismo patrón que parts y classes: upload → scan → status → commit.
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.boms.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@@ -1,528 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
if not class_code:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
|
||||
if len(class_code) > 8:
|
||||
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
|
||||
|
||||
desc_es = (row.get("DESCRIPCIONE") or "").strip()
|
||||
if desc_es and len(desc_es) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
|
||||
desc_en = (row.get("DESCRIPCIONI") or "").strip()
|
||||
if desc_en and len(desc_en) > 500:
|
||||
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
|
||||
|
||||
material_key = (row.get("CLAVEMAT") or "").strip()
|
||||
if material_key:
|
||||
if len(material_key) > 10:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
|
||||
if valid_material_keys is not None and material_key not in valid_material_keys:
|
||||
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
|
||||
|
||||
uom = (row.get("UNIMED") or "").strip()
|
||||
if uom:
|
||||
if len(uom) > 5:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
|
||||
if valid_uom_codes is not None and uom not in valid_uom_codes:
|
||||
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
|
||||
|
||||
fraction = (row.get("FRACCION") or "").strip()
|
||||
if fraction and len(fraction) > 20:
|
||||
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
|
||||
us_fraction = (row.get("FRACCIONAME") or "").strip()
|
||||
if us_fraction and len(us_fraction) > 16:
|
||||
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
|
||||
sub_key = (row.get("CLAVESUB") or "").strip()
|
||||
if sub_key and len(sub_key) > 5:
|
||||
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
|
||||
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if iva_exempt and len(iva_exempt) > 4:
|
||||
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
|
||||
|
||||
rev_fisica = row.get("REVFISICA")
|
||||
if rev_fisica is not None and rev_fisica != "":
|
||||
try:
|
||||
v = int(rev_fisica)
|
||||
if v < -32768 or v > 32767:
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info(f"Classes import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info(f"Classes import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
for m in session.query(MaterialType.key).all():
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_uom_codes.add(u[0])
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: could not load FK sets: {e}")
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code: Dict[str, Class] = {}
|
||||
for c in (
|
||||
session.query(Class)
|
||||
.filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_code[c.class_code] = c
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_class(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
|
||||
)
|
||||
continue
|
||||
|
||||
class_code = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
if existing:
|
||||
existing.description_es = desc_es
|
||||
existing.description_en = desc_en
|
||||
existing.material_key = material_key
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.fraction = fraction
|
||||
existing.us_fraction = us_fraction
|
||||
existing.sub_key = sub_key
|
||||
existing.physical_review = physical_review
|
||||
existing.iva_exempt_fraction = iva_exempt_fraction
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
class_code=class_code,
|
||||
description_es=desc_es,
|
||||
description_en=desc_en,
|
||||
material_key=material_key,
|
||||
unit_of_measure=unit_of_measure,
|
||||
fraction=fraction,
|
||||
us_fraction=us_fraction,
|
||||
sub_key=sub_key,
|
||||
physical_review=physical_review,
|
||||
iva_exempt_fraction=iva_exempt_fraction,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"Classes import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -1,45 +0,0 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION", "DESCRIPCION ES"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION EN", "DESCRIPTION"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["UNIDAD MEDIDA", "UNIT", "UOM"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION MEX"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION USA", "US FRACTION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REV FISICA", "PHYSICAL REVIEW"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["EXENTA IVA", "FRACCION EXENTA IVA"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -12,7 +12,7 @@ from api.v1.common.tenant_crud_routes import TenantCRUDRoutes, validate_access_t
|
||||
|
||||
from .dto import ClassCreateDTO, ClassCreateDTOFA, ClassResponseDTO, ClassResponseDTOFA, ClassUpdateDTO, ClassWithFADataResponse
|
||||
from .service import ClassService
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.classes.routes import router as imports_router
|
||||
|
||||
# Create a new router for custom endpoints
|
||||
router = APIRouter()
|
||||
|
||||
@@ -1,500 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientOrProviderEnum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con cb_ ni a76.imports)
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cp_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CP_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
"""Mapea valor CSV a ClientOrProviderEnum. Retorna None si no reconocido."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
v = str(val).strip().lower()
|
||||
if v in ("client", "cliente", "c"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor", "p"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos", "b", "cliente y proveedor"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active(val: Optional[str]) -> bool:
|
||||
"""Interpreta ACTIVO: 1/true/si/yes -> True, 0/false/no -> False. Default True."""
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_row_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Cliente/Proveedor. Retorna error dict o None."""
|
||||
rfc = (row.get("RFC") or "").strip()
|
||||
if not rfc:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Requerido"}
|
||||
if len(rfc) > 30:
|
||||
return {"line": line_num, "col": "RFC", "msg": "Máximo 30 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if tipo_raw and _parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente, Proveedor o Ambos.",
|
||||
}
|
||||
|
||||
name = (row.get("NOMBRE") or "").strip()
|
||||
if len(name) > 256:
|
||||
return {"line": line_num, "col": "NOMBRE", "msg": "Máximo 256 caracteres"}
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
if short_name and len(short_name) > 10:
|
||||
return {"line": line_num, "col": "SHORT_NAME", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
curp = (row.get("CURP") or "").strip()
|
||||
if curp and len(curp) > 19:
|
||||
return {"line": line_num, "col": "CURP", "msg": "Máximo 19 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CP import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ClientProvider.
|
||||
Upsert por (tenant_id, company_id, rfc).
|
||||
"""
|
||||
logger.info(f"CP import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cp_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cp_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CP_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CP import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
# Cargar existentes por (tenant_id, company_id, rfc); rfc puede ser None en BD, usamos '' como key
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
key = (cp.rfc or "").strip()
|
||||
existing_by_rfc[key] = cp
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_client_provider(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
rfc = _str_or_none(row_norm.get("RFC"), 30)
|
||||
if not rfc:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
|
||||
client_or_provider = _parse_client_or_provider(row_norm.get("TIPO"))
|
||||
if client_or_provider is None:
|
||||
client_or_provider = ClientOrProviderEnum.BOTH
|
||||
|
||||
is_active = _parse_active(row_norm.get("ACTIVO"))
|
||||
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 256)
|
||||
existing.short_name = _str_or_none(row_norm.get("SHORT_NAME"), 10)
|
||||
existing.curp = _str_or_none(row_norm.get("CURP"), 19)
|
||||
existing.client_or_provider = client_or_provider
|
||||
existing.responsible = _str_or_none(row_norm.get("RESPONSABLE"), 80)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
existing.incoterm = _str_or_none(row_norm.get("INCOTERM"), 19)
|
||||
existing.is_active = is_active
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_cp = ClientProvider(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
rfc=rfc,
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 256),
|
||||
short_name=_str_or_none(row_norm.get("SHORT_NAME"), 10),
|
||||
curp=_str_or_none(row_norm.get("CURP"), 19),
|
||||
client_or_provider=client_or_provider,
|
||||
responsible=_str_or_none(row_norm.get("RESPONSABLE"), 80),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
incoterm=_str_or_none(row_norm.get("INCOTERM"), 19),
|
||||
is_active=is_active,
|
||||
)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
inserted_count += 1
|
||||
|
||||
# Opcional: crear dirección si hay email/teléfono/dirección
|
||||
email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), 100)
|
||||
if email or phone or address_str:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_str,
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
phone=phone,
|
||||
email=email,
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 50),
|
||||
)
|
||||
session.add(addr)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CP import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CP import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CP import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -1,56 +0,0 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls).
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Definir cabeceras según la primera fila del XLS oficial (frontend/static/csv/EstructuraCatClienteProv.xls).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"client_providers": [
|
||||
{"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]},
|
||||
{"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
|
||||
{"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]},
|
||||
{"canonical": "CURP", "aliases": []},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO", "DIRECCION FISCAL", "CALLE"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]},
|
||||
{"canonical": "ESTADO", "aliases": []},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]},
|
||||
{"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "INCOTERM", "aliases": []},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla client_providers."""
|
||||
cols = TEMPLATE_COLUMNS.get("client_providers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -20,7 +20,7 @@ from .dto import (
|
||||
)
|
||||
from .service import ClientProviderService
|
||||
from .models import ClientProvider
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.clients_and_providers.routes import router as imports_router
|
||||
|
||||
# Create main router to add custom endpoints
|
||||
router = APIRouter(prefix="/clients-providers")
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from .routes import router
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("client", "access_token")
|
||||
def test_list_clients_and_providers(client, access_token):
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
response = client.get("/client_and_provider/", headers=headers)
|
||||
assert response.status_code == 200
|
||||
assert "items" in response.json()
|
||||
assert "page" in response.json()
|
||||
assert "page_size" in response.json()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("client", "access_token")
|
||||
def test_get_client_or_provider_not_found(client, access_token):
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
response = client.get("/client_and_provider/invalid_id", headers=headers)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_create_client_or_provider_forbidden():
|
||||
response = client.post(
|
||||
"/client_and_provider/", json={"name": "Test Client/Provider"}
|
||||
)
|
||||
assert response.status_code in (403, 405, 404)
|
||||
|
||||
|
||||
def test_update_client_or_provider_forbidden():
|
||||
response = client.put(
|
||||
"/client_and_provider/1", json={"name": "Updated Client/Provider"}
|
||||
)
|
||||
assert response.status_code in (403, 405, 404)
|
||||
@@ -7,37 +7,46 @@ import io
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# Importar configs de cada módulo
|
||||
from api.v1.modules.a76.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.facturas.template_config import (
|
||||
TEMPLATE_COLUMNS as IMPORTS_TEMPLATE_COLUMNS,
|
||||
_resolve_template_columns as resolve_imports_template,
|
||||
)
|
||||
from api.v1.modules.a76.parts.imports.template_config import TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.boms.imports.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.classes.imports.template_config import TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.customs_brokers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.parts.template_config import (
|
||||
TEMPLATE_COLUMNS as PARTS_TEMPLATE_COLUMNS,
|
||||
TEMPLATE_DOWNLOAD_HEADERS as PARTS_TEMPLATE_DOWNLOAD_HEADERS,
|
||||
)
|
||||
from api.v1.modules.a76.layouts_csv.boms.template_config import TEMPLATE_COLUMNS as BOMS_TEMPLATE_COLUMNS
|
||||
from api.v1.modules.a76.layouts_csv.classes.template_config import (
|
||||
TEMPLATE_COLUMNS as CLASSES_TEMPLATE_COLUMNS,
|
||||
TEMPLATE_DOWNLOAD_HEADERS as CLASSES_TEMPLATE_DOWNLOAD_HEADERS,
|
||||
)
|
||||
from api.v1.modules.a76.layouts_csv.customs_brokers.template_config import (
|
||||
TEMPLATE_COLUMNS as CUSTOMS_BROKERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.clients_and_providers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.clients_and_providers.template_config import (
|
||||
TEMPLATE_COLUMNS as CLIENTS_PROVIDERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.exchange_rate.template_config import (
|
||||
TEMPLATE_COLUMNS as EXCHANGE_RATE_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.us_tariff_fractions.template_config import (
|
||||
TEMPLATE_COLUMNS as US_TARIFF_FRACTIONS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.pedmientos.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.pedmientos.template_config import (
|
||||
TEMPLATE_COLUMNS as PEDIMENTOS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.vehicles.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.vehicles.template_config import (
|
||||
TEMPLATE_COLUMNS as VEHICLES_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.drivers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.drivers.template_config import (
|
||||
TEMPLATE_COLUMNS as DRIVERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.transportation.trailers.imports.template_config import (
|
||||
from api.v1.modules.a76.layouts_csv.trailers.template_config import (
|
||||
TEMPLATE_COLUMNS as TRAILERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
from api.v1.modules.a76.layouts_csv.transportistas.template_config import (
|
||||
TEMPLATE_COLUMNS as TRANSPORTERS_TEMPLATE_COLUMNS,
|
||||
)
|
||||
|
||||
|
||||
def _canonicals_from_columns(cols: Optional[List[Dict]]) -> List[str]:
|
||||
@@ -56,15 +65,26 @@ def _build_registry() -> Dict[str, List[str]]:
|
||||
registry[tid] = _canonicals_from_columns(cols)
|
||||
|
||||
# part_numbers (parts); "items" usa la misma plantilla
|
||||
part_cols = PARTS_TEMPLATE_COLUMNS.get("part_numbers")
|
||||
registry["part_numbers"] = _canonicals_from_columns(part_cols)
|
||||
registry["items"] = _canonicals_from_columns(part_cols)
|
||||
registry["part_numbers"] = (
|
||||
PARTS_TEMPLATE_DOWNLOAD_HEADERS
|
||||
if PARTS_TEMPLATE_DOWNLOAD_HEADERS
|
||||
else _canonicals_from_columns(PARTS_TEMPLATE_COLUMNS.get("part_numbers"))
|
||||
)
|
||||
registry["items"] = (
|
||||
PARTS_TEMPLATE_DOWNLOAD_HEADERS
|
||||
if PARTS_TEMPLATE_DOWNLOAD_HEADERS
|
||||
else _canonicals_from_columns(PARTS_TEMPLATE_COLUMNS.get("part_numbers"))
|
||||
)
|
||||
|
||||
# boms
|
||||
registry["boms"] = _canonicals_from_columns(BOMS_TEMPLATE_COLUMNS.get("boms"))
|
||||
|
||||
# material_classes
|
||||
registry["material_classes"] = _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes"))
|
||||
# material_classes: cabeceras de descarga según plantilla usuario (CLAVE, CLASE, DESCRIPCION, etc.)
|
||||
registry["material_classes"] = (
|
||||
CLASSES_TEMPLATE_DOWNLOAD_HEADERS
|
||||
if CLASSES_TEMPLATE_DOWNLOAD_HEADERS
|
||||
else _canonicals_from_columns(CLASSES_TEMPLATE_COLUMNS.get("material_classes"))
|
||||
)
|
||||
|
||||
# customs_brokers
|
||||
registry["customs_brokers"] = _canonicals_from_columns(CUSTOMS_BROKERS_TEMPLATE_COLUMNS.get("customs_brokers"))
|
||||
@@ -92,6 +112,9 @@ def _build_registry() -> Dict[str, List[str]]:
|
||||
# trailers
|
||||
registry["trailers"] = _canonicals_from_columns(TRAILERS_TEMPLATE_COLUMNS.get("trailers"))
|
||||
|
||||
# transporters (transportistas)
|
||||
registry["transporters"] = _canonicals_from_columns(TRANSPORTERS_TEMPLATE_COLUMNS.get("transporters"))
|
||||
|
||||
return registry
|
||||
|
||||
|
||||
@@ -111,6 +134,7 @@ TEMPLATE_FILENAMES: Dict[str, str] = {
|
||||
"transports": "EstructuraCatTransportes.csv",
|
||||
"drivers": "EstructuraCatConductor.csv",
|
||||
"trailers": "EstructuraCatTrailers.csv",
|
||||
"transporters": "EstructuraCatTransportistas.csv",
|
||||
"imp_temp_header": "EstructuraEncFacImpoTemp.csv",
|
||||
"imp_temp_details": "EstructuraParFacImpoTempAF.csv",
|
||||
"imp_def_header": "EstructuraEncFacImpoDef.csv",
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# CSV import flow for Agentes Aduanales (Customs Brokers).
|
||||
# Replicates the same two-phase flow as a76.imports: upload → scan → waiting_confirmation → commit.
|
||||
@@ -1,447 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis keys (prefijo propio para no colisionar con a76.imports)
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"cb_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{CB_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _validate_row_customs_broker(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Agente Aduanal. Retorna error dict o None."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Requerido"}
|
||||
if len(clave) > 5:
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Máximo 5 caracteres"}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {"line": line_num, "col": "CLAVE", "msg": "Solo letras y números"}
|
||||
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if licencia and (len(licencia) > 4 or not licencia.isdigit()):
|
||||
return {"line": line_num, "col": "LICENCIA", "msg": "Máximo 4 dígitos numéricos"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"CB import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Guardar números de línea con error en Redis para insert_valid_rows
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar CustomsBroker.
|
||||
"""
|
||||
logger.info(f"CB import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"cb_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"cb_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{CB_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"CB import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header)
|
||||
err = _validate_row_customs_broker(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
clave = (row_norm.get("CLAVE") or "").strip()[:5]
|
||||
if not clave:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
existing.type = _str_or_none(row_norm.get("TIPO"), 9)
|
||||
existing.name = _str_or_none(row_norm.get("NOMBRE"), 80)
|
||||
existing.address = _str_or_none(row_norm.get("DIRECCION"), 1500)
|
||||
existing.postal_code = _str_or_none(row_norm.get("CODIGO POSTAL"), 15)
|
||||
existing.city = _str_or_none(row_norm.get("CIUDAD"), 30)
|
||||
existing.state = _str_or_none(row_norm.get("ESTADO"), 30)
|
||||
existing.phone = _str_or_none(row_norm.get("TELEFONO"), 30)
|
||||
existing.fax = _str_or_none(row_norm.get("FAX"), 30)
|
||||
existing.email = _str_or_none(row_norm.get("EMAIL"), 100)
|
||||
existing.country = _str_or_none(row_norm.get("PAIS"), 3)
|
||||
existing.tax_id = _str_or_none(row_norm.get("RFC"), 30)
|
||||
existing.personal_id = _str_or_none(row_norm.get("PERSONAL_ID"), 20)
|
||||
existing.position = _str_or_none(row_norm.get("POSICION"), 30)
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
existing.license = lic[:4] if lic and lic.isdigit() else None
|
||||
existing.company = _str_or_none(row_norm.get("EMPRESA"), 200)
|
||||
existing.contact = _str_or_none(row_norm.get("CONTACTO"), 80)
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
license_val = lic[:4] if lic and lic.isdigit() else None
|
||||
new_broker = CustomsBroker(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
broker_key=clave,
|
||||
type=_str_or_none(row_norm.get("TIPO"), 9),
|
||||
name=_str_or_none(row_norm.get("NOMBRE"), 80),
|
||||
address=_str_or_none(row_norm.get("DIRECCION"), 1500),
|
||||
postal_code=_str_or_none(row_norm.get("CODIGO POSTAL"), 15),
|
||||
city=_str_or_none(row_norm.get("CIUDAD"), 30),
|
||||
state=_str_or_none(row_norm.get("ESTADO"), 30),
|
||||
phone=_str_or_none(row_norm.get("TELEFONO"), 30),
|
||||
fax=_str_or_none(row_norm.get("FAX"), 30),
|
||||
email=_str_or_none(row_norm.get("EMAIL"), 100),
|
||||
country=_str_or_none(row_norm.get("PAIS"), 3),
|
||||
tax_id=_str_or_none(row_norm.get("RFC"), 30),
|
||||
personal_id=_str_or_none(row_norm.get("PERSONAL_ID"), 20),
|
||||
position=_str_or_none(row_norm.get("POSICION"), 30),
|
||||
license=license_val,
|
||||
company=_str_or_none(row_norm.get("EMPRESA"), 200),
|
||||
contact=_str_or_none(row_norm.get("CONTACTO"), 80),
|
||||
)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"CB import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CB import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
# Limpieza
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"CB import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -7,7 +7,7 @@ from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from . import dto, services
|
||||
from .imports.routes import router as imports_router
|
||||
from ..layouts_csv.customs_brokers.routes import router as imports_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@@ -1,465 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
DATE_FORMATS = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"er_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{ER_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _parse_date(val: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse date string; supports YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _validate_row_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Tipo de Cambio. Retorna error dict o None."""
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if _parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
|
||||
local_raw = (row.get("MONEDA_LOCAL") or "").strip().upper()
|
||||
if local_raw and len(local_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_LOCAL", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
foreign_raw = (row.get("MONEDA_EXTRANJERA") or "").strip().upper()
|
||||
if foreign_raw and len(foreign_raw) > 7:
|
||||
return {"line": line_num, "col": "MONEDA_EXTRANJERA", "msg": "Máximo 7 caracteres"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"ER import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, insertar/actualizar ExchangeRate (upsert por fecha).
|
||||
"""
|
||||
logger.info(f"ER import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"er_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"er_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{ER_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"ER import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_exchange_rate(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
parsed_date = _parse_date(row_norm.get("FECHA"))
|
||||
if not parsed_date:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA: no parseable"})
|
||||
continue
|
||||
|
||||
try:
|
||||
v = float((row_norm.get("VALOR") or "").strip().replace(",", "."))
|
||||
value_decimal = Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "VALOR: no numérico"})
|
||||
continue
|
||||
|
||||
local_currency = _str_or_none(row_norm.get("MONEDA_LOCAL"), 7)
|
||||
if local_currency:
|
||||
local_currency = local_currency.upper()
|
||||
foreign_currency = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), 7)
|
||||
if foreign_currency:
|
||||
foreign_currency = foreign_currency.upper()
|
||||
|
||||
key_date = parsed_date.date()
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
|
||||
if existing:
|
||||
existing.value = value_decimal
|
||||
existing.local_currency = local_currency or None
|
||||
existing.foreign_currency = foreign_currency or None
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
date=parsed_date,
|
||||
value=value_decimal,
|
||||
local_currency=local_currency,
|
||||
foreign_currency=foreign_currency,
|
||||
)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"ER import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"ER import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"ER import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -35,7 +35,7 @@ from fastapi import APIRouter
|
||||
custom_router = APIRouter(prefix="/exchange-rate", tags=[])
|
||||
|
||||
# CSV import: add to custom_router BEFORE including it in master, so /exchange-rate/imports/* is registered
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.exchange_rate.routes import router as imports_router
|
||||
custom_router.include_router(imports_router, prefix="/imports", tags=["exchange_rate / csv_import"])
|
||||
|
||||
@custom_router.get("/test-ping")
|
||||
|
||||
@@ -1,474 +0,0 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Fracción Americana (US Tariff Fractions).
|
||||
Flujo en dos fases: scan_file (validación) → insert_valid_rows (commit).
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FA_IMPORT_FILE_PREFIX = "fa_import_file:"
|
||||
FA_IMPORT_META_PREFIX = "fa_import_meta:"
|
||||
FA_IMPORT_ERROR_LINES_PREFIX = "fa_import_error_lines:"
|
||||
FA_IMPORT_REDIS_TTL = 3600 # 1 hour
|
||||
|
||||
TEMPLATE_ID = "us_tariff_fractions"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _worker_upload_dir() -> str:
|
||||
return os.path.join(os.getcwd(), "uploads", "temp")
|
||||
|
||||
|
||||
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_FILE_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode file from Redis: {e}")
|
||||
return None
|
||||
upload_dir = _worker_upload_dir()
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"fa_{job_id}.csv")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(raw)
|
||||
return file_path
|
||||
|
||||
|
||||
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
|
||||
r = _get_redis()
|
||||
data = r.get(f"{FA_IMPORT_META_PREFIX}{job_id}")
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to decode meta from Redis: {e}")
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def _delete_import_from_redis(job_id: str) -> None:
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to delete Redis keys: {e}")
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
|
||||
|
||||
def _normalize_code(raw: Optional[str]) -> str:
|
||||
"""Normalize fraction code: strip and remove dots/dashes, max 16 chars."""
|
||||
if not raw:
|
||||
return ""
|
||||
s = str(raw).strip().replace(".", "").replace("-", "")
|
||||
return s[:16] if len(s) > 16 else s
|
||||
|
||||
|
||||
def _validate_row_us_tariff_fraction(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Valida una fila para Fracción Americana. Retorna error dict o None."""
|
||||
code_raw = (row.get("FRACCION_ARANCELARIA") or "").strip()
|
||||
if not code_raw:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
code_norm = _normalize_code(code_raw)
|
||||
if not code_norm:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Requerido"}
|
||||
if len(code_norm) > 16:
|
||||
return {"line": line_num, "col": "FRACCION_ARANCELARIA", "msg": "Máximo 16 caracteres"}
|
||||
|
||||
prefix_raw = (row.get("PREFIJO") or "").strip()
|
||||
if prefix_raw and len(prefix_raw) > 10:
|
||||
return {"line": line_num, "col": "PREFIJO", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
um_raw = (row.get("UNIDAD_DE_MEDIDA") or "").strip()
|
||||
if um_raw and len(um_raw) > 10:
|
||||
return {"line": line_num, "col": "UNIDAD_DE_MEDIDA", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
tipo_raw = (row.get("TIPO_DE_ADVALOREM") or "").strip()
|
||||
if tipo_raw and len(tipo_raw) > 10:
|
||||
return {"line": line_num, "col": "TIPO_DE_ADVALOREM", "msg": "Máximo 10 caracteres"}
|
||||
|
||||
adv_pct = row.get("ADVALOREM_PCT")
|
||||
if adv_pct is not None and str(adv_pct).strip():
|
||||
try:
|
||||
v = float(str(adv_pct).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_PCT", "msg": "Debe ser un número"}
|
||||
|
||||
adv_dlls = row.get("ADVALOREM_DLLS")
|
||||
if adv_dlls is not None and str(adv_dlls).strip():
|
||||
try:
|
||||
v = float(str(adv_dlls).strip().replace(",", "."))
|
||||
if v < 0:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser >= 0"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "ADVALOREM_DLLS", "msg": "Debe ser un número"}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
"""
|
||||
Fase 1: Leer CSV, validar filas, escribir errores en JSONL.
|
||||
Devuelve waiting_confirmation con total_rows, error_count, valid_rows, errors.
|
||||
"""
|
||||
logger.info(f"FA import: starting scan for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
error_dir = os.path.join(os.path.dirname(file_path).replace("temp", "errors"), "")
|
||||
os.makedirs(error_dir, exist_ok=True)
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
total_rows = 0
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to read meta: {e}")
|
||||
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f_in, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
error_count += 1
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append(
|
||||
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
|
||||
)
|
||||
processed_rows += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import scan failed: {e}")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
error_lines_list = []
|
||||
try:
|
||||
if os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
error_lines_list.append(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
if error_lines_list:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}",
|
||||
json.dumps(error_lines_list).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: failed to store error lines in Redis: {e}")
|
||||
|
||||
return {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": processed_rows,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_float(val: Any) -> Optional[float]:
|
||||
if val is None or str(val).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(str(val).strip().replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
"""
|
||||
Fase 2: Re-leer CSV, omitir filas con error, upsert USTariffFraction por (tenant_id, company_id, code).
|
||||
"""
|
||||
logger.info(f"FA import: starting commit for job {job_id}")
|
||||
|
||||
file_path = _ensure_worker_has_file_from_redis(job_id)
|
||||
if not file_path:
|
||||
alt_path = os.path.join(_worker_upload_dir(), f"fa_{job_id}.csv")
|
||||
if not os.path.exists(alt_path):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
|
||||
}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_worker_has_meta_from_redis(job_id, file_path)
|
||||
|
||||
base_dir = os.path.dirname(file_path)
|
||||
error_dir = base_dir.replace("temp", "errors")
|
||||
error_path = os.path.join(error_dir, f"fa_{job_id}.jsonl")
|
||||
|
||||
error_lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(f"{FA_IMPORT_ERROR_LINES_PREFIX}{job_id}")
|
||||
if raw:
|
||||
error_lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug(f"FA import: could not load error lines from Redis: {e}")
|
||||
if not error_lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
error_lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
tenant_id = None
|
||||
company_id = None
|
||||
meta = {}
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
meta = json.load(f) or {}
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not tenant_id or not company_id:
|
||||
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.DictReader(f, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, normalize_header, TEMPLATE_ID)
|
||||
err = _validate_row_us_tariff_fraction(row_norm, i)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append(
|
||||
{
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
code = _normalize_code(row_norm.get("FRACCION_ARANCELARIA"))
|
||||
if not code:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FRACCION_ARANCELARIA: vacío"})
|
||||
continue
|
||||
|
||||
prefix = _str_or_none(row_norm.get("PREFIJO"), 10)
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIDAD_DE_MEDIDA"), 10)
|
||||
description = _str_or_none(row_norm.get("DESCRIPCION"))
|
||||
type_code = _str_or_none(row_norm.get("TIPO_DE_ADVALOREM"), 10)
|
||||
ad_valorem = _parse_float(row_norm.get("ADVALOREM_PCT"))
|
||||
fixed_cost_raw = _parse_float(row_norm.get("ADVALOREM_DLLS"))
|
||||
fixed_cost = Decimal(str(round(fixed_cost_raw, 8))) if fixed_cost_raw is not None else None
|
||||
|
||||
existing = (
|
||||
session.query(USTariffFraction)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
USTariffFraction.code == code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.prefix = prefix
|
||||
existing.type_code = type_code
|
||||
existing.ad_valorem = ad_valorem
|
||||
existing.fixed_cost = fixed_cost
|
||||
existing.unit_of_measure = unit_of_measure
|
||||
existing.description = description
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_row = USTariffFraction(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
code=code,
|
||||
prefix=prefix,
|
||||
type_code=type_code,
|
||||
ad_valorem=ad_valorem,
|
||||
fixed_cost=fixed_cost,
|
||||
unit_of_measure=unit_of_measure,
|
||||
description=description,
|
||||
)
|
||||
session.add(new_row)
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error(f"FA import DB error: {db_err}")
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
total_skipped = skipped_invalid
|
||||
if inserted_count == 0 and total_skipped > 0:
|
||||
response = {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {total_skipped} rechazados.",
|
||||
}
|
||||
elif inserted_count == 0:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FA import task failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
if file_path and os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
if os.path.exists(error_path):
|
||||
os.remove(error_path)
|
||||
meta_path_clean = file_path.replace(".csv", ".meta.json")
|
||||
if os.path.exists(meta_path_clean):
|
||||
os.remove(meta_path_clean)
|
||||
_delete_import_from_redis(job_id)
|
||||
except Exception as cleanup_err:
|
||||
logger.warning(f"FA import cleanup failed: {cleanup_err}")
|
||||
|
||||
if response is None:
|
||||
response = {
|
||||
"status": "failed",
|
||||
"error": "Error inesperado",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return response
|
||||
@@ -31,7 +31,7 @@ crud_router = TenantCRUDRoutes(
|
||||
)
|
||||
|
||||
# Master router with prefix so all routes live under /us-tariff-fractions
|
||||
from .imports.routes import router as imports_router
|
||||
from api.v1.modules.a76.layouts_csv.us_tariff_fractions.routes import router as imports_router
|
||||
main_router = APIRouter(prefix="/us-tariff-fractions", tags=["a76 / general catalogs / us tariff fractions"])
|
||||
main_router.include_router(imports_router, prefix="/imports", tags=["us_tariff_fractions / csv_import"])
|
||||
main_router.include_router(crud_router.router)
|
||||
|
||||
1
backend/api/v1/modules/a76/layouts_csv/__init__.py
Normal file
1
backend/api/v1/modules/a76/layouts_csv/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# Lógica centralizada de cargas por CSV (routes, tasks, validaciones, template_config por proceso)
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers, fk_loader for boms CSV import
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (BOMs).
|
||||
"""
|
||||
import re
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_decimal_required_min(
|
||||
row: Dict[str, Any], col: str, line_num: int, min_val: Decimal
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
try:
|
||||
v = Decimal(str(val))
|
||||
if v < min_val:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser mayor o igual a cero"}
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
|
||||
|
||||
def _optional_number(val: Any) -> bool:
|
||||
if val is None:
|
||||
return True
|
||||
s = re.sub(r"\s+", "", str(val).strip())
|
||||
if not s:
|
||||
return True
|
||||
try:
|
||||
float(s.replace(",", "."))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def check_optional_number(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
if not _optional_number(row.get(col)):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
allowed: Optional[set],
|
||||
msg: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or allowed is None or len(allowed) == 0:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación de import CSV de BOMs.
|
||||
"""
|
||||
from typing import Set
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_boms_fk_sets(tenant_id: int, company_id: int) -> Set[str]:
|
||||
"""Carga valid_part_numbers (Part.part_number por tenant/company)."""
|
||||
valid_part_numbers: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.parts.models import Part
|
||||
for p in (
|
||||
session.query(Part.part_number)
|
||||
.filter(
|
||||
Part.tenant_id == tenant_id,
|
||||
Part.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
valid_part_numbers.add(p[0])
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("BOMs import: could not load parts: %s", e)
|
||||
return valid_part_numbers
|
||||
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para BOM (para cuando exista tabla BOM).
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def _decimal_or_none(val: Any) -> Optional[Decimal]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val).replace(",", "."))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_bom_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_part_numbers: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para BOM.
|
||||
Para uso futuro cuando exista tabla BOM.
|
||||
"""
|
||||
parent = _str_or_none(row_norm.get("NUMPARTE_PADRE"), 70)
|
||||
component = _str_or_none(row_norm.get("NUMPARTE_COMPONENTE"), 70)
|
||||
quantity = _decimal_or_none(row_norm.get("CANTIDAD"))
|
||||
uom = _str_or_none(row_norm.get("UNIMED"), 10)
|
||||
version_bom = _decimal_or_none(row_norm.get("VERSION_BOM"))
|
||||
version_bill = _decimal_or_none(row_norm.get("VERSION_BILL"))
|
||||
return {
|
||||
"parent_part_number": parent,
|
||||
"component_part_number": component,
|
||||
"quantity": quantity,
|
||||
"uom": uom,
|
||||
"version_bom": version_bom,
|
||||
"version_bill": version_bill,
|
||||
}
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -78,7 +79,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
168
backend/api/v1/modules/a76/layouts_csv/boms/tasks.py
Normal file
168
backend/api/v1/modules/a76/layouts_csv/boms/tasks.py
Normal file
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de BOMs.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y cuenta filas válidas.
|
||||
Usa layouts_csv.common y common.fk_loader, validators.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_bom
|
||||
from .common.fk_loader import load_boms_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "bom"
|
||||
|
||||
# Para routes.py
|
||||
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
||||
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
||||
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
||||
BOM_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("BOMs import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("BOMs import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("BOMs import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
valid_count = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
valid_count += 1
|
||||
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
||||
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, 0, 0, skipped_details,
|
||||
)
|
||||
if valid_count > 0 and inserted_count == 0:
|
||||
response["message"] = (
|
||||
f"WIP: {valid_count} filas válidas. "
|
||||
"La tabla BOM aún no existe en el sistema; no se insertó nada."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("BOMs import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_bom
|
||||
|
||||
__all__ = ["validate_row_bom"]
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de BOMs.
|
||||
"""
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_max_length,
|
||||
check_decimal_required_min,
|
||||
check_optional_number,
|
||||
check_in_set,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_required_parent(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "NUMPARTE_PADRE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_required_component(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "NUMPARTE_COMPONENTE", 70, line_num, required=True)
|
||||
|
||||
|
||||
def validate_row_quantity(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_decimal_required_min(row, "CANTIDAD", line_num, Decimal("0"))
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_max_length(row, "UNIMED", 10, line_num)
|
||||
|
||||
|
||||
def validate_row_optional_numbers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
err = check_optional_number(row, "VERSION_BOM", line_num)
|
||||
if err:
|
||||
return err
|
||||
return check_optional_number(row, "VERSION_BILL", line_num)
|
||||
|
||||
|
||||
def validate_row_fks(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not valid_part_numbers or len(valid_part_numbers) == 0:
|
||||
return None
|
||||
err = check_in_set(
|
||||
row, "NUMPARTE_PADRE", line_num,
|
||||
valid_part_numbers, "Parte padre no existe en catálogo",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return check_in_set(
|
||||
row, "NUMPARTE_COMPONENTE", line_num,
|
||||
valid_part_numbers, "Parte componente no existe en catálogo",
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila BOM.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required_parent,
|
||||
validate_row_required_component,
|
||||
validate_row_quantity,
|
||||
validate_row_lengths,
|
||||
validate_row_optional_numbers,
|
||||
validate_row_fks,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_bom(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_part_numbers: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de BOMs.
|
||||
Encadena: requeridos (padre, componente, cantidad) → longitudes → opcionales numéricos → FKs.
|
||||
"""
|
||||
err = validate_row_required_parent(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_required_component(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_quantity(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_optional_numbers(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fks(row, line_num, valid_part_numbers)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
# layouts_csv.cambio_regimen_regularizacion — carga CSV Cambio de régimen y Regularización (encabezado y partidas)
|
||||
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
Rutas de importación CSV para Cambio de régimen y Regularización (encabezado y partidas).
|
||||
Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, UploadFile, Depends, Form, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Literal, Optional, Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse, CommitRequest
|
||||
from .tasks import scan_file, insert_valid_rows, JOB_TYPE, CRREG_IMPORT_REDIS_TTL
|
||||
from ..common import storage as common_storage
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _default_template_id(model_target: str, document_type: Optional[str]) -> str:
|
||||
if document_type == "regulariz":
|
||||
return "regulariz_header" if model_target == "invoice_header" else "regulariz_details"
|
||||
return "cam_reg_header" if model_target == "invoice_header" else "cam_reg_details"
|
||||
|
||||
|
||||
@router.post("/upload/{model_target}", response_model=ImportJobResponse)
|
||||
async def upload_import_file(
|
||||
model_target: Literal["invoice_header", "invoice_details"],
|
||||
file: UploadFile = File(...),
|
||||
footer_config: Optional[str] = Form(None),
|
||||
template_id: Optional[str] = Form(None),
|
||||
document_type: Optional[str] = Query(None, description="cam_reg | regulariz"),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""Subir CSV, guardar en Redis, encolar scan. template_id/document_type distinguen Cambio de régimen vs Regularización."""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
except Exception as e:
|
||||
logger.error("Cambio régimen/Regularización import: access validation failed: %s", e)
|
||||
raise HTTPException(status_code=403, detail="Invalid company access")
|
||||
|
||||
if not file.filename or not file.filename.lower().endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="Solo se permiten archivos .csv")
|
||||
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id)
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"footer_config": footer_config,
|
||||
"document_type": document_type or "cam_reg",
|
||||
"template_id": template_id or _default_template_id(model_target, document_type),
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(file_key, base64.b64encode(contents), ex=CRREG_IMPORT_REDIS_TTL)
|
||||
r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=CRREG_IMPORT_REDIS_TTL)
|
||||
except Exception as e:
|
||||
logger.error("Cambio régimen/Regularización import: Redis store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
with open(csv_path, "wb") as f:
|
||||
f.write(contents)
|
||||
meta_path = csv_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning("Cambio régimen/Regularización import: local file save failed: %s", e)
|
||||
|
||||
scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id)
|
||||
|
||||
return ImportJobResponse(
|
||||
job_id=job_id,
|
||||
status="queued",
|
||||
message="Archivo subido. Escaneo iniciado.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status")
|
||||
async def get_import_status(job_id: str):
|
||||
"""Polling: estado del escaneo o del commit."""
|
||||
task_result = celery_app.AsyncResult(job_id)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return {"status": "processing", "progress": 0}
|
||||
if task_result.state == "PROGRESS":
|
||||
info = task_result.info or {}
|
||||
return {
|
||||
"status": "processing",
|
||||
"progress": info.get("current", 0),
|
||||
"total": info.get("total", 0),
|
||||
}
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"):
|
||||
return task_result.result
|
||||
|
||||
logger.warning("Cambio régimen/Regularización import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
if tb and isinstance(tb, str):
|
||||
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
|
||||
if lines:
|
||||
err_msg = lines[-1]
|
||||
if not err_msg:
|
||||
try:
|
||||
exc = task_result.get(propagate=False)
|
||||
if exc is not None:
|
||||
err_msg = str(exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not err_msg:
|
||||
result = getattr(task_result, "result", None)
|
||||
if result is not None and not isinstance(result, dict):
|
||||
err_msg = str(result)
|
||||
elif isinstance(result, dict) and (result.get("error") or result.get("message")):
|
||||
err_msg = result.get("error") or result.get("message")
|
||||
return {"status": "failed", "error": err_msg or "Task failed"}
|
||||
|
||||
|
||||
@router.post("/{job_id}/commit")
|
||||
async def commit_import_job(job_id: str, body: CommitRequest):
|
||||
"""Usuario confirma; se encola la tarea de commit (por ahora sin inserción real)."""
|
||||
task = insert_valid_rows.delay(job_id, body.model_target)
|
||||
return {
|
||||
"status": "committing",
|
||||
"message": "Proceso de commit iniciado.",
|
||||
"commit_job_id": task.id,
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, Literal
|
||||
|
||||
|
||||
class ImportJobResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
|
||||
class CommitRequest(BaseModel):
|
||||
model_target: Literal["invoice_header", "invoice_details"]
|
||||
|
||||
|
||||
class ImportJobStatus(BaseModel):
|
||||
status: str
|
||||
job_id: str
|
||||
total_rows: Optional[int] = 0
|
||||
error_count: Optional[int] = 0
|
||||
valid_rows: Optional[int] = 0
|
||||
error: Optional[str] = None
|
||||
inserted: Optional[int] = 0
|
||||
error_file: Optional[str] = None
|
||||
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Cambio de régimen y Regularización (encabezado y partidas).
|
||||
Flujo: scan_file (sin validaciones) → insert_valid_rows (sin inserción en BD).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
Validaciones independientes por document_type (cam_reg vs regulariz) se añadirán después.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from core.celery_app import celery_app
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "crreg"
|
||||
CRREG_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _ensure_file(job_id: str) -> Optional[str]:
|
||||
return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Cambio régimen/Regularización import")
|
||||
|
||||
|
||||
def _ensure_meta(job_id: str, file_path: str) -> bool:
|
||||
return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Cambio régimen/Regularización import")
|
||||
|
||||
|
||||
def _norm_row(row: Dict[str, Any], template_id: str) -> Dict[str, Any]:
|
||||
return row_from_template(row, template_id, common_normalize.normalize_header)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="api.v1.modules.a76.layouts_csv.cambio_regimen_regularizacion.tasks.scan_file")
|
||||
def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
"""
|
||||
Scan CSV sin validaciones: leer, normalizar con plantilla, devolver total_rows y 0 errores.
|
||||
"""
|
||||
logger.info("Cambio régimen/Regularización import: starting scan for job %s target %s", job_id, model_target)
|
||||
|
||||
file_path = _ensure_file(job_id)
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
if os.path.getsize(file_path) == 0:
|
||||
return {"status": "failed", "error": "El archivo está vacío."}
|
||||
_ensure_meta(job_id, file_path)
|
||||
|
||||
try:
|
||||
common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
document_type = meta.get("document_type") or "cam_reg"
|
||||
template_id = meta.get("template_id") or (
|
||||
"cam_reg_header" if model_target == "invoice_header" else "cam_reg_details"
|
||||
)
|
||||
if document_type == "regulariz" and not meta.get("template_id"):
|
||||
template_id = "regulariz_header" if model_target == "invoice_header" else "regulariz_details"
|
||||
|
||||
total_rows = 0
|
||||
processed_rows = 0
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path, has_header=True)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
def on_progress(current: int, total: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total})
|
||||
|
||||
try:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None):
|
||||
if i % 500 == 0:
|
||||
on_progress(i, total_rows)
|
||||
_norm_row(row, template_id)
|
||||
processed_rows += 1
|
||||
except Exception as e:
|
||||
logger.error("Cambio régimen/Regularización import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, 0, [])
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="api.v1.modules.a76.layouts_csv.cambio_regimen_regularizacion.tasks.insert_valid_rows")
|
||||
def insert_valid_rows(self, job_id: str, model_target: str):
|
||||
"""
|
||||
Commit sin inserción en BD: leer CSV, omitir líneas de error (vacío por ahora), cleanup, devolver finished con inserted=0.
|
||||
"""
|
||||
logger.info("Cambio régimen/Regularización import: starting commit for job %s target %s", job_id, model_target)
|
||||
|
||||
file_path = _ensure_file(job_id)
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
_ensure_meta(job_id, file_path)
|
||||
|
||||
try:
|
||||
common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
document_type = meta.get("document_type") or "cam_reg"
|
||||
template_id = meta.get("template_id") or (
|
||||
"cam_reg_header" if model_target == "invoice_header" else "cam_reg_details"
|
||||
)
|
||||
if document_type == "regulariz" and not meta.get("template_id"):
|
||||
template_id = "regulariz_header" if model_target == "invoice_header" else "regulariz_details"
|
||||
|
||||
try:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None):
|
||||
if i in error_lines:
|
||||
continue
|
||||
_norm_row(row, template_id)
|
||||
except Exception as e:
|
||||
logger.error("Cambio régimen/Regularización import commit read failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": 0,
|
||||
"skipped_invalid": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_details": [],
|
||||
"message": "Proceso base listo; validaciones e inserción pendientes.",
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
Plantillas CSV para Cambio de régimen y Regularización (encabezado y partidas).
|
||||
Por ahora misma estructura que encabezado/partidas de exportación; luego se ajustan columnas si difieren.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
# Cambio de régimen: cam_reg_header, cam_reg_details
|
||||
# Regularización: regulariz_header, regulariz_details
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"cam_reg_header": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
|
||||
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
|
||||
{"canonical": "FECHA EMISION"},
|
||||
{"canonical": "CLAVE PROVEEDOR"},
|
||||
{"canonical": "CLAVE VENDIDO A"},
|
||||
{"canonical": "CLAVE ENVIADO A"},
|
||||
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
|
||||
{"canonical": "ADUANA DE CRUCE"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
{"canonical": "CLAVE INCOTERM"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "TIPO DE CAMBIO"},
|
||||
{"canonical": "TIPO PESO"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "REMESA"},
|
||||
{"canonical": "AGENTE ADUANAL"},
|
||||
{"canonical": "FLETES"},
|
||||
{"canonical": "VALOR SEGUROS"},
|
||||
{"canonical": "SEGUROS"},
|
||||
{"canonical": "EMBALAJES"},
|
||||
{"canonical": "OTROS INCREMENTABLES"},
|
||||
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
|
||||
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
{"canonical": "FACTURA ALTERNA"},
|
||||
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
|
||||
{"canonical": "OBSERVACIONES E"},
|
||||
{"canonical": "OBSERVACIONES I"},
|
||||
{"canonical": "E DOCUMENT"},
|
||||
{"canonical": "NUM OPERACION"},
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "PRECINTO"},
|
||||
],
|
||||
"cam_reg_details": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
|
||||
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
|
||||
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
|
||||
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
|
||||
{"canonical": "CANTIDAD"},
|
||||
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
|
||||
{"canonical": "FRACCION"},
|
||||
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
],
|
||||
"regulariz_header": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
|
||||
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
|
||||
{"canonical": "FECHA EMISION"},
|
||||
{"canonical": "CLAVE PROVEEDOR"},
|
||||
{"canonical": "CLAVE VENDIDO A"},
|
||||
{"canonical": "CLAVE ENVIADO A"},
|
||||
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
|
||||
{"canonical": "ADUANA DE CRUCE"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
{"canonical": "CLAVE INCOTERM"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "TIPO DE CAMBIO"},
|
||||
{"canonical": "TIPO PESO"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "REMESA"},
|
||||
{"canonical": "AGENTE ADUANAL"},
|
||||
{"canonical": "FLETES"},
|
||||
{"canonical": "VALOR SEGUROS"},
|
||||
{"canonical": "SEGUROS"},
|
||||
{"canonical": "EMBALAJES"},
|
||||
{"canonical": "OTROS INCREMENTABLES"},
|
||||
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
|
||||
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
{"canonical": "FACTURA ALTERNA"},
|
||||
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
|
||||
{"canonical": "OBSERVACIONES E"},
|
||||
{"canonical": "OBSERVACIONES I"},
|
||||
{"canonical": "E DOCUMENT"},
|
||||
{"canonical": "NUM OPERACION"},
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "PRECINTO"},
|
||||
],
|
||||
"regulariz_details": [
|
||||
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
|
||||
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
|
||||
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
|
||||
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
|
||||
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
|
||||
{"canonical": "CANTIDAD"},
|
||||
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
|
||||
{"canonical": "DESCRIPCION"},
|
||||
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
|
||||
{"canonical": "FRACCION"},
|
||||
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]:
|
||||
return TEMPLATE_COLUMNS.get(template_id)
|
||||
|
||||
|
||||
def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name."""
|
||||
cols = _resolve_template_columns(template_id)
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV -> dict con nombres canónicos de la plantilla."""
|
||||
lookup = build_normalized_lookup(template_id, normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -0,0 +1 @@
|
||||
# common validators, mappers, fk_loader for classes CSV import
|
||||
@@ -0,0 +1,108 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (clases de materiales).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_min_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
min_len: int,
|
||||
line_num: int,
|
||||
msg: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; error si longitud menor que min_len."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) < min_len:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": msg or f"Mínimo {min_len} caracteres",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_int_range(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
min_val: int,
|
||||
max_val: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; devuelve error si no es int o está fuera de rango."""
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
v = int(val)
|
||||
if v < min_val or v > max_val:
|
||||
return {"line": line_num, "col": col, "msg": "Valor fuera de rango"}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser número entero"}
|
||||
return None
|
||||
|
||||
|
||||
def check_in_set(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
allowed: Optional[set],
|
||||
msg: str = "No existe en el catálogo",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor y allowed no es None."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or allowed is None:
|
||||
return None
|
||||
if val not in allowed:
|
||||
return {"line": line_num, "col": col, "msg": msg}
|
||||
return None
|
||||
|
||||
|
||||
def check_decimal_max(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
max_val: float,
|
||||
msg: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Solo valida si hay valor; error si no es numérico o si es mayor que max_val (Clarion Col H)."""
|
||||
val = row.get(col)
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
v = float(val)
|
||||
if v > max_val:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": msg or f"El valor no puede ser mayor a {max_val}.",
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser un número."}
|
||||
return None
|
||||
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación/mapeo de import CSV de clases de materiales.
|
||||
Clarion: Tipo Activo Fijo, U.M., Fracción Mex (GFracGenSifra + histórico), Fracción Ame (GFracAme), Código Producto CP (si existe).
|
||||
"""
|
||||
from typing import Set, Tuple
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
|
||||
def load_classes_fk_sets(
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Tuple[Set[str], Set[str], Set[str], Set[str], Set[str]]:
|
||||
"""
|
||||
Carga todos los conjuntos necesarios para validación CSV de clases (paridad Clarion).
|
||||
Devuelve (valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp).
|
||||
- valid_fraction_mex_8: códigos de 8 caracteres válidos (TariffFraction + HistoricalTariffFraction).
|
||||
- valid_fraction_ame: códigos de fracción americana (USTariffFraction por tenant/company).
|
||||
- valid_product_codes_cp: códigos de producto/servicio CP (vacío si no existe catálogo).
|
||||
"""
|
||||
valid_material_keys: Set[str] = set()
|
||||
valid_uom_codes: Set[str] = set()
|
||||
valid_fraction_mex_8: Set[str] = set()
|
||||
valid_fraction_ame: Set[str] = set()
|
||||
valid_product_codes_cp: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.material_types.models import MaterialType
|
||||
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
|
||||
from api.v1.modules.a76.general_catalogs.fractions.tariff_fractions.models import TariffFraction
|
||||
from api.v1.modules.a76.general_catalogs.fractions.historical_tariff_fractions.models import HistoricalTariffFraction
|
||||
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
|
||||
|
||||
for m in session.query(MaterialType.key).all():
|
||||
if m[0]:
|
||||
valid_material_keys.add(m[0])
|
||||
for u in (
|
||||
session.query(UnitOfMeasure.code)
|
||||
.filter(
|
||||
UnitOfMeasure.tenant_id == tenant_id,
|
||||
UnitOfMeasure.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if u[0]:
|
||||
valid_uom_codes.add(u[0])
|
||||
|
||||
for row in session.query(TariffFraction.code).all():
|
||||
if row[0]:
|
||||
code = row[0].strip()
|
||||
valid_fraction_mex_8.add(code[:8])
|
||||
|
||||
for row in (
|
||||
session.query(HistoricalTariffFraction.historical_fraction)
|
||||
.filter(
|
||||
HistoricalTariffFraction.tenant_id == tenant_id,
|
||||
HistoricalTariffFraction.company_id == company_id,
|
||||
HistoricalTariffFraction.historical_fraction.isnot(None),
|
||||
)
|
||||
.distinct()
|
||||
.all()
|
||||
):
|
||||
if row[0] and row[0].strip():
|
||||
valid_fraction_mex_8.add(row[0].strip()[:8])
|
||||
|
||||
for row in (
|
||||
session.query(USTariffFraction.code)
|
||||
.filter(
|
||||
USTariffFraction.tenant_id == tenant_id,
|
||||
USTariffFraction.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if row[0]:
|
||||
valid_fraction_ame.add(row[0].strip())
|
||||
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("Classes import: could not load FK sets: %s", e)
|
||||
return (
|
||||
valid_material_keys,
|
||||
valid_uom_codes,
|
||||
valid_fraction_mex_8,
|
||||
valid_fraction_ame,
|
||||
valid_product_codes_cp,
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Class (clases de materiales).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _int_or_none(val: Any) -> Optional[int]:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def row_to_class_data(
|
||||
row_norm: Dict[str, Any],
|
||||
valid_material_keys: Set[str],
|
||||
valid_uom_codes: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Mapea una fila normalizada del CSV a un diccionario de datos para Class.
|
||||
class_code se normaliza a mayúsculas (Clip(Upper) Clarion).
|
||||
"""
|
||||
raw_clase = _str_or_none(row_norm.get("CLASE"), 8)
|
||||
class_code = raw_clase.upper() if raw_clase else None
|
||||
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
|
||||
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
|
||||
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
|
||||
if material_key and material_key not in valid_material_keys:
|
||||
material_key = None
|
||||
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
|
||||
if unit_of_measure and unit_of_measure not in valid_uom_codes:
|
||||
unit_of_measure = None
|
||||
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
|
||||
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
|
||||
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
|
||||
physical_review = _int_or_none(row_norm.get("REVFISICA"))
|
||||
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
|
||||
|
||||
return {
|
||||
"class_code": class_code,
|
||||
"description_es": desc_es,
|
||||
"description_en": desc_en,
|
||||
"material_key": material_key,
|
||||
"unit_of_measure": unit_of_measure,
|
||||
"fraction": fraction,
|
||||
"us_fraction": us_fraction,
|
||||
"sub_key": sub_key,
|
||||
"physical_review": physical_review,
|
||||
"iva_exempt_fraction": iva_exempt_fraction,
|
||||
}
|
||||
|
||||
|
||||
def row_to_class_data_merge_existing(
|
||||
row_norm: Dict[str, Any],
|
||||
existing_data: Dict[str, Any],
|
||||
valid_material_keys: Set[str],
|
||||
valid_uom_codes: Set[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Para modo actualizar (parcial): valores del CSV si no vacíos, sino los de la clase existente (Clarion VALIDA_PARCIAL_CLASE).
|
||||
"""
|
||||
data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes)
|
||||
if not data["class_code"]:
|
||||
return data
|
||||
for key in ("description_es", "description_en", "material_key", "unit_of_measure",
|
||||
"fraction", "us_fraction", "sub_key", "physical_review", "iva_exempt_fraction"):
|
||||
if data.get(key) is None or (isinstance(data[key], str) and not data[key].strip()):
|
||||
data[key] = existing_data.get(key)
|
||||
return data
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -39,6 +40,8 @@ def _get_redis():
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
actualizar: bool = Query(False, description="Modo actualizar (ACT): validación parcial si la clase existe"),
|
||||
siempre_toda: bool = Query(False, description="Forzar siempre validación completa"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
@@ -59,6 +62,8 @@ async def upload_import_file(
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "material_classes",
|
||||
"actualizar": actualizar,
|
||||
"siempre_toda": siempre_toda,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -78,7 +83,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
@@ -119,6 +124,27 @@ async def get_import_status(job_id: str):
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
|
||||
# Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica
|
||||
if isinstance(result, str):
|
||||
try:
|
||||
parsed = json.loads(result)
|
||||
if isinstance(parsed, dict) and (
|
||||
parsed.get("status") == "waiting_confirmation"
|
||||
or (parsed.get("job_id") and "total_rows" in parsed)
|
||||
):
|
||||
return parsed
|
||||
if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"):
|
||||
return parsed
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
# Si el resultado tiene forma de escaneo (waiting_confirmation), devolverlo para que el front muestre el modal
|
||||
if isinstance(result, dict) and (
|
||||
result.get("status") == "waiting_confirmation"
|
||||
or (result.get("job_id") and "total_rows" in result)
|
||||
):
|
||||
return result
|
||||
|
||||
logger.warning("Classes import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
tb = getattr(task_result, "traceback", None)
|
||||
309
backend/api/v1/modules/a76/layouts_csv/classes/tasks.py
Normal file
309
backend/api/v1/modules/a76/layouts_csv/classes/tasks.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clases de Materiales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import csv_reader as common_csv
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template, detect_headers_or_data
|
||||
from .validators import validate_row_class, validate_row_class_partial
|
||||
from .common.mappers import row_to_class_data, row_to_class_data_merge_existing
|
||||
from .common.fk_loader import load_classes_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cls"
|
||||
|
||||
# Para routes.py
|
||||
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
|
||||
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
|
||||
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
|
||||
CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("Classes import: starting scan for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header)
|
||||
try:
|
||||
total_rows = common_csv.count_csv_rows(file_path, has_header=has_header)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
actualizar = meta.get("actualizar", False)
|
||||
siempre_toda = meta.get("siempre_toda", False)
|
||||
|
||||
valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets(
|
||||
tenant_id, company_id
|
||||
)
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
existing_class_codes = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
for c in session.query(Class.class_code).filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
).all():
|
||||
if c[0]:
|
||||
existing_class_codes.add(c[0].strip().upper())
|
||||
except Exception as e:
|
||||
logger.warning("Classes import: could not load existing class codes: %s", e)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_class(
|
||||
row_norm,
|
||||
i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
actualizar=actualizar,
|
||||
siempre_toda=siempre_toda,
|
||||
existing_class_codes=existing_class_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("Classes import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("Classes import: starting commit for job %s", job_id)
|
||||
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
actualizar = meta.get("actualizar", False)
|
||||
siempre_toda = meta.get("siempre_toda", False)
|
||||
|
||||
from api.v1.modules.a76.classes.models import Class
|
||||
|
||||
valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets(
|
||||
tenant_id, company_id
|
||||
)
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
response = None
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_code = {}
|
||||
for c in session.query(Class).filter(
|
||||
Class.tenant_id == tenant_id,
|
||||
Class.company_id == company_id,
|
||||
).all():
|
||||
key = (c.class_code or "").strip().upper()
|
||||
if key:
|
||||
existing_by_code[key] = c
|
||||
|
||||
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
class_code_raw = (row_norm.get("CLASE") or "").strip().upper()[:8]
|
||||
use_partial = actualizar and class_code_raw and class_code_raw in existing_by_code and not siempre_toda
|
||||
|
||||
if use_partial:
|
||||
err = validate_row_class_partial(
|
||||
row_norm, i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
else:
|
||||
err = validate_row_class(
|
||||
row_norm,
|
||||
i,
|
||||
valid_material_keys=valid_material_keys,
|
||||
valid_uom_codes=valid_uom_codes,
|
||||
actualizar=actualizar,
|
||||
siempre_toda=siempre_toda,
|
||||
existing_class_codes=set(existing_by_code.keys()),
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
if use_partial:
|
||||
existing = existing_by_code.get(class_code_raw)
|
||||
existing_data = {
|
||||
"description_es": existing.description_es,
|
||||
"description_en": existing.description_en,
|
||||
"material_key": existing.material_key,
|
||||
"unit_of_measure": existing.unit_of_measure,
|
||||
"fraction": existing.fraction,
|
||||
"us_fraction": existing.us_fraction,
|
||||
"sub_key": existing.sub_key,
|
||||
"physical_review": existing.physical_review,
|
||||
"iva_exempt_fraction": existing.iva_exempt_fraction,
|
||||
}
|
||||
data = row_to_class_data_merge_existing(
|
||||
row_norm, existing_data, valid_material_keys, valid_uom_codes,
|
||||
)
|
||||
else:
|
||||
data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes)
|
||||
|
||||
class_code = data.get("class_code")
|
||||
if not class_code:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
existing = existing_by_code.get(class_code)
|
||||
if existing:
|
||||
existing.description_es = data["description_es"]
|
||||
existing.description_en = data["description_en"]
|
||||
existing.material_key = data["material_key"]
|
||||
existing.unit_of_measure = data["unit_of_measure"]
|
||||
existing.fraction = data["fraction"]
|
||||
existing.us_fraction = data["us_fraction"]
|
||||
existing.sub_key = data["sub_key"]
|
||||
existing.physical_review = data["physical_review"]
|
||||
existing.iva_exempt_fraction = data["iva_exempt_fraction"]
|
||||
session.add(existing)
|
||||
inserted_count += 1
|
||||
else:
|
||||
new_class = Class(
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
**data,
|
||||
)
|
||||
session.add(new_class)
|
||||
existing_by_code[class_code] = new_class
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("Classes import DB error: %s", db_err)
|
||||
return common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error=str(db_err),
|
||||
)
|
||||
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
response = common_responses.commit_result(
|
||||
"warning", 0, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
message=f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
)
|
||||
elif inserted_count == 0:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
error="No hay registros válidos en el archivo CSV",
|
||||
)
|
||||
else:
|
||||
response = common_responses.commit_result(
|
||||
"finished", inserted_count, skipped_invalid, 0, 0,
|
||||
skipped_details,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Classes import task failed")
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error=str(e),
|
||||
)
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
response = common_responses.commit_result(
|
||||
"failed", 0, skipped_invalid, 0, 0,
|
||||
skipped_details, error="Error inesperado",
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clases de Materiales (EstructuraCatClasesAF.xls).
|
||||
Cabeceras de descarga = Clarion: CLAVE CLASE, DESCRIPCION ESPAÑOL, DESCRIPCION INGLES, etc.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
|
||||
|
||||
# Valores que indican que la primera fila es cabecera (primera columna normalizada)
|
||||
FIRST_COLUMN_HEADER_VALUES = ("CLAVE CLASE", "CLASE")
|
||||
|
||||
|
||||
def detect_headers_or_data(
|
||||
file_path: str,
|
||||
normalize_header_fn,
|
||||
encoding: str = "utf-8-sig",
|
||||
) -> Tuple[Optional[List[str]], bool]:
|
||||
"""
|
||||
Lee la primera línea del CSV y decide si es cabecera o dato.
|
||||
Devuelve (fieldnames, has_header).
|
||||
- Si la primera celda normalizada está en FIRST_COLUMN_HEADER_VALUES -> has_header=True, fieldnames=None
|
||||
(la primera fila es cabecera; iter_csv_rows sin fieldnames).
|
||||
- Si no -> has_header=False, fieldnames=TEMPLATE_DOWNLOAD_HEADERS (la primera fila es dato).
|
||||
"""
|
||||
try:
|
||||
with open(file_path, "r", encoding=encoding) as f:
|
||||
sample = f.read(2048)
|
||||
except Exception:
|
||||
return None, True
|
||||
lines = sample.splitlines()
|
||||
if not lines:
|
||||
return None, True
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = csv.excel
|
||||
reader = csv.reader(io.StringIO(lines[0]), dialect=dialect)
|
||||
first_row = next(reader, None)
|
||||
if not first_row:
|
||||
return None, True
|
||||
first_cell = (first_row[0] or "").strip()
|
||||
first_cell_norm = normalize_header_fn(first_cell)
|
||||
if first_cell_norm in FIRST_COLUMN_HEADER_VALUES:
|
||||
return None, True
|
||||
return list(TEMPLATE_DOWNLOAD_HEADERS), False
|
||||
|
||||
# Cabeceras que se escriben al descargar la plantilla CSV (igual que Clarion EstructuraCatClasesAF)
|
||||
TEMPLATE_DOWNLOAD_HEADERS: List[str] = [
|
||||
"CLAVE CLASE",
|
||||
"DESCRIPCION ESPAÑOL",
|
||||
"DESCRIPCION INGLES",
|
||||
"TIPO DE MATERIAL",
|
||||
"U.M. COMERCIAL",
|
||||
"FRACCION ARANCELARIA",
|
||||
"FRACCION AMERICANA",
|
||||
"TASA DE DEPRECIACION",
|
||||
"REVISION FISICA (1/0)",
|
||||
"CODIGO DE PRODUCTO/SERVICIO CP",
|
||||
]
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"material_classes": [
|
||||
{"canonical": "CLASE", "aliases": ["CLAVE CLASE", "CLASS", "CODIGO", "CLASE CODIGO"]},
|
||||
{"canonical": "DESCRIPCIONE", "aliases": ["DESCRIPCION ESPAÑOL", "DESCRIPCION", "DESCRIPCION ES", "DESCRIPCION ESPAÑOL"]},
|
||||
{"canonical": "DESCRIPCIONI", "aliases": ["DESCRIPCION INGLES", "DESCRIPCION EN", "DESCRIPTION", "DESCRIPCION INGLES"]},
|
||||
{"canonical": "CLAVEMAT", "aliases": ["TIPO DE MATERIAL", "MATERIAL", "TIPOMAT", "CLAVE MATERIAL"]},
|
||||
{"canonical": "UNIMED", "aliases": ["U.M. COMERCIAL", "UNIDAD MEDIDA", "UNIT", "UOM", "TIPO DE MU.M.", "U.M. COMERCIAL"]},
|
||||
{"canonical": "FRACCION", "aliases": ["FRACCION ARANCELARIA", "FRACCION MEX", "COM FRACCION"]},
|
||||
{"canonical": "FRACCIONAME", "aliases": ["FRACCION AMERICANA", "FRACCION USA", "US FRACTION", "FRACCION"]},
|
||||
{"canonical": "TASADEPRECIA", "aliases": ["TASA DE DEPRECIACION", "TASA DEPRECIACIÓN", "TASA DEPRECIACION"]},
|
||||
{"canonical": "CLAVESUB", "aliases": ["SUB KEY", "CLAVE SUB"]},
|
||||
{"canonical": "REVFISICA", "aliases": ["REVISION FISICA (1/0)", "REV FISICA", "PHYSICAL REVIEW", "TASA DE REVISION", "REVISION FISICA"]},
|
||||
{"canonical": "FRACCIONEXENTAIVA", "aliases": ["CODIGO DE PRODUCTO/SERVICIO CP", "EXENTA IVA", "FRACCION EXENTA IVA", "CODIGO PRODUCTO SERVICIO CP"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
cols = TEMPLATE_COLUMNS.get("material_classes")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
elif key_norm.startswith("CLAVE CLASE"):
|
||||
# CSV leído con delimitador incorrecto: primera columna es "CLAVE CLASE,..." -> usar primer valor como CLASE
|
||||
if "CLASE" not in out and value:
|
||||
first_val = (value.split(",")[0] if "," in str(value) else value).strip()
|
||||
if first_val:
|
||||
out["CLASE"] = first_val
|
||||
return out
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_class, validate_row_class_partial
|
||||
|
||||
__all__ = ["validate_row_class", "validate_row_class_partial"]
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de clases de materiales.
|
||||
Paridad con Clarion: VALIDACIONES_CLASE (Col A len, Col D/E/F/G en catálogo, Col F len≥8, Col H ≤100, Col J en catálogo CP).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_max_length,
|
||||
check_int_range,
|
||||
check_decimal_max,
|
||||
)
|
||||
|
||||
MSG_CLASE_VACIO = "Error: (Col. A) La columna de Clase esta vacio y no se pueden hacer las validaciones."
|
||||
MSG_CLASE_VACIO_SOLUCION = "Capturar en la Columna A una Clase nueva o una ya existente a la cual desee actualizar campos"
|
||||
|
||||
|
||||
def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""CLASE obligatorio; mensaje Clarion si está vacío. Acepta clave 'CLASE' o 'CLAVE CLASE' (por si el CSV no se normalizó)."""
|
||||
val = (row.get("CLASE") or row.get("CLAVE CLASE") or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": "CLASE", "msg": f"{MSG_CLASE_VACIO} {MSG_CLASE_VACIO_SOLUCION}"}
|
||||
if len(val) > 8:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLASE",
|
||||
"msg": f"Error: (Col. A) La Clase: {val} supera la longitud de caracteres. Capturar en la columna A una Clase de 8 caracteres como máximo.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_required_full(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Obligatorios en validación completa: B, D, E, F (Clarion VALIDA_TODA_CLASE)."""
|
||||
cols_labels = [
|
||||
("DESCRIPCIONE", "(Col.B) Descripción Español"),
|
||||
("CLAVEMAT", "(Col.D) Tipo de Material"),
|
||||
("UNIMED", "(Col.E) U.M. Comercial"),
|
||||
("FRACCION", "(Col.F) Fraccion Arancelaria Mex."),
|
||||
]
|
||||
missing = [(col, label) for col, label in cols_labels if not (row.get(col) or "").strip()]
|
||||
if not missing:
|
||||
return None
|
||||
campos = ", ".join(label for _, label in missing)
|
||||
first_col = missing[0][0]
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": first_col,
|
||||
"msg": f"Existen campos vacios que son obligatorios, es la {campos}. Revisar la línea del archivo y capturar los campos con la información correcta.",
|
||||
}
|
||||
|
||||
|
||||
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("DESCRIPCIONE", 500),
|
||||
("DESCRIPCIONI", 500),
|
||||
("CLAVEMAT", 10),
|
||||
("UNIMED", 5),
|
||||
("FRACCION", 20),
|
||||
("FRACCIONAME", 16),
|
||||
("CLAVESUB", 5),
|
||||
("FRACCIONEXENTAIVA", 4),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_fraction_min(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""FRACCION (Col F): si no vacío, mínimo 8 caracteres (Clarion VALIDACIONES_CLASE)."""
|
||||
val = (row.get("FRACCION") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) < 8:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCION",
|
||||
"msg": f"La Fraccion {val} no alcanza la longitud de 8 caracteres. Capturar en la columna F una Fracción de 8 caracteres.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_int_range(row, "REVFISICA", line_num, -32768, 32767)
|
||||
|
||||
|
||||
def validate_row_fks(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]],
|
||||
valid_uom_codes: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val_d = (row.get("CLAVEMAT") or "").strip()
|
||||
if val_d and valid_material_keys is not None and val_d not in valid_material_keys:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLAVEMAT",
|
||||
"msg": f"Error: (Col. D) El Tipo de Activo Fijo: {val_d} no existe en el Catálogo de Tipos de Activo Fijo. Revisar este Tipo de Activo Fijo en el archivo, en caso de ser correcto actualice los catálogos fijos.",
|
||||
}
|
||||
val_e = (row.get("UNIMED") or "").strip()
|
||||
if val_e and valid_uom_codes is not None and val_e not in valid_uom_codes:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "UNIMED",
|
||||
"msg": f"Error: (Col. E) La Unidad de Medida Comercial: {val_e} no existe en el Catálogo de U.M. Revisar esta Unidad de Medida en el archivo, en caso de ser correcta actualice los catálogos.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_fraction_mex_8(value: str) -> str:
|
||||
"""Primeros 8 caracteres si len>=10, sino hasta 8 (Clarion SUB(ColumnaF, 1, 8))."""
|
||||
if not value:
|
||||
return ""
|
||||
v = value.strip()
|
||||
if len(v) >= 10:
|
||||
return v[:8]
|
||||
return v[:8] if len(v) > 8 else v
|
||||
|
||||
|
||||
def validate_row_fraction_mex_catalog(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_fraction_mex_8: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col F: si no vacía, debe existir en catálogo Mex (GFracGenSifra) o Histórico (Clarion)."""
|
||||
val = (row.get("FRACCION") or "").strip()
|
||||
if not val or valid_fraction_mex_8 is None:
|
||||
return None
|
||||
code_8 = _normalize_fraction_mex_8(val)
|
||||
if not code_8:
|
||||
return None
|
||||
if code_8 in valid_fraction_mex_8:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCION",
|
||||
"msg": (
|
||||
f"Error: (Col. F) La Fraccion Mexicana: {val} no existe en el Catálogo de Fracciones Arancelarias Sifr@ ni en el Historico. "
|
||||
"Revisar esta Fracción Arancelaria en el archivo, en caso de ser correcta Actualizar las Fracciones Arancelarias."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def validate_row_fraction_ame_catalog(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_fraction_ame: Optional[Set[str]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col G: si no vacía, debe existir en catálogo Fracciones Americanas (Clarion GFracAme)."""
|
||||
val = (row.get("FRACCIONAME") or "").strip()
|
||||
if not val or valid_fraction_ame is None:
|
||||
return None
|
||||
if val in valid_fraction_ame:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCIONAME",
|
||||
"msg": (
|
||||
f"Error: (Col. G) La Fraccion Americana: {val} no existe en el Catálogo de Fracciones Americanas. "
|
||||
"Dar de alta la Fracción Americana en el Catálogo de Fracciones Americanas."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def validate_row_tasa_depreciacion(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col H: si viene informada, no puede ser mayor al 100 % (Clarion)."""
|
||||
val = row.get("TASADEPRECIA")
|
||||
if val is None or val == "":
|
||||
return None
|
||||
err = check_decimal_max(row, "TASADEPRECIA", line_num, 100.0, msg=None)
|
||||
if err and "100" in (err.get("msg") or ""):
|
||||
v = (row.get("TASADEPRECIA") or "").strip()
|
||||
err["msg"] = f"Error: (Col. H) La Tasa de Depreciación: {v} no puede ser mayor al 100 %. Ajustar la Tasa de Depreciacion."
|
||||
return err
|
||||
|
||||
|
||||
def validate_row_codigo_producto_cp(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_product_codes_cp: Optional[Set[str]],
|
||||
class_code: str = "",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col J: si no vacía y existe catálogo CP, debe existir en GCodigosProductoCP (Clarion)."""
|
||||
val = (row.get("FRACCIONEXENTAIVA") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if valid_product_codes_cp is None or len(valid_product_codes_cp) == 0:
|
||||
return None
|
||||
if val in valid_product_codes_cp:
|
||||
return None
|
||||
cl = class_code or (row.get("CLASE") or "").strip()
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FRACCIONEXENTAIVA",
|
||||
"msg": (
|
||||
f"Error: (Col. J) La clase: {cl} tiene asignado un código de producto inexistente. "
|
||||
"Capturar un código de producto correcto."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def validaciones_clase(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]],
|
||||
valid_uom_codes: Optional[Set[str]],
|
||||
valid_fraction_mex_8: Optional[Set[str]] = None,
|
||||
valid_fraction_ame: Optional[Set[str]] = None,
|
||||
valid_product_codes_cp: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Reglas compartidas Clarion (VALIDACIONES_CLASE): longitudes, tipos, FKs, fracciones, tasa, código CP."""
|
||||
err = validate_row_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fraction_min(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_types(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fks(row, line_num, valid_material_keys, valid_uom_codes)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fraction_mex_catalog(row, line_num, valid_fraction_mex_8)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fraction_ame_catalog(row, line_num, valid_fraction_ame)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_tasa_depreciacion(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
class_code = (row.get("CLASE") or "").strip()
|
||||
err = validate_row_codigo_producto_cp(
|
||||
row, line_num, valid_product_codes_cp, class_code
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila de clase de material.
|
||||
Flujo Clarion: no ACT → siempre VALIDA_TODA_CLASE; ACT y clase existe → VALIDA_PARCIAL_CLASE;
|
||||
ACT y clase no existe → VALIDA_TODA_CLASE (validación completa) y si pasa se crea en el insert (ADD).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from .common import (
|
||||
validate_row_required,
|
||||
validate_row_required_full,
|
||||
validaciones_clase,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_class(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
actualizar: bool = False,
|
||||
siempre_toda: bool = False,
|
||||
existing_class_codes: Optional[Set[str]] = None,
|
||||
valid_fraction_mex_8: Optional[Set[str]] = None,
|
||||
valid_fraction_ame: Optional[Set[str]] = None,
|
||||
valid_product_codes_cp: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Igual que Clarion:
|
||||
- No ACT (actualizar=False): siempre TODA (B,D,E,F obligatorios + validaciones_clase).
|
||||
- ACT y clase existe: PARCIAL (solo CLASE + validaciones_clase); en insert se actualiza (PUT).
|
||||
- ACT y clase no existe: TODA (validación completa); si pasa, en insert se crea (ADD).
|
||||
siempre_toda fuerza TODA en todos los casos.
|
||||
"""
|
||||
err = validate_row_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
class_code = (row.get("CLASE") or "").strip().upper()[:8]
|
||||
use_full = siempre_toda or not actualizar
|
||||
if actualizar and existing_class_codes is not None and class_code in existing_class_codes:
|
||||
use_full = False
|
||||
|
||||
if use_full:
|
||||
err = validate_row_required_full(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
# En Actualizar, si la clase no existe se valida completa y si pasa se crea en insert (como Clarion ADD).
|
||||
|
||||
return validaciones_clase(
|
||||
row,
|
||||
line_num,
|
||||
valid_material_keys,
|
||||
valid_uom_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_class_partial(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_material_keys: Optional[Set[str]] = None,
|
||||
valid_uom_codes: Optional[Set[str]] = None,
|
||||
valid_fraction_mex_8: Optional[Set[str]] = None,
|
||||
valid_fraction_ame: Optional[Set[str]] = None,
|
||||
valid_product_codes_cp: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Validación parcial (modo Act, clase existente): solo CLASE + validaciones_clase."""
|
||||
err = validate_row_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return validaciones_clase(
|
||||
row,
|
||||
line_num,
|
||||
valid_material_keys,
|
||||
valid_uom_codes,
|
||||
valid_fraction_mex_8=valid_fraction_mex_8,
|
||||
valid_fraction_ame=valid_fraction_ame,
|
||||
valid_product_codes_cp=valid_product_codes_cp,
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for clients_and_providers)
|
||||
@@ -0,0 +1,284 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de clientes y proveedores.
|
||||
Paridad Clarion: procedencia E/N, tipo C/P/A, clave máx 8, SECON, Prosec, Vinculación,
|
||||
Es Empresa Certificada, Transformador/SubMaq, desfase.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum
|
||||
|
||||
RFC_MAX = 30
|
||||
NAME_MAX = 256
|
||||
SHORT_NAME_MAX = 10
|
||||
# Clarion: Col C máx 8 caracteres
|
||||
SHORT_NAME_MAX_CLARION = 8
|
||||
CURP_MAX = 19
|
||||
|
||||
# Valores permitidos Col Q (Tipo programa SECON)
|
||||
TIPO_PROGRAMA_SECON_VALIDOS = frozenset(
|
||||
{"IMMEX", "Maquila", "Pitex", "Ecex", "RECIME", "Pronex", "Ninguno"}
|
||||
)
|
||||
|
||||
|
||||
def check_required_max(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_client_or_provider(val: Optional[str]) -> Optional[ClientOrProviderEnum]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
s = str(val).strip()
|
||||
v = s.lower()
|
||||
# Una sola letra: C, P, A o B (Clarion: A=Ambos; B también usado como Ambos)
|
||||
if len(v) == 1:
|
||||
if v == "c":
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v == "p":
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("a", "b"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
if v in ("client", "cliente"):
|
||||
return ClientOrProviderEnum.CLIENT
|
||||
if v in ("provider", "proveedor"):
|
||||
return ClientOrProviderEnum.PROVIDER
|
||||
if v in ("both", "ambos"):
|
||||
return ClientOrProviderEnum.BOTH
|
||||
return None
|
||||
|
||||
|
||||
def check_tipo_client_provider(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if not tipo_raw:
|
||||
return None
|
||||
if parse_client_or_provider(tipo_raw) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": "Valor no válido. Use Cliente (C), Proveedor (P), Ambos (A) o dejar vacío.",
|
||||
"solution": "Capturar una opción valida: C para Cliente, P para Proveedor, A para Ambos o dejar el campo vacio.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_procedencia(
|
||||
row: Dict[str, Any], line_num: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col A: E o N. Si PAIS (Col L) tiene valor: N → MEX, E → no MEX."""
|
||||
val = (row.get("PROCEDENCIA") or "").strip().upper()
|
||||
if not val:
|
||||
return None
|
||||
if val not in ("E", "N"):
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PROCEDENCIA",
|
||||
"msg": f"Error: (Col. A) El tipo de cliente: {val} es incorrecto.",
|
||||
"solution": "Capturar en la columna A el Tipo de cliente correcto: E para Extranjero y N para Nacional.",
|
||||
}
|
||||
pais = (row.get("PAIS") or "").strip().upper()
|
||||
if not pais:
|
||||
return None
|
||||
if val == "N" and pais != "MEX":
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PAIS",
|
||||
"msg": f"Error: (Col. L) El tipo de cliente es Nacional y tiene el país {pais}, es incorrecto.",
|
||||
"solution": "Capturar en la columna L el país con clave MEX, ya que es un cliente Nacional.",
|
||||
}
|
||||
if val == "E" and pais == "MEX":
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PAIS",
|
||||
"msg": f"Error: (Col. L) El tipo de cliente es Extranjero y tiene el país MEX, es incorrecto.",
|
||||
"solution": "Capturar en la columna L un país con clave diferente de MEX, ya que es un cliente Extranjero.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_short_name_max_clarion(
|
||||
row: Dict[str, Any], line_num: int, max_len: int = SHORT_NAME_MAX_CLARION
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col C: Clave cliente/proveedor máx 8 caracteres (Clarion)."""
|
||||
val = (row.get("SHORT_NAME") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "SHORT_NAME",
|
||||
"msg": f"Error: (Col. C) La Clave de Cliente/Proveedor supera la longitud de caracteres (máx {max_len}).",
|
||||
"solution": f"Capturar en la columna C una Clave de Cliente/Proveedor de {max_len} caracteres como máximo.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_tipo_programa_secon(
|
||||
row: Dict[str, Any], line_num: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col Q: IMMEX, Maquila, Pitex, Ecex, RECIME, Pronex, Ninguno. Si no Ninguno → R y S obligatorios; si Ninguno → R y S vacíos."""
|
||||
q = (row.get("TIPO_PROGRAMA_SECON") or "").strip()
|
||||
r = (row.get("NUM_PROGRAMA_SECON") or "").strip()
|
||||
s = (row.get("FECHA_AUT_SECON") or "").strip()
|
||||
if q and q not in TIPO_PROGRAMA_SECON_VALIDOS:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO_PROGRAMA_SECON",
|
||||
"msg": f"Error: (Col. Q) El Tipo de Programa SECON: {q} es incorrecto.",
|
||||
"solution": "Capturar los Tipos de Programa correctos: IMMEX, Maquila, Pitex, Ecex, RECIME, Pronex o Ninguno.",
|
||||
}
|
||||
if not q or q == "Ninguno":
|
||||
if r:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "NUM_PROGRAMA_SECON",
|
||||
"msg": "Error: (Col. R) El Tipo de Programa es Ninguno y está capturado el número de programa.",
|
||||
"solution": "Borrar la información en la columna R del archivo o asignar un Programa en la columna Q.",
|
||||
}
|
||||
if s:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA_AUT_SECON",
|
||||
"msg": "Error: (Col. S) El Tipo de Programa es Ninguno y está capturada la Fecha de autorización.",
|
||||
"solution": "Borrar la información en la columna S del archivo o asignar un Programa en la columna Q.",
|
||||
}
|
||||
return None
|
||||
# No es Ninguno: R y S obligatorios
|
||||
if not r:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "NUM_PROGRAMA_SECON",
|
||||
"msg": f"Error: (Col. R) El Tipo de Programa es: {q} y no está capturado el número de programa.",
|
||||
"solution": "Capturarlo en la columna R del archivo.",
|
||||
}
|
||||
if not s:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA_AUT_SECON",
|
||||
"msg": f"Error: (Col. S) El Tipo de Programa es: {q} y no está capturada la Fecha de autorización.",
|
||||
"solution": "Capturarla en la columna S del archivo.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_es_prosec_num_aut(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col T (SI/NO): si SI → Col U obligatoria; si no SI → Col U vacía."""
|
||||
t = (row.get("ES_PROSEC") or "").strip().upper()
|
||||
u = (row.get("NUM_AUT_PROSEC") or "").strip()
|
||||
if t and t not in ("SI", "NO"):
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "ES_PROSEC",
|
||||
"msg": f"Error: (Col. T) La opción de si Es Prosec? {t} no es valida.",
|
||||
"solution": "Capturar una opción valida: SI, NO, o dejar el campo vacio (se asigna NO).",
|
||||
}
|
||||
if t == "SI" and not u:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "NUM_AUT_PROSEC",
|
||||
"msg": "Error: (Col. U) Es Prosec? es SI y no está capturado el número de permiso.",
|
||||
"solution": "Capturar en la columna U el número de Permiso PROSEC o cambiar la opcion a NO en la columna T.",
|
||||
}
|
||||
if t != "SI" and u:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "NUM_AUT_PROSEC",
|
||||
"msg": "Error: (Col. U) Es Prosec? no es SI y está capturado el número de permiso.",
|
||||
"solution": "Borrar la información de la columna U o cambiar la opcion a SI en la columna T.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_vinculacion(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col V: solo 0, 1 o 2 (o vacío → 0)."""
|
||||
val = (row.get("VINCULACION") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if val not in ("0", "1", "2"):
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "VINCULACION",
|
||||
"msg": f"Error: (Col. V) La opción de Vinculación: {val} no es valida.",
|
||||
"solution": "Capturar una opción valida: 0, 1, 2 o dejar el campo vacio (se asigna 0).",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_es_empresa_certificada_registro(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col W (SI/NO): si SI → Col X obligatoria; si no SI → Col X vacía."""
|
||||
w = (row.get("ES_EMPRESA_CERTIFICADA") or "").strip().upper()
|
||||
x = (row.get("REGISTRO_EMPRESA_CERT") or "").strip()
|
||||
if w and w not in ("SI", "NO"):
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "ES_EMPRESA_CERTIFICADA",
|
||||
"msg": f"Error: (Col. W) La opción de si Es Empresa Certificada? {w} no es valida.",
|
||||
"solution": "Capturar una opción valida: SI, NO, o dejar el campo vacio (se asigna NO).",
|
||||
}
|
||||
if w == "SI" and not x:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "REGISTRO_EMPRESA_CERT",
|
||||
"msg": "Error: (Col. X) Es Empresa Certificada? es SI y no está capturado el número de empresa certificada.",
|
||||
"solution": "Capturar en la columna X el número de Empresa Certificada o cambiar la opción a NO en la columna W.",
|
||||
}
|
||||
if w != "SI" and x:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "REGISTRO_EMPRESA_CERT",
|
||||
"msg": "Error: (Col. X) Es Empresa Certificada? no es SI y está capturado el número de empresa certificada.",
|
||||
"solution": "Borrar la información de la columna X o cambiar la opción a SI en la columna W.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_transformador_submaq(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col AF: primera letra T, S o N (Transformador, SubMaquila, Ninguno) o vacío → Ninguno."""
|
||||
val = (row.get("TRANSFORMA_SUBMAQ") or "").strip().upper()
|
||||
if not val:
|
||||
return None
|
||||
first = val[:1] if val else ""
|
||||
if first not in ("T", "S", "N"):
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TRANSFORMA_SUBMAQ",
|
||||
"msg": f"Error: (Col. AF) La opción (SUBMAQUILA/TRANSFORMADOR/NINGUNO): {val} no es valida.",
|
||||
"solution": "Capturar una opción valida: Transformador, SubMaquila, Ninguno o dejar el campo vacio (se asigna Ninguno).",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Si COL_EXTRA (Col AH) tiene valor → advertencia de desfase."""
|
||||
val = (row.get("COL_EXTRA") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "COL_EXTRA",
|
||||
"msg": "Advertencia: Podría existir un desfase en esta línea.",
|
||||
"solution": "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta.",
|
||||
}
|
||||
|
||||
|
||||
def parse_active(val: Optional[str]) -> bool:
|
||||
if not val or not str(val).strip():
|
||||
return True
|
||||
v = str(val).strip().lower()
|
||||
if v in ("1", "true", "si", "sí", "yes", "s", "x"):
|
||||
return True
|
||||
if v in ("0", "false", "no", "n"):
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,200 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para ClientProvider, ClientProviderAddress y ClientProviderPrograms.
|
||||
Paridad Clarion: columnas A–AG a modelos.
|
||||
|
||||
El CSV se alimenta en base a las tablas/modelos:
|
||||
- cp_data: atributos de ClientProvider (clients_and_providers). Claves = nombres de columna del modelo.
|
||||
- address_data: atributos de ClientProviderAddress (clients_and_providers_address). Se asignan en tasks.
|
||||
- programs_data: atributos de ClientProviderPrograms (clients_and_providers_programs). Se asignan en tasks.
|
||||
|
||||
Las validaciones (validators) aplican reglas de negocio Clarion y respetan longitudes máximas de los modelos.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientOrProviderEnum
|
||||
|
||||
from .common_validators import (
|
||||
parse_client_or_provider,
|
||||
parse_active,
|
||||
)
|
||||
|
||||
MAX_LEN = {
|
||||
"rfc": 30,
|
||||
"name": 256,
|
||||
"short_name": 10,
|
||||
"curp": 19,
|
||||
"responsible": 80,
|
||||
"position": 30,
|
||||
"incoterm": 19,
|
||||
"email": 100,
|
||||
"phone": 30,
|
||||
"address": 100,
|
||||
"postal_code": 15,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"country": 3,
|
||||
"contact": 50,
|
||||
"extra_information": 399,
|
||||
"web_key": 40,
|
||||
"program": 7,
|
||||
"program_number": 40,
|
||||
"prosec_authorization": 20,
|
||||
"secon_authorization": 20,
|
||||
"manufacturer_id": 25,
|
||||
"tax_id_programs": 30,
|
||||
"broker": 6,
|
||||
"import_broker": 6,
|
||||
"transfer_key": 8,
|
||||
"certified_company_registry": 40,
|
||||
"neighborhood": 40,
|
||||
"exterior_number": 20,
|
||||
"fax": 30,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _parse_date_to_yyyymmdd(val: Any) -> Optional[int]:
|
||||
"""Convierte fecha dd/mm/yyyy o similar a entero YYYYMMDD para secon_auth_date."""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
s = str(val).strip()
|
||||
for fmt in ("%d/%m/%Y", "%Y-%m-%d", "%d-%m-%Y", "%Y/%m/%d"):
|
||||
try:
|
||||
d = datetime.strptime(s[:10], fmt)
|
||||
return d.year * 10000 + d.month * 100 + d.day
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def row_to_client_provider_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
||||
"""
|
||||
Mapea fila normalizada a datos para ClientProvider, ClientProviderAddress y ClientProviderPrograms.
|
||||
Devuelve (cp_data, address_data_or_none, programs_data_or_none).
|
||||
Para compatibilidad con Clarion: se requiere RFC o SHORT_NAME para considerar la fila válida.
|
||||
"""
|
||||
rfc = _str_or_none(row_norm.get("RFC"), MAX_LEN["rfc"])
|
||||
short_name = _str_or_none(row_norm.get("SHORT_NAME"), MAX_LEN["short_name"])
|
||||
if not rfc and not short_name:
|
||||
return ({}, None, None)
|
||||
|
||||
client_or_provider = parse_client_or_provider(row_norm.get("TIPO")) or ClientOrProviderEnum.BOTH
|
||||
procedencia = _str_or_none(row_norm.get("PROCEDENCIA"), 1)
|
||||
if procedencia:
|
||||
procedencia = procedencia.upper()[:1]
|
||||
|
||||
# Vinculación 0/1/2 → string
|
||||
vinc = (row_norm.get("VINCULACION") or "").strip()
|
||||
linking = None
|
||||
if vinc in ("0", "1", "2"):
|
||||
linking = vinc
|
||||
|
||||
# Transformador/SubMaquila: primera letra T/S/N
|
||||
trans = (row_norm.get("TRANSFORMA_SUBMAQ") or "").strip().upper()[:1]
|
||||
transform_subassembly = trans if trans in ("T", "S", "N") else None
|
||||
|
||||
cp_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"rfc": rfc or None,
|
||||
"name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]),
|
||||
"short_name": short_name,
|
||||
"curp": _str_or_none(row_norm.get("CURP"), MAX_LEN["curp"]),
|
||||
"client_or_provider": client_or_provider,
|
||||
"type_nat_foreign": procedencia,
|
||||
"linking": linking,
|
||||
"transform_subassembly": transform_subassembly,
|
||||
"extra_information": _str_or_none(row_norm.get("INFORMACION_EXTRA"), MAX_LEN["extra_information"]),
|
||||
"web_key": _str_or_none(row_norm.get("CLAVE_WEB"), MAX_LEN["web_key"]),
|
||||
"responsible": _str_or_none(row_norm.get("RESPONSABLE"), MAX_LEN["responsible"]),
|
||||
"position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]),
|
||||
"incoterm": _str_or_none(row_norm.get("INCOTERM"), MAX_LEN["incoterm"]),
|
||||
"is_active": parse_active(row_norm.get("ACTIVO")),
|
||||
"is_national_provider": True if procedencia == "N" else (False if procedencia == "E" else None),
|
||||
}
|
||||
|
||||
# Address
|
||||
email = _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"])
|
||||
phone = _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"])
|
||||
address_str = _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"])
|
||||
address_data = None
|
||||
if email or phone or address_str or _str_or_none(row_norm.get("CODIGO POSTAL")) or _str_or_none(row_norm.get("COLONIA")) or _str_or_none(row_norm.get("NUM_EXT")):
|
||||
address_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"streets": address_str,
|
||||
"exterior_number": _str_or_none(row_norm.get("NUM_EXT"), MAX_LEN["exterior_number"]),
|
||||
"neighborhood": _str_or_none(row_norm.get("COLONIA"), MAX_LEN["neighborhood"]),
|
||||
"postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"phone": phone,
|
||||
"fax_number": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]),
|
||||
"email": email,
|
||||
"contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]),
|
||||
}
|
||||
|
||||
# Programs (SECON, PROSEC, empresa certificada, etc.)
|
||||
tipo_prog = _str_or_none(row_norm.get("TIPO_PROGRAMA_SECON"), MAX_LEN["program"])
|
||||
if tipo_prog and tipo_prog.lower() == "ninguno":
|
||||
tipo_prog = None
|
||||
num_prog = _str_or_none(row_norm.get("NUM_PROGRAMA_SECON"), MAX_LEN["program_number"])
|
||||
fecha_secon = _parse_date_to_yyyymmdd(row_norm.get("FECHA_AUT_SECON"))
|
||||
es_prosec = (row_norm.get("ES_PROSEC") or "").strip().upper()
|
||||
prosec_val = "1" if es_prosec == "SI" else ("0" if es_prosec else None)
|
||||
num_aut_prosec = _str_or_none(row_norm.get("NUM_AUT_PROSEC"), MAX_LEN["prosec_authorization"])
|
||||
es_cert = (row_norm.get("ES_EMPRESA_CERTIFICADA") or "").strip().upper()
|
||||
is_certified = "S" if es_cert == "SI" else ("N" if es_cert else None)
|
||||
reg_cert = _str_or_none(row_norm.get("REGISTRO_EMPRESA_CERT"), MAX_LEN["certified_company_registry"])
|
||||
vinc_prop = row_norm.get("VINCULACION")
|
||||
applied_proportion = None
|
||||
if vinc_prop is not None and str(vinc_prop).strip() in ("0", "1", "2"):
|
||||
try:
|
||||
applied_proportion = Decimal(str(vinc_prop).strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
programs_data = None
|
||||
if (
|
||||
tipo_prog or num_prog or fecha_secon is not None or prosec_val or num_aut_prosec
|
||||
or is_certified or reg_cert
|
||||
or _str_or_none(row_norm.get("MANUFACTURER_ID"))
|
||||
or _str_or_none(row_norm.get("TAX_ID_PROGRAMS"))
|
||||
or _str_or_none(row_norm.get("BROKER_EXPO"))
|
||||
or _str_or_none(row_norm.get("BROKER_IMPO"))
|
||||
or _str_or_none(row_norm.get("CLAVE_TRANSFER"))
|
||||
or applied_proportion is not None
|
||||
):
|
||||
programs_data = {
|
||||
"program": tipo_prog[:7] if tipo_prog else None,
|
||||
"program_number": num_prog,
|
||||
"secon_authorization": num_prog,
|
||||
"secon_auth_date": fecha_secon,
|
||||
"prosec": prosec_val,
|
||||
"prosec_authorization": num_aut_prosec,
|
||||
"is_certified_company": is_certified,
|
||||
"certified_company_registry": reg_cert,
|
||||
"manufacturer_id": _str_or_none(row_norm.get("MANUFACTURER_ID"), MAX_LEN["manufacturer_id"]),
|
||||
"tax_id": _str_or_none(row_norm.get("TAX_ID_PROGRAMS"), MAX_LEN["tax_id_programs"]),
|
||||
"broker": _str_or_none(row_norm.get("BROKER_EXPO"), MAX_LEN["broker"]),
|
||||
"import_broker": _str_or_none(row_norm.get("BROKER_IMPO"), MAX_LEN["import_broker"]),
|
||||
"transfer_key": _str_or_none(row_norm.get("CLAVE_TRANSFER"), MAX_LEN["transfer_key"]),
|
||||
"applied_proportion": applied_proportion,
|
||||
}
|
||||
|
||||
return (cp_data, address_data, programs_data)
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -81,7 +82,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
@@ -0,0 +1,423 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Clientes y Proveedores.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_client_provider
|
||||
from .common.mappers import row_to_client_provider_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cp"
|
||||
|
||||
# Para routes.py
|
||||
CP_IMPORT_FILE_PREFIX = "cp_import_file:"
|
||||
CP_IMPORT_META_PREFIX = "cp_import_meta:"
|
||||
CP_IMPORT_ERROR_LINES_PREFIX = "cp_import_error_lines:"
|
||||
CP_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
actualizar = meta.get("actualizar", False)
|
||||
existing_short_names: Set[str] = set()
|
||||
if actualizar:
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if (cp.short_name or "").strip():
|
||||
existing_short_names.add((cp.short_name or "").strip())
|
||||
except Exception as e:
|
||||
logger.warning("CP import: could not load existing short_names for ACT: %s", e)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_client_provider(
|
||||
row_norm, i,
|
||||
actualizar=actualizar,
|
||||
existing_short_names=existing_short_names if actualizar else None,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("CP import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("CP import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CP import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CP import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
actualizar = meta.get("actualizar", False)
|
||||
existing_short_names: Set[str] = set()
|
||||
if actualizar:
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if (cp.short_name or "").strip():
|
||||
existing_short_names.add((cp.short_name or "").strip())
|
||||
except Exception as e:
|
||||
logger.warning("CP commit: could not load existing short_names for ACT: %s", e)
|
||||
|
||||
from api.v1.modules.a76.clients_and_providers.models import (
|
||||
ClientProvider,
|
||||
ClientProviderAddress,
|
||||
ClientProviderPrograms,
|
||||
)
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_rfc: Dict[str, ClientProvider] = {}
|
||||
existing_by_short_name: Dict[str, ClientProvider] = {}
|
||||
for cp in (
|
||||
session.query(ClientProvider)
|
||||
.filter(
|
||||
ClientProvider.tenant_id == tenant_id,
|
||||
ClientProvider.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
rfc_key = (cp.rfc or "").strip()
|
||||
if rfc_key:
|
||||
existing_by_rfc[rfc_key] = cp
|
||||
sn_key = (cp.short_name or "").strip()
|
||||
if sn_key:
|
||||
existing_by_short_name[sn_key] = cp
|
||||
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_client_provider(
|
||||
row_norm, i,
|
||||
actualizar=actualizar,
|
||||
existing_short_names=existing_short_names if actualizar else None,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
cp_data, address_data, programs_data = row_to_client_provider_data(row_norm, tenant_id, company_id)
|
||||
if not cp_data:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "Fila sin RFC ni Clave"})
|
||||
continue
|
||||
if actualizar:
|
||||
short_name_key = (cp_data.get("short_name") or "").strip()
|
||||
if not short_name_key:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "Clave (SHORT_NAME) requerida en modo Actualizar"})
|
||||
continue
|
||||
existing = existing_by_short_name.get(short_name_key)
|
||||
if not existing:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "Clave no existe en catálogo"})
|
||||
continue
|
||||
# Merge: fill from existing when csv value is empty
|
||||
for k, v in cp_data.items():
|
||||
if k in ("tenant_id", "company_id"):
|
||||
continue
|
||||
if v is None or (isinstance(v, str) and not v.strip()):
|
||||
existing_val = getattr(existing, k, None)
|
||||
if existing_val is not None:
|
||||
cp_data[k] = existing_val
|
||||
for k, v in cp_data.items():
|
||||
if k not in ("tenant_id", "company_id"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
# Update address if present
|
||||
if address_data and existing.address:
|
||||
addr = existing.address
|
||||
for k, v in address_data.items():
|
||||
if k not in ("tenant_id", "company_id") and v is not None:
|
||||
setattr(addr, k, v)
|
||||
session.add(addr)
|
||||
elif address_data:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=existing.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_data.get("streets"),
|
||||
postal_code=address_data.get("postal_code"),
|
||||
city=address_data.get("city"),
|
||||
state=address_data.get("state"),
|
||||
country=address_data.get("country"),
|
||||
phone=address_data.get("phone"),
|
||||
email=address_data.get("email"),
|
||||
contact=address_data.get("contact"),
|
||||
exterior_number=address_data.get("exterior_number"),
|
||||
neighborhood=address_data.get("neighborhood"),
|
||||
fax_number=address_data.get("fax_number"),
|
||||
)
|
||||
session.add(addr)
|
||||
# Update or create programs
|
||||
if programs_data:
|
||||
prog = session.query(ClientProviderPrograms).filter(
|
||||
ClientProviderPrograms.client_id == existing.id,
|
||||
).first()
|
||||
if prog:
|
||||
for k, v in programs_data.items():
|
||||
if v is not None:
|
||||
setattr(prog, k, v)
|
||||
session.add(prog)
|
||||
else:
|
||||
prog = ClientProviderPrograms(
|
||||
client_id=existing.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
**{k: v for k, v in programs_data.items() if v is not None},
|
||||
)
|
||||
session.add(prog)
|
||||
else:
|
||||
# Alta / Reemplazar: key por RFC
|
||||
if not cp_data.get("rfc"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "RFC requerido"})
|
||||
continue
|
||||
rfc = cp_data["rfc"]
|
||||
existing = existing_by_rfc.get(rfc)
|
||||
if existing:
|
||||
for k, v in cp_data.items():
|
||||
if k not in ("tenant_id", "company_id", "rfc"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
if address_data and existing.address:
|
||||
addr = existing.address
|
||||
for k, v in address_data.items():
|
||||
if k not in ("tenant_id", "company_id") and v is not None:
|
||||
setattr(addr, k, v)
|
||||
session.add(addr)
|
||||
elif address_data:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=existing.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_data.get("streets"),
|
||||
postal_code=address_data.get("postal_code"),
|
||||
city=address_data.get("city"),
|
||||
state=address_data.get("state"),
|
||||
country=address_data.get("country"),
|
||||
phone=address_data.get("phone"),
|
||||
email=address_data.get("email"),
|
||||
contact=address_data.get("contact"),
|
||||
exterior_number=address_data.get("exterior_number"),
|
||||
neighborhood=address_data.get("neighborhood"),
|
||||
fax_number=address_data.get("fax_number"),
|
||||
)
|
||||
session.add(addr)
|
||||
if programs_data:
|
||||
prog = session.query(ClientProviderPrograms).filter(
|
||||
ClientProviderPrograms.client_id == existing.id,
|
||||
).first()
|
||||
if prog:
|
||||
for k, v in programs_data.items():
|
||||
if v is not None:
|
||||
setattr(prog, k, v)
|
||||
session.add(prog)
|
||||
else:
|
||||
prog = ClientProviderPrograms(
|
||||
client_id=existing.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
**{k: v for k, v in programs_data.items() if v is not None},
|
||||
)
|
||||
session.add(prog)
|
||||
else:
|
||||
new_cp = ClientProvider(**cp_data)
|
||||
session.add(new_cp)
|
||||
session.flush()
|
||||
existing_by_rfc[rfc] = new_cp
|
||||
if (new_cp.short_name or "").strip():
|
||||
existing_by_short_name[(new_cp.short_name or "").strip()] = new_cp
|
||||
inserted_count += 1
|
||||
if address_data:
|
||||
addr = ClientProviderAddress(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
streets=address_data.get("streets"),
|
||||
postal_code=address_data.get("postal_code"),
|
||||
city=address_data.get("city"),
|
||||
state=address_data.get("state"),
|
||||
country=address_data.get("country"),
|
||||
phone=address_data.get("phone"),
|
||||
email=address_data.get("email"),
|
||||
contact=address_data.get("contact"),
|
||||
exterior_number=address_data.get("exterior_number"),
|
||||
neighborhood=address_data.get("neighborhood"),
|
||||
fax_number=address_data.get("fax_number"),
|
||||
)
|
||||
session.add(addr)
|
||||
if programs_data:
|
||||
prog = ClientProviderPrograms(
|
||||
client_id=new_cp.id,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
**{k: v for k, v in programs_data.items() if v is not None},
|
||||
)
|
||||
session.add(prog)
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("CP import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("CP import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if inserted_count == 0 and updated_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron ni actualizaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0 and updated_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("CP import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Clientes y Proveedores (EstructuraCatClienteProv.xls).
|
||||
Layout Clarion: Col A = PROCEDENCIA (E/N), B = TIPO (C/P/A), C = CLAVE (máx 8), D = NOMBRE, E = RFC, F–AG.
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
Correspondencia Clarion → canonical: A=PROCEDENCIA, B=TIPO, C=SHORT_NAME, D=NOMBRE, E=RFC, F=CALLES(DIRECCION),
|
||||
G=NUM_EXT, H=CODIGO POSTAL, I=COLONIA, J=CIUDAD, K=ESTADO, L=PAIS, M=TELEFONO, N=FAX, O=EMAIL, P=CURP,
|
||||
Q=TIPO_PROGRAMA_SECON, R=NUM_PROGRAMA_SECON, S=FECHA_AUT_SECON, T=ES_PROSEC, U=NUM_AUT_PROSEC, V=VINCULACION,
|
||||
W=ES_EMPRESA_CERTIFICADA, X=REGISTRO_EMPRESA_CERT, Y=INFORMACION_EXTRA, Z=CONTACTO, AA=MANUFACTURER_ID,
|
||||
AB=TAX_ID_PROGRAMS, AC=BROKER_EXPO, AD=BROKER_IMPO, AE=CLAVE_TRANSFER, AF=TRANSFORMA_SUBMAQ, AG=CLAVE_WEB,
|
||||
AH=COL_EXTRA (desfase).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"client_providers": [
|
||||
# Col A - Procedencia (E=Extranjero, N=Nacional)
|
||||
{"canonical": "PROCEDENCIA", "aliases": ["TIPO PROCEDENCIA", "EXTranjero/Nacional", "E/N"]},
|
||||
# Col B - Tipo Cliente (C/P/A)
|
||||
{"canonical": "TIPO", "aliases": ["CLIENT_OR_PROVIDER", "TIPO ENTIDAD", "CLIENTE O PROVEEDOR"]},
|
||||
# Col C - Clave cliente/proveedor (máx 8 Clarion)
|
||||
{"canonical": "SHORT_NAME", "aliases": ["CLAVE", "CLAVE CORTA", "NOMBRE CORTO", "SIGLAS"]},
|
||||
# Col D - Nombre
|
||||
{"canonical": "NOMBRE", "aliases": ["RAZON SOCIAL", "NAME", "RAZON SOCIAL O NOMBRE"]},
|
||||
# Col E - RFC
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID", "IDENTIFICADOR FISCAL", "IDENTIFICACION FISCAL"]},
|
||||
# Col F - Calles
|
||||
{"canonical": "DIRECCION", "aliases": ["CALLES", "DOMICILIO", "DIRECCION FISCAL", "CALLE"]},
|
||||
# Col G - Número exterior
|
||||
{"canonical": "NUM_EXT", "aliases": ["NUM EXTERIOR", "NUMERO EXTERIOR", "NO EXT"]},
|
||||
# Col H - Código postal
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
# Col I - Colonia
|
||||
{"canonical": "COLONIA", "aliases": []},
|
||||
# Col J - Ciudad
|
||||
{"canonical": "CIUDAD", "aliases": ["MUNICIPIO"]},
|
||||
# Col K - Estado
|
||||
{"canonical": "ESTADO", "aliases": []},
|
||||
# Col L - País M3
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY", "PAIS M3"]},
|
||||
# Col M - Teléfono
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL", "TELEFONO CONTACTO"]},
|
||||
# Col N - Fax
|
||||
{"canonical": "FAX", "aliases": ["NUMERO DE FAX", "FAX NUMBER"]},
|
||||
# Col O - Email
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
|
||||
# Col P - CURP
|
||||
{"canonical": "CURP", "aliases": []},
|
||||
# Col Q - Tipo programa SECON
|
||||
{"canonical": "TIPO_PROGRAMA_SECON", "aliases": ["PROGRAMA", "TIPO PROGRAMA SECON", "TIPO PROGRAMA"]},
|
||||
# Col R - Número programa SECON
|
||||
{"canonical": "NUM_PROGRAMA_SECON", "aliases": ["NUM PROGRAMA SECON", "NUMERO PROGRAMA"]},
|
||||
# Col S - Fecha autorización SECON
|
||||
{"canonical": "FECHA_AUT_SECON", "aliases": ["FECHA AUT SECON", "FECHA AUTORIZACION SECON"]},
|
||||
# Col T - Es Prosec?
|
||||
{"canonical": "ES_PROSEC", "aliases": ["ES PROSEC", "PROSEC"]},
|
||||
# Col U - Número autorización PROSEC
|
||||
{"canonical": "NUM_AUT_PROSEC", "aliases": ["NUM AUT PROSEC", "NUMERO AUTORIZACION PROSEC"]},
|
||||
# Col V - Vinculación (0/1/2)
|
||||
{"canonical": "VINCULACION", "aliases": []},
|
||||
# Col W - Es Empresa Certificada?
|
||||
{"canonical": "ES_EMPRESA_CERTIFICADA", "aliases": ["ES EMPRESA CERTIFICADA", "EMPRESA CERTIFICADA"]},
|
||||
# Col X - Registro empresa certificada
|
||||
{"canonical": "REGISTRO_EMPRESA_CERT", "aliases": ["REGISTRO EMPRESA CERTIFICADA", "REGISTRO EMP CERT"]},
|
||||
# Col Y - Información extra
|
||||
{"canonical": "INFORMACION_EXTRA", "aliases": ["INFORMACION EXTRA", "INFO EXTRA"]},
|
||||
# Col Z - Contacto
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT", "PERSONA CONTACTO"]},
|
||||
# Col AA - Manufacturer ID
|
||||
{"canonical": "MANUFACTURER_ID", "aliases": ["MANUFACTURERID", "MANUFACTURER ID"]},
|
||||
# Col AB - Tax ID (programas)
|
||||
{"canonical": "TAX_ID_PROGRAMS", "aliases": ["TAX ID", "TAXID PROGRAMS"]},
|
||||
# Col AC - Broker exportación
|
||||
{"canonical": "BROKER_EXPO", "aliases": ["BROKER EXPO", "BROKER EXPORTACION"]},
|
||||
# Col AD - Broker importación
|
||||
{"canonical": "BROKER_IMPO", "aliases": ["BROKER IMPO", "BROKER IMPORTACION"]},
|
||||
# Col AE - Clave transfer
|
||||
{"canonical": "CLAVE_TRANSFER", "aliases": ["CLAVE TRANSFER", "TRANSFER KEY"]},
|
||||
# Col AF - Transformador/SubMaquila/Ninguno (T/S/N)
|
||||
{"canonical": "TRANSFORMA_SUBMAQ", "aliases": ["TRANSFORMADOR SUBMAQUILA", "TRASFORMA SUBMAQ"]},
|
||||
# Col AG - Clave interface web
|
||||
{"canonical": "CLAVE_WEB", "aliases": ["CLAVE WEB", "CLAVE INTERFACE WEB", "WEB KEY"]},
|
||||
# Col AH - Desfase (si tiene valor → advertencia)
|
||||
{"canonical": "COL_EXTRA", "aliases": ["DESFASE", "COLUMNA EXTRA"]},
|
||||
# Legacy / otros
|
||||
{"canonical": "RESPONSABLE", "aliases": ["RESPONSABLE AREA"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "INCOTERM", "aliases": []},
|
||||
{"canonical": "ACTIVO", "aliases": ["IS_ACTIVE", "ACTIVE", "ESTADO ACTIVO"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla client_providers."""
|
||||
cols = TEMPLATE_COLUMNS.get("client_providers")
|
||||
if not cols:
|
||||
return {}
|
||||
lookup: Dict[str, str] = {}
|
||||
for item in cols:
|
||||
canonical = item["canonical"]
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
return lookup
|
||||
|
||||
|
||||
def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any]:
|
||||
"""Fila CSV con solo columnas de la plantilla, en nombres canónicos."""
|
||||
lookup = build_normalized_lookup(normalize_header_fn)
|
||||
if not lookup:
|
||||
return {normalize_header_fn(k): v for k, v in row.items()}
|
||||
out: Dict[str, Any] = {}
|
||||
for csv_header, value in row.items():
|
||||
key_norm = normalize_header_fn(csv_header)
|
||||
if key_norm in lookup:
|
||||
out[lookup[key_norm]] = value
|
||||
return out
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_client_provider
|
||||
|
||||
__all__ = ["validate_row_client_provider"]
|
||||
@@ -0,0 +1,177 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de clientes y proveedores.
|
||||
Paridad Clarion: VALIDA_TODA_CLIENTE_O_PROV, VALIDA_PARCIAL_CLIENTE_O_PROV, VALIDACIONES_CLIENTE_O_PROV.
|
||||
Origen de reglas: código legacy Clarion (EstructuraCatClienteProv).
|
||||
Mapa columnas: A=PROCEDENCIA, B=TIPO, C=SHORT_NAME, D=NOMBRE, E=RFC, F–AG (ver template_config).
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
RFC_MAX,
|
||||
NAME_MAX,
|
||||
SHORT_NAME_MAX_CLARION,
|
||||
CURP_MAX,
|
||||
check_required_max,
|
||||
check_max_length,
|
||||
check_tipo_client_provider,
|
||||
check_procedencia,
|
||||
check_short_name_max_clarion,
|
||||
check_tipo_programa_secon,
|
||||
check_es_prosec_num_aut,
|
||||
check_vinculacion,
|
||||
check_es_empresa_certificada_registro,
|
||||
check_transformador_submaq,
|
||||
check_desfase,
|
||||
)
|
||||
|
||||
|
||||
# --- Mensajes obligatorios (VALIDA_TODA) ---
|
||||
MSG_CAMPOS_OBLIGATORIOS = "Existen campos vacíos que son obligatorios: {campos}."
|
||||
MSG_CAMPOS_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta."
|
||||
MSG_CLAVE_VACIA = (
|
||||
"Error: La Clave de Cliente/Proveedor está vacía y no se pueden hacer las validaciones."
|
||||
)
|
||||
MSG_CLAVE_VACIA_SOLUCION = (
|
||||
"Capturar una Clave de Cliente/Proveedor nueva o existente a la cual desee agregar, remplazar o actualizar campos."
|
||||
)
|
||||
MSG_CLAVE_NO_EXISTE = "Error: (Col.C) Clave de Proveedor/Cliente no existe."
|
||||
MSG_CLAVE_NO_EXISTE_SOLUCION = "La clave debe existir en el catálogo cuando el modo es Actualizar."
|
||||
|
||||
|
||||
def validate_row_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Si COL_EXTRA tiene valor → advertencia de desfase."""
|
||||
return check_desfase(row, line_num)
|
||||
|
||||
|
||||
def validate_row_clave_vacia(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Clave (SHORT_NAME) vacía → error inmediato."""
|
||||
val = (row.get("SHORT_NAME") or "").strip()
|
||||
if not val:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "SHORT_NAME",
|
||||
"msg": MSG_CLAVE_VACIA,
|
||||
"solution": MSG_CLAVE_VACIA_SOLUCION,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validaciones_cliente_o_prov(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
VALIDACIONES_CLIENTE_O_PROV: reglas de dominio compartidas.
|
||||
Col A (E/N + coherencia con L), B (C/P/A), C (máx 8), Q/R/S, T/U, V, W/X, AF.
|
||||
"""
|
||||
err = check_procedencia(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_tipo_client_provider(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_short_name_max_clarion(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_tipo_programa_secon(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_es_prosec_num_aut(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_vinculacion(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_es_empresa_certificada_registro(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_transformador_submaq(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def valida_toda_cliente_o_prov(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
actualizar: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
VALIDA_TODA: obligatorios Col A (Procedencia), Col D (Nombre) cuando no es ACT.
|
||||
Si modo ACT y clave no existe → error se devuelve antes (en validate_row_client_provider).
|
||||
Luego ejecuta VALIDACIONES_CLIENTE_O_PROV.
|
||||
"""
|
||||
campos_oblig = []
|
||||
# Col A - Procedencia obligatoria
|
||||
if not (row.get("PROCEDENCIA") or "").strip():
|
||||
campos_oblig.append("(Col.A) Tipo Cliente Procedencia (E/N)")
|
||||
# Col D - Nombre obligatorio solo cuando no es actualizar
|
||||
if not actualizar and not (row.get("NOMBRE") or "").strip():
|
||||
campos_oblig.append("(Col.D) Nombre")
|
||||
if campos_oblig:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PROCEDENCIA" if not (row.get("PROCEDENCIA") or "").strip() else "NOMBRE",
|
||||
"msg": MSG_CAMPOS_OBLIGATORIOS.format(campos=", ".join(campos_oblig)),
|
||||
"solution": MSG_CAMPOS_OBLIGATORIOS_SOLUCION,
|
||||
}
|
||||
return validaciones_cliente_o_prov(row, line_num)
|
||||
|
||||
|
||||
def valida_parcial_cliente_o_prov(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""VALIDA_PARCIAL: solo VALIDACIONES_CLIENTE_O_PROV (no exige A ni D)."""
|
||||
return validaciones_cliente_o_prov(row, line_num)
|
||||
|
||||
|
||||
def validate_row_client_provider(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
actualizar: bool = False,
|
||||
existing_short_names: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de clientes y proveedores.
|
||||
- Desfase (COL_EXTRA) primero.
|
||||
- Clave vacía → error.
|
||||
- Si actualizar y clave no existe en existing_short_names → error "Clave no existe".
|
||||
- Si actualizar y clave existe → VALIDA_PARCIAL (solo validaciones comunes).
|
||||
- Si no actualizar o clave no existe → VALIDA_TODA (A y D obligatorios cuando aplique, luego comunes).
|
||||
Además se validan RFC (requerido max 30), longitudes NOMBRE/SHORT_NAME/CURP para compatibilidad.
|
||||
"""
|
||||
err = validate_row_desfase(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_clave_vacia(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
short_name = (row.get("SHORT_NAME") or "").strip()
|
||||
existing = existing_short_names or set()
|
||||
use_partial = actualizar and short_name in existing
|
||||
|
||||
if actualizar and short_name and short_name not in existing:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "SHORT_NAME",
|
||||
"msg": MSG_CLAVE_NO_EXISTE,
|
||||
"solution": MSG_CLAVE_NO_EXISTE_SOLUCION,
|
||||
}
|
||||
|
||||
if use_partial:
|
||||
err = valida_parcial_cliente_o_prov(row, line_num)
|
||||
else:
|
||||
err = valida_toda_cliente_o_prov(row, line_num, actualizar=actualizar)
|
||||
if err:
|
||||
return err
|
||||
|
||||
# Compatibilidad: RFC requerido y longitudes (como antes)
|
||||
err = check_required_max(row, "RFC", RFC_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "NOMBRE", NAME_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "SHORT_NAME", 10, line_num) # modelo permite 10
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(row, "CURP", CURP_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila cliente/proveedor.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common import validate_row_client_provider
|
||||
|
||||
__all__ = ["validate_row_client_provider"]
|
||||
@@ -0,0 +1 @@
|
||||
# Shared utilities for layouts_csv imports (storage, normalize, csv, meta, responses)
|
||||
96
backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py
Normal file
96
backend/api/v1/modules/a76/layouts_csv/common/csv_reader.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
Lectura de CSV con detección de delimitador (compartida por layouts_csv).
|
||||
Si se pasa fieldnames, no se usa la primera fila como cabecera y se toma como dato (CSV sin cabeceras).
|
||||
Si se pasa headerless_first_cell_values, se detecta si la primera fila es cabecera o dato por el valor de la primera celda.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
from typing import Iterator, Tuple, Dict, Any, Optional, List, Set
|
||||
|
||||
|
||||
def _normalize_empty_headers(headers: List[str]) -> List[str]:
|
||||
"""Sustituye cabeceras vacías por _COL_0_, _COL_1_, ... para que DictReader no colapse columnas."""
|
||||
result: List[str] = []
|
||||
empty_idx = 0
|
||||
for h in headers:
|
||||
if (h or "").strip() == "":
|
||||
result.append(f"_COL_{empty_idx}_")
|
||||
empty_idx += 1
|
||||
else:
|
||||
result.append(h)
|
||||
return result
|
||||
|
||||
|
||||
def iter_csv_rows(
|
||||
file_path: str,
|
||||
fieldnames: Optional[List[str]] = None,
|
||||
headerless_first_cell_values: Optional[Set[str]] = None,
|
||||
headerless_second_cell_key_pattern: Optional[str] = None,
|
||||
) -> Iterator[Tuple[int, Dict[str, Any]]]:
|
||||
"""
|
||||
Abre el CSV, detecta dialecto y devuelve (line_num, row_dict) por cada fila.
|
||||
line_num empieza en 1 (primera fila de datos).
|
||||
Si fieldnames y headerless_first_cell_values se pasan: si la primera celda de la primera fila
|
||||
(quitando BOM, strip, upper) está en headerless_first_cell_values, se trata como dato y se usan fieldnames.
|
||||
headerless_second_cell_key_pattern se ignora si no se usa (reservado para otros layouts).
|
||||
"""
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
if fieldnames and headerless_first_cell_values is not None:
|
||||
first_line = f.readline()
|
||||
if not first_line:
|
||||
return
|
||||
row_reader = csv.reader(io.StringIO(first_line), dialect=dialect)
|
||||
first_cells = next(row_reader, None)
|
||||
if not first_cells:
|
||||
return
|
||||
first_cell_clean = (first_cells[0] or "").lstrip("\ufeff").strip().upper()
|
||||
use_headerless = first_cell_clean in headerless_first_cell_values
|
||||
if use_headerless:
|
||||
pad = len(fieldnames) - len(first_cells)
|
||||
cells = first_cells[: len(fieldnames)] + ([""] * pad if pad > 0 else [])
|
||||
yield 1, dict(zip(fieldnames, cells))
|
||||
reader = csv.DictReader(f, fieldnames=fieldnames, dialect=dialect, restval="")
|
||||
for i, row in enumerate(reader, start=2):
|
||||
yield i, dict(row)
|
||||
return
|
||||
f.seek(0)
|
||||
first_line = f.readline()
|
||||
if not first_line:
|
||||
return
|
||||
row_reader = csv.reader(io.StringIO(first_line), dialect=dialect)
|
||||
raw_headers = next(row_reader, None)
|
||||
if not raw_headers:
|
||||
return
|
||||
normalized = _normalize_empty_headers(raw_headers)
|
||||
reader = csv.DictReader(f, fieldnames=normalized, dialect=dialect, restval="")
|
||||
for i, row in enumerate(reader, start=1):
|
||||
yield i, row
|
||||
elif fieldnames:
|
||||
reader = csv.DictReader(f, fieldnames=fieldnames, dialect=dialect)
|
||||
for i, row in enumerate(reader, start=1):
|
||||
yield i, dict(row)
|
||||
else:
|
||||
first_line = f.readline()
|
||||
if not first_line:
|
||||
return
|
||||
row_reader = csv.reader(io.StringIO(first_line), dialect=dialect)
|
||||
raw_headers = next(row_reader, None)
|
||||
if not raw_headers:
|
||||
return
|
||||
normalized = _normalize_empty_headers(raw_headers)
|
||||
reader = csv.DictReader(f, fieldnames=normalized, dialect=dialect, restval="")
|
||||
for i, row in enumerate(reader, start=1):
|
||||
yield i, row
|
||||
|
||||
|
||||
def count_csv_rows(file_path: str, has_header: bool = True) -> int:
|
||||
"""Cuenta filas del CSV. Si has_header=True (por defecto), no cuenta la cabecera."""
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_lines = sum(1 for _ in f)
|
||||
return total_lines if not has_header else max(0, total_lines - 1)
|
||||
35
backend/api/v1/modules/a76/layouts_csv/common/meta.py
Normal file
35
backend/api/v1/modules/a76/layouts_csv/common/meta.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Carga y guardado de meta (tenant_id, company_id) para imports CSV.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, Tuple
|
||||
|
||||
|
||||
def load_meta(file_path: str) -> Dict[str, Any]:
|
||||
"""Carga meta desde archivo .meta.json asociado al CSV. Devuelve dict vacío si no existe o falla."""
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
if not os.path.exists(meta_path):
|
||||
return {}
|
||||
try:
|
||||
with open(meta_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def require_tenant_context(file_path: str) -> Tuple[int, int]:
|
||||
"""
|
||||
Obtiene tenant_id y company_id del meta. Lanza ValueError si faltan.
|
||||
"""
|
||||
meta = load_meta(file_path)
|
||||
tenant_id = meta.get("tenant_id")
|
||||
company_id = meta.get("company_id")
|
||||
if not tenant_id or not company_id:
|
||||
raise ValueError("Falta contexto (tenant/company)")
|
||||
return int(tenant_id), int(company_id)
|
||||
|
||||
|
||||
def get_meta_path(file_path: str) -> str:
|
||||
"""Ruta del archivo .meta.json para un CSV."""
|
||||
return file_path.replace(".csv", ".meta.json")
|
||||
16
backend/api/v1/modules/a76/layouts_csv/common/normalize.py
Normal file
16
backend/api/v1/modules/a76/layouts_csv/common/normalize.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""
|
||||
Normalización de cabeceras CSV (compartida por todos los módulos layouts_csv).
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def normalize_header(name: Optional[str]) -> str:
|
||||
"""Normaliza nombre de columna: NFKD, mayúsculas, sin acentos, espacios colapsados."""
|
||||
if not name:
|
||||
return ""
|
||||
name = unicodedata.normalize("NFKD", str(name)).upper()
|
||||
name = "".join(ch for ch in name if not unicodedata.combining(ch))
|
||||
name = re.sub(r"[^A-Z0-9]+", " ", name)
|
||||
return re.sub(r"\s+", " ", name).strip()
|
||||
55
backend/api/v1/modules/a76/layouts_csv/common/responses.py
Normal file
55
backend/api/v1/modules/a76/layouts_csv/common/responses.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Helpers para construir respuestas de scan y commit (formato unificado).
|
||||
"""
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
def scan_result(
|
||||
job_id: str,
|
||||
processed_rows: int,
|
||||
error_count: int,
|
||||
errors_detail: List[Dict[str, Any]],
|
||||
total_rows_in_file: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Respuesta de scan_file (waiting_confirmation).
|
||||
Si total_rows_in_file no se pasa, se usa processed_rows como total (comportamiento anterior).
|
||||
"""
|
||||
total = total_rows_in_file if total_rows_in_file is not None else processed_rows
|
||||
out: Dict[str, Any] = {
|
||||
"status": "waiting_confirmation",
|
||||
"job_id": job_id,
|
||||
"total_rows": total,
|
||||
"error_count": error_count,
|
||||
"valid_rows": processed_rows - error_count,
|
||||
"errors": errors_detail,
|
||||
}
|
||||
if message:
|
||||
out["message"] = message
|
||||
return out
|
||||
|
||||
|
||||
def commit_result(
|
||||
status: str,
|
||||
inserted: int,
|
||||
skipped_invalid: int,
|
||||
skipped_missing_fk: int,
|
||||
skipped_duplicate: int,
|
||||
skipped_details: List[Dict[str, Any]],
|
||||
message: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Respuesta de insert_valid_rows (finished / warning / failed)."""
|
||||
out = {
|
||||
"status": status,
|
||||
"inserted": inserted,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_missing_fk": skipped_missing_fk,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
if message:
|
||||
out["message"] = message
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
171
backend/api/v1/modules/a76/layouts_csv/common/storage.py
Normal file
171
backend/api/v1/modules/a76/layouts_csv/common/storage.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""
|
||||
Redis y rutas de archivos para imports CSV (compartido por módulos layouts_csv).
|
||||
Cada módulo usa un job_type (ej. "part", "cls", "bom") para prefijos y nombres de archivo.
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, Set, List
|
||||
|
||||
from core.paths import layout_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def upload_dir() -> str:
|
||||
"""Directorio temporal para CSV en el worker."""
|
||||
return layout_path("imports", "temp")
|
||||
|
||||
|
||||
def error_dir() -> str:
|
||||
"""Directorio de archivos JSONL de errores."""
|
||||
return layout_path("imports", "errors")
|
||||
|
||||
|
||||
def storage_keys(job_type: str, job_id: str) -> tuple:
|
||||
"""Prefijos Redis para un job_type y job_id. Devuelve (file_key, meta_key, error_lines_key)."""
|
||||
# Facturas usa prefijo "import_" sin tipo (compatibilidad con rutas existentes)
|
||||
if job_type == "" or job_type == "invoice":
|
||||
prefix = "import_"
|
||||
else:
|
||||
prefix = f"{job_type}_import_"
|
||||
return (
|
||||
f"{prefix}file:{job_id}",
|
||||
f"{prefix}meta:{job_id}",
|
||||
f"{prefix}error_lines:{job_id}",
|
||||
)
|
||||
|
||||
|
||||
def file_path_for_job(job_type: str, job_id: str) -> str:
|
||||
"""Ruta local del archivo CSV para un job."""
|
||||
if job_type == "" or job_type == "invoice":
|
||||
return os.path.join(upload_dir(), f"{job_id}.csv")
|
||||
return os.path.join(upload_dir(), f"{job_type}_{job_id}.csv")
|
||||
|
||||
|
||||
def error_path_for_job(job_type: str, job_id: str) -> str:
|
||||
"""Ruta del archivo JSONL de errores para un job."""
|
||||
os.makedirs(error_dir(), exist_ok=True)
|
||||
if job_type == "" or job_type == "invoice":
|
||||
return os.path.join(error_dir(), f"{job_id}.jsonl")
|
||||
return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl")
|
||||
|
||||
|
||||
def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]:
|
||||
"""
|
||||
Descarga contenido del CSV desde Redis y lo escribe en disco.
|
||||
Devuelve la ruta del archivo o None si no hay datos o falla.
|
||||
"""
|
||||
file_key, _, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(file_key)
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e)
|
||||
return None
|
||||
path = file_path_for_job(job_type, job_id)
|
||||
os.makedirs(upload_dir(), exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(raw)
|
||||
return path
|
||||
|
||||
|
||||
def ensure_meta_from_redis(job_type: str, job_id: str, file_path: str, log_prefix: str = "") -> bool:
|
||||
"""Descarga meta desde Redis y la escribe en .meta.json. Devuelve True si hubo datos."""
|
||||
_, meta_key, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(meta_key)
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(data.decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode meta from Redis: %s", log_prefix or job_type, e)
|
||||
return False
|
||||
meta_path = file_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f)
|
||||
return True
|
||||
|
||||
|
||||
def store_error_lines(job_type: str, job_id: str, line_numbers: List[int]) -> None:
|
||||
"""Guarda la lista de números de línea con error en Redis."""
|
||||
_, _, error_key = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(error_key, json.dumps(line_numbers).encode("utf-8"), ex=IMPORT_REDIS_TTL)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to store error lines in Redis: %s", e)
|
||||
|
||||
|
||||
def get_error_lines(job_type: str, job_id: str, error_path: str) -> Set[int]:
|
||||
"""
|
||||
Obtiene el conjunto de líneas con error: primero desde Redis, si está vacío desde el JSONL.
|
||||
"""
|
||||
_, _, error_key = storage_keys(job_type, job_id)
|
||||
lines = set()
|
||||
try:
|
||||
r = _get_redis()
|
||||
raw = r.get(error_key)
|
||||
if raw:
|
||||
lines = set(json.loads(raw.decode("utf-8")))
|
||||
except Exception as e:
|
||||
logger.debug("Could not load error lines from Redis: %s", e)
|
||||
if not lines and os.path.exists(error_path):
|
||||
with open(error_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
err = json.loads(line)
|
||||
if "line" in err:
|
||||
lines.add(err["line"])
|
||||
except Exception:
|
||||
pass
|
||||
return lines
|
||||
|
||||
|
||||
def delete_import_from_redis(job_type: str, job_id: str) -> None:
|
||||
"""Borra claves Redis del import (file, meta, error_lines)."""
|
||||
file_key, meta_key, error_key = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(file_key, meta_key, error_key)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to delete import keys from Redis: %s", e)
|
||||
|
||||
|
||||
def cleanup_import_job(
|
||||
job_type: str,
|
||||
job_id: str,
|
||||
file_path: Optional[str] = None,
|
||||
error_path: Optional[str] = None,
|
||||
meta_path: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Elimina archivos locales y claves Redis del job."""
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove file %s: %s", file_path, e)
|
||||
if error_path and os.path.exists(error_path):
|
||||
try:
|
||||
os.remove(error_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove error file %s: %s", error_path, e)
|
||||
if meta_path and os.path.exists(meta_path):
|
||||
try:
|
||||
os.remove(meta_path)
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: failed to remove meta %s: %s", meta_path, e)
|
||||
delete_import_from_redis(job_type, job_id)
|
||||
@@ -0,0 +1 @@
|
||||
# layouts_csv.customs_brokers
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers, fk_loader
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de agentes aduanales (customs brokers).
|
||||
Paridad Clarion: VALIDACIONES_AGENTE_ADUANAL, tipo MEX/AME, patente obligatoria si MEX,
|
||||
país en catálogo, RFC/CURP máx, desfase.
|
||||
"""
|
||||
import re
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
BROKER_KEY_MAX = 5
|
||||
LICENSE_MAX = 4
|
||||
RFC_MAX = 30
|
||||
CURP_MAX = 19
|
||||
|
||||
|
||||
def check_required_broker_key(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col B: Clave de Agente Aduanal obligatoria, máx 5 caracteres."""
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
if not clave:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLAVE",
|
||||
"msg": f"Error: (Celda B{line_num}) La Clave de Agente Aduanal está vacía y no se pueden hacer las validaciones.",
|
||||
"solution": f"Capturar en la Celda B{line_num} una Clave de Agente Aduanal nueva o existente a la cual desee agregar, remplazar o actualizar campos",
|
||||
}
|
||||
if len(clave) > BROKER_KEY_MAX:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLAVE",
|
||||
"msg": f"Error: (Col. B) La Clave de Agente Aduanal: {clave} supera la longitud de caracteres.",
|
||||
"solution": f"Capturar en la columna B una Clave de Agente Aduanal de {BROKER_KEY_MAX} caracteres como máximo.",
|
||||
}
|
||||
if not re.match(r"^[a-zA-Z0-9]+$", clave):
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLAVE",
|
||||
"msg": "Error: (Col. B) La Clave de Agente Aduanal solo puede contener letras y números.",
|
||||
"solution": "Capturar en la columna B una Clave alfanumérica.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_license(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col C: Patente/Licencia opcional; si tiene valor, máx 4 dígitos."""
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if not licencia:
|
||||
return None
|
||||
if len(licencia) > LICENSE_MAX or not licencia.isdigit():
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "LICENCIA",
|
||||
"msg": f"Error: (Col. C) La Patente debe ser de hasta {LICENSE_MAX} dígitos numéricos.",
|
||||
"solution": f"Capturar en la columna C una Patente de hasta {LICENSE_MAX} dígitos.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Si COL_EXTRA (Col O) tiene valor → advertencia de desfase."""
|
||||
val = (row.get("COL_EXTRA") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "COL_EXTRA",
|
||||
"msg": "Advertencia: Podría existir un desfase en esta línea.",
|
||||
"solution": "Revisar esta línea del archivo CSV y verificar cada campo esté en la posición correcta.",
|
||||
}
|
||||
|
||||
|
||||
def parse_tipo_agente_aduanal(val: Optional[str]) -> Optional[str]:
|
||||
"""
|
||||
Parsea TIPO (Col A): letra (M/A) o palabra (MEX/Mexicano, AME/Americano).
|
||||
Devuelve 'M' o 'A'; si no reconoce, None.
|
||||
"""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
s = str(val).strip()
|
||||
v = s.upper()
|
||||
if len(v) == 1:
|
||||
if v == "M":
|
||||
return "M"
|
||||
if v == "A":
|
||||
return "A"
|
||||
v_lower = s.lower()
|
||||
if v_lower in ("mex", "mexicano"):
|
||||
return "M"
|
||||
if v_lower in ("ame", "americano"):
|
||||
return "A"
|
||||
return None
|
||||
|
||||
|
||||
def check_tipo_mex_ame(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Col A: TIPO debe ser MEX o AME (flexible: M/MEX/Mexicano, A/AME/Americano).
|
||||
Si TIPO=M entonces PAIS (Col J) debe ser MEX; si TIPO=A entonces PAIS no debe ser MEX.
|
||||
"""
|
||||
tipo_raw = (row.get("TIPO") or "").strip()
|
||||
if not tipo_raw:
|
||||
return None
|
||||
tipo = parse_tipo_agente_aduanal(tipo_raw)
|
||||
if tipo is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO",
|
||||
"msg": f"Error: (Col. A) El tipo de agente aduanal: {tipo_raw} es incorrecto.",
|
||||
"solution": "Capturar en columna A el Tipo de agente aduanal correcto, MEX para Mexicano y AME para Americano.",
|
||||
}
|
||||
pais = (row.get("PAIS") or "").strip().upper()
|
||||
if not pais:
|
||||
return None
|
||||
if tipo == "M" and pais != "MEX":
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PAIS",
|
||||
"msg": f"Error: (Col. J) El tipo de cliente: {tipo_raw} tiene el país {pais}, es incorrecto.",
|
||||
"solution": "Capturar en la columna J el país con clave MEX, ya que es un Agente Aduanal Mexicano",
|
||||
}
|
||||
if tipo == "A" and pais == "MEX":
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PAIS",
|
||||
"msg": f"Error: (Col. J) El tipo de cliente: {tipo_raw} tiene el país {pais}, es incorrecto.",
|
||||
"solution": "Capturar en la columna J un país con clave diferente de MEX, ya que es un Agente Aduanal Americano.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_patente_obligatoria_si_mex(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col C: Patente obligatoria si TIPO es MEX."""
|
||||
tipo = parse_tipo_agente_aduanal((row.get("TIPO") or "").strip())
|
||||
if tipo != "M":
|
||||
return None
|
||||
licencia = (row.get("LICENCIA") or "").strip()
|
||||
if not licencia:
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "LICENCIA",
|
||||
"msg": f"Error: (Col. C) La Patente para el Agente Aduanal con Clave: {clave} no está capturada.",
|
||||
"solution": "Capturar en la columna C una Patente de Agente Aduanal.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_rfc_max(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col E: RFC opcional; si tiene valor, máx 30 caracteres."""
|
||||
val = (row.get("RFC") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > RFC_MAX:
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "RFC",
|
||||
"msg": f"Error: (Col. E) El RFC de Agente Aduanal: {clave} supera la longitud de caracteres.",
|
||||
"solution": f"Capturar en la columna E un RFC de {RFC_MAX} caracteres como máximo.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_curp_max(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col N: CURP/PERSONAL_ID opcional; si tiene valor, máx 19 caracteres."""
|
||||
val = (row.get("PERSONAL_ID") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > CURP_MAX:
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PERSONAL_ID",
|
||||
"msg": f"Error: (Col. N) El CURP de Agente Aduanal: {clave} supera la longitud de caracteres.",
|
||||
"solution": f"Capturar en la columna N un CURP de Agente Aduanal de {CURP_MAX} caracteres como máximo.",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_pais_catalogo(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_country_m3: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col J: Si PAIS tiene valor, debe existir en catálogo (clave M3)."""
|
||||
val = (row.get("PAIS") or "").strip()
|
||||
if not val or valid_country_m3 is None:
|
||||
return None
|
||||
if val.upper() in valid_country_m3:
|
||||
return None
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "PAIS",
|
||||
"msg": f"Error: (Col. J) El Pais del Agente Aduanal: {clave} no está en el Catálogo de Países.",
|
||||
"solution": "Capturar en la columna J un País válido.",
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación de import CSV de agentes aduanales.
|
||||
Clarion: catálogo de países (GPaises / m3_key).
|
||||
"""
|
||||
from typing import Set
|
||||
import logging
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_customs_brokers_fk_sets() -> Set[str]:
|
||||
"""
|
||||
Carga valid_country_m3 (códigos país m3_key) para validar Col J PAIS.
|
||||
"""
|
||||
valid_country_m3: Set[str] = set()
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.public.reference_data.countries.models import Country
|
||||
for row in session.query(Country.m3_key).all():
|
||||
if row[0]:
|
||||
valid_country_m3.add((row[0] or "").strip().upper())
|
||||
except Exception as e:
|
||||
logger.warning("Customs brokers import: could not load country m3 keys: %s", e)
|
||||
return valid_country_m3
|
||||
@@ -0,0 +1,77 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para CustomsBroker.
|
||||
Clarion: TIPO normalizado a M/A con parse_tipo_agente_aduanal; CURP/personal_id máx 19.
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import parse_tipo_agente_aduanal, CURP_MAX
|
||||
|
||||
MAX_LEN = {
|
||||
"broker_key": 5,
|
||||
"type": 9,
|
||||
"name": 80,
|
||||
"address": 1500,
|
||||
"postal_code": 15,
|
||||
"city": 30,
|
||||
"state": 30,
|
||||
"phone": 30,
|
||||
"fax": 30,
|
||||
"email": 100,
|
||||
"country": 3,
|
||||
"tax_id": 30,
|
||||
"personal_id": 20,
|
||||
"position": 30,
|
||||
"license": 4,
|
||||
"company": 200,
|
||||
"contact": 80,
|
||||
}
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _license_value(row_norm: Dict[str, Any]) -> Optional[str]:
|
||||
lic = (row_norm.get("LICENCIA") or "").strip()
|
||||
if not lic or not lic.isdigit():
|
||||
return None
|
||||
return lic[:4]
|
||||
|
||||
|
||||
def row_to_customs_broker_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for CustomsBroker model (create or update)."""
|
||||
clave = _str_or_none(row_norm.get("CLAVE"), MAX_LEN["broker_key"])
|
||||
if not clave:
|
||||
return {}
|
||||
tipo_raw = (row_norm.get("TIPO") or "").strip()
|
||||
tipo_normalized = parse_tipo_agente_aduanal(tipo_raw) if tipo_raw else None
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"broker_key": clave,
|
||||
"type": tipo_normalized or _str_or_none(row_norm.get("TIPO"), MAX_LEN["type"]),
|
||||
"name": _str_or_none(row_norm.get("NOMBRE"), MAX_LEN["name"]),
|
||||
"address": _str_or_none(row_norm.get("DIRECCION"), MAX_LEN["address"]),
|
||||
"postal_code": _str_or_none(row_norm.get("CODIGO POSTAL"), MAX_LEN["postal_code"]),
|
||||
"city": _str_or_none(row_norm.get("CIUDAD"), MAX_LEN["city"]),
|
||||
"state": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["state"]),
|
||||
"phone": _str_or_none(row_norm.get("TELEFONO"), MAX_LEN["phone"]),
|
||||
"fax": _str_or_none(row_norm.get("FAX"), MAX_LEN["fax"]),
|
||||
"email": _str_or_none(row_norm.get("EMAIL"), MAX_LEN["email"]),
|
||||
"country": _str_or_none(row_norm.get("PAIS"), MAX_LEN["country"]),
|
||||
"tax_id": _str_or_none(row_norm.get("RFC"), MAX_LEN["tax_id"]),
|
||||
"personal_id": _str_or_none(row_norm.get("PERSONAL_ID"), CURP_MAX),
|
||||
"position": _str_or_none(row_norm.get("POSICION"), MAX_LEN["position"]),
|
||||
"license": _license_value(row_norm),
|
||||
"company": _str_or_none(row_norm.get("EMPRESA"), MAX_LEN["company"]),
|
||||
"contact": _str_or_none(row_norm.get("CONTACTO"), MAX_LEN["contact"]),
|
||||
}
|
||||
@@ -14,6 +14,7 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -81,7 +82,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
273
backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py
Normal file
273
backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py
Normal file
@@ -0,0 +1,273 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Agentes Aduanales.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List, Set
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import (
|
||||
row_from_template,
|
||||
CUSTOMS_BROKERS_FIELDNAMES_ORDER,
|
||||
CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL,
|
||||
)
|
||||
from .validators import validate_row_customs_broker
|
||||
from .common.mappers import row_to_customs_broker_data
|
||||
from .common.fk_loader import load_customs_brokers_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "cb"
|
||||
|
||||
# Para routes.py
|
||||
CB_IMPORT_FILE_PREFIX = "cb_import_file:"
|
||||
CB_IMPORT_META_PREFIX = "cb_import_meta:"
|
||||
CB_IMPORT_ERROR_LINES_PREFIX = "cb_import_error_lines:"
|
||||
CB_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
actualizar = meta.get("actualizar", False)
|
||||
existing_broker_keys: Set[str] = set()
|
||||
if actualizar:
|
||||
try:
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
with CoreSessionLocal() as session:
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if (b.broker_key or "").strip():
|
||||
existing_broker_keys.add((b.broker_key or "").strip())
|
||||
except Exception as e:
|
||||
logger.warning("CB import: could not load existing broker_keys for ACT: %s", e)
|
||||
|
||||
valid_country_m3 = load_customs_brokers_fk_sets()
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(
|
||||
file_path,
|
||||
fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER,
|
||||
headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL,
|
||||
):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_customs_broker(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
existing_broker_keys=existing_broker_keys,
|
||||
valid_country_m3=valid_country_m3,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("CB import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("CB import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "CB import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "CB import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
actualizar = meta.get("actualizar", False)
|
||||
valid_country_m3 = load_customs_brokers_fk_sets()
|
||||
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
|
||||
inserted_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_key: Dict[str, CustomsBroker] = {}
|
||||
for b in (
|
||||
session.query(CustomsBroker)
|
||||
.filter(
|
||||
CustomsBroker.tenant_id == tenant_id,
|
||||
CustomsBroker.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
existing_by_key[b.broker_key] = b
|
||||
|
||||
existing_broker_keys = set(existing_by_key.keys())
|
||||
|
||||
for i, row in common_csv_reader.iter_csv_rows(
|
||||
file_path,
|
||||
fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER,
|
||||
headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL,
|
||||
):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_customs_broker(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
existing_broker_keys=existing_broker_keys,
|
||||
valid_country_m3=valid_country_m3,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_customs_broker_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("broker_key"):
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
clave = data["broker_key"]
|
||||
existing = existing_by_key.get(clave)
|
||||
if existing:
|
||||
for k, v in data.items():
|
||||
if k not in ("tenant_id", "company_id", "broker_key"):
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
else:
|
||||
new_broker = CustomsBroker(**data)
|
||||
session.add(new_broker)
|
||||
existing_by_key[clave] = new_broker
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("CB import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("CB import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if inserted_count == 0 and skipped_invalid > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados.",
|
||||
}
|
||||
if inserted_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": 0,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("CB import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
@@ -1,5 +1,8 @@
|
||||
"""
|
||||
Configuración de plantilla CSV para Agentes Aduanales (EstructuraCatAgenteAduanal.xls).
|
||||
Layout Clarion: A=TIPO, B=CLAVE AADUANAL, C=PATENTE, D=NOMBRE, E=RFC, F=DIRECCION, G=CODIGO POSTAL,
|
||||
H=CIUDAD, I=ESTADO, J=PAIS, K=TELEFONO, L=NUMERO FAX, M=CORREO ELECTRONICO, N=CURP, O=COL_EXTRA (desfase).
|
||||
Encabezado ejemplo: "TIPO(MEX=Mexicano,AME=AMERICANO)",CLAVE AADUANAL,PATENTE,NOMBRE,RFC,...
|
||||
Solo se leen columnas definidas aquí; el resto se ignora.
|
||||
"""
|
||||
|
||||
@@ -7,26 +10,66 @@ from typing import Dict, List, Any, Optional
|
||||
|
||||
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
"customs_brokers": [
|
||||
{"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "ID"]},
|
||||
{"canonical": "TIPO"},
|
||||
{"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]},
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO"]},
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
{"canonical": "CIUDAD"},
|
||||
{"canonical": "ESTADO"},
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]},
|
||||
{"canonical": "FAX"},
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL"]},
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]},
|
||||
{"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL"]},
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
# Col A - TIPO (MEX/Mexicano, AME/Americano)
|
||||
{"canonical": "TIPO", "aliases": ["TIPO(MEX=Mexicano,AME=AMERICANO)"]},
|
||||
# Col B - Clave agente aduanal (máx 5)
|
||||
{"canonical": "CLAVE", "aliases": ["BROKER_KEY", "CLAVE AGENTE", "CLAVE AADUANAL", "CLAVE ADUANAL", "ID"]},
|
||||
# Col C - Patente / Licencia (obligatoria si TIPO=MEX)
|
||||
{"canonical": "LICENCIA", "aliases": ["PATENTE", "LICENSE"]},
|
||||
# Col D - Nombre
|
||||
{"canonical": "NOMBRE", "aliases": ["NOMBRE COMPLETO", "RAZON SOCIAL"]},
|
||||
# Col E - RFC (máx 30)
|
||||
{"canonical": "RFC", "aliases": ["TAX_ID", "TAXID"]},
|
||||
# Col F - Dirección
|
||||
{"canonical": "DIRECCION", "aliases": ["DOMICILIO"]},
|
||||
# Col G - Código postal
|
||||
{"canonical": "CODIGO POSTAL", "aliases": ["CODIGOPOSTAL", "CP", "C.P."]},
|
||||
# Col H - Ciudad
|
||||
{"canonical": "CIUDAD"},
|
||||
# Col I - Estado
|
||||
{"canonical": "ESTADO"},
|
||||
# Col J - País (clave M3)
|
||||
{"canonical": "PAIS", "aliases": ["COUNTRY"]},
|
||||
# Col K - Teléfono
|
||||
{"canonical": "TELEFONO", "aliases": ["PHONE", "TEL"]},
|
||||
# Col L - Fax
|
||||
{"canonical": "FAX", "aliases": ["NUMERO FAX"]},
|
||||
# Col M - Email
|
||||
{"canonical": "EMAIL", "aliases": ["CORREO", "E-MAIL", "CORREO ELECTRONICO"]},
|
||||
# Col N - CURP / Personal ID (máx 19)
|
||||
{"canonical": "PERSONAL_ID", "aliases": ["PERSONALID", "ID PERSONAL", "CURP"]},
|
||||
# Col O - Desfase (si tiene valor → advertencia)
|
||||
{"canonical": "COL_EXTRA", "aliases": ["COLUMNA EXTRA", "DESFASE"]},
|
||||
# Otros opcionales (no en encabezado oficial)
|
||||
{"canonical": "POSICION", "aliases": ["CARGO", "PUESTO"]},
|
||||
{"canonical": "EMPRESA", "aliases": ["COMPANY"]},
|
||||
{"canonical": "CONTACTO", "aliases": ["CONTACT"]},
|
||||
],
|
||||
}
|
||||
|
||||
# Orden oficial de columnas (para CSV sin encabezado o detección)
|
||||
CUSTOMS_BROKERS_FIELDNAMES_ORDER = [
|
||||
"TIPO",
|
||||
"CLAVE",
|
||||
"LICENCIA",
|
||||
"NOMBRE",
|
||||
"RFC",
|
||||
"DIRECCION",
|
||||
"CODIGO POSTAL",
|
||||
"CIUDAD",
|
||||
"ESTADO",
|
||||
"PAIS",
|
||||
"TELEFONO",
|
||||
"FAX",
|
||||
"EMAIL",
|
||||
"PERSONAL_ID",
|
||||
]
|
||||
|
||||
# Si la primera celda de la primera fila está en este set, se trata como CSV sin encabezado
|
||||
CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL = frozenset(
|
||||
{"MEX", "AME", "M", "A", "MEXICANO", "AMERICANO"}
|
||||
)
|
||||
|
||||
|
||||
def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
"""normalized_header -> canonical_name para plantilla customs_brokers."""
|
||||
@@ -39,6 +82,8 @@ def build_normalized_lookup(normalize_header_fn) -> Dict[str, str]:
|
||||
lookup[normalize_header_fn(canonical)] = canonical
|
||||
for alias in item.get("aliases") or []:
|
||||
lookup[normalize_header_fn(alias)] = canonical
|
||||
for idx, name in enumerate(CUSTOMS_BROKERS_FIELDNAMES_ORDER):
|
||||
lookup[normalize_header_fn(f"_COL_{idx}_")] = name
|
||||
return lookup
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_customs_broker
|
||||
|
||||
__all__ = ["validate_row_customs_broker"]
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de agentes aduanales.
|
||||
Paridad Clarion: VALIDA_TODA_AGENTE_ADUANAL, VALIDA_PARCIAL_AGENTE_ADUANAL, VALIDACIONES_AGENTE_ADUANAL.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_broker_key,
|
||||
check_optional_license,
|
||||
check_desfase,
|
||||
check_tipo_mex_ame,
|
||||
check_patente_obligatoria_si_mex,
|
||||
check_rfc_max,
|
||||
check_curp_max,
|
||||
check_pais_catalogo,
|
||||
)
|
||||
|
||||
MSG_CLAVE_NO_EXISTE = "Error: (Col. B) Clave de A. Aduanal No Existe en el Catalogo."
|
||||
MSG_CLAVE_NO_EXISTE_SOLUCION = "La clave debe existir en el catálogo cuando el modo es Actualizar."
|
||||
MSG_CAMPOS_OBLIGATORIOS = "Existen campos vacíos que son obligatorios, es la {campos}."
|
||||
MSG_CAMPOS_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta."
|
||||
|
||||
|
||||
def validaciones_agente_aduanal(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_country_m3: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
VALIDACIONES_AGENTE_ADUANAL: reglas de dominio compartidas.
|
||||
Tipo MEX/AME + coherencia País, patente si MEX, RFC/CURP máx, país en catálogo, licencia formato.
|
||||
"""
|
||||
err = check_tipo_mex_ame(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_patente_obligatoria_si_mex(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_license(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_rfc_max(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_curp_max(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_pais_catalogo(row, line_num, valid_country_m3)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def valida_toda_agente_aduanal(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_country_m3: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
VALIDA_TODA: TIPO (Col A) y NOMBRE (Col D) obligatorios; luego VALIDACIONES_AGENTE_ADUANAL.
|
||||
"""
|
||||
campos_oblig = []
|
||||
if not (row.get("TIPO") or "").strip():
|
||||
campos_oblig.append("(Col.A) Tipo")
|
||||
if not (row.get("NOMBRE") or "").strip():
|
||||
campos_oblig.append("(Col.D) Nombre")
|
||||
if campos_oblig:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TIPO" if not (row.get("TIPO") or "").strip() else "NOMBRE",
|
||||
"msg": MSG_CAMPOS_OBLIGATORIOS.format(campos=", ".join(campos_oblig)),
|
||||
"solution": MSG_CAMPOS_OBLIGATORIOS_SOLUCION,
|
||||
}
|
||||
return validaciones_agente_aduanal(row, line_num, valid_country_m3)
|
||||
|
||||
|
||||
def valida_parcial_agente_aduanal(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_country_m3: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""VALIDA_PARCIAL: solo VALIDACIONES_AGENTE_ADUANAL (no exige TIPO ni NOMBRE)."""
|
||||
return validaciones_agente_aduanal(row, line_num, valid_country_m3)
|
||||
|
||||
|
||||
def validate_row_customs_broker(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
actualizar: bool = False,
|
||||
existing_broker_keys: Optional[Set[str]] = None,
|
||||
valid_country_m3: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de agentes aduanales.
|
||||
1. Desfase (COL_EXTRA) primero.
|
||||
2. Clave vacía → error.
|
||||
3. Si actualizar y clave no existe en existing_broker_keys → error.
|
||||
4. Si actualizar y clave existe → valida_parcial_agente_aduanal.
|
||||
5. Si no actualizar o clave no existe → valida_toda_agente_aduanal.
|
||||
"""
|
||||
err = check_desfase(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_required_broker_key(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
clave = (row.get("CLAVE") or "").strip()
|
||||
existing = existing_broker_keys or set()
|
||||
use_partial = actualizar and clave in existing
|
||||
|
||||
if actualizar and clave and clave not in existing:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "CLAVE",
|
||||
"msg": MSG_CLAVE_NO_EXISTE,
|
||||
"solution": MSG_CLAVE_NO_EXISTE_SOLUCION,
|
||||
}
|
||||
|
||||
if use_partial:
|
||||
err = valida_parcial_agente_aduanal(row, line_num, valid_country_m3)
|
||||
else:
|
||||
err = valida_toda_agente_aduanal(row, line_num, valid_country_m3)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila agente aduanal.
|
||||
"""
|
||||
from .common import validate_row_customs_broker
|
||||
|
||||
__all__ = ["validate_row_customs_broker"]
|
||||
@@ -0,0 +1,4 @@
|
||||
# common validators, mappers, fk_loader for drivers CSV import
|
||||
from .fk_loader import load_drivers_fk_sets
|
||||
|
||||
__all__ = ["load_drivers_fk_sets"]
|
||||
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Helpers reutilizables para validación de filas CSV (conductores).
|
||||
Paridad Clarion: VALIDACIONES_CONDUCTOR, obligatorios A/C, catálogos transportista/países, Sexo M/F, Si/No, tipo identificación, desfase.
|
||||
"""
|
||||
import re
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
|
||||
MAX_LEN = {
|
||||
"transporter_key": 5,
|
||||
"driver_name": 80,
|
||||
"license_number": 29,
|
||||
"express_line_id": 17,
|
||||
"ace_id": 20,
|
||||
"gender": 1,
|
||||
"birth_country": 3,
|
||||
"hazardous_material_auth": 2,
|
||||
"hazardous_material_state": 30,
|
||||
"first_name": 20,
|
||||
"last_name": 20,
|
||||
"id_key1": 40,
|
||||
"id_number1": 20,
|
||||
"id_state1": 30,
|
||||
"id_country1": 3,
|
||||
"id_key2": 40,
|
||||
"id_number2": 20,
|
||||
"id_state2": 30,
|
||||
"id_country2": 3,
|
||||
}
|
||||
|
||||
# Clarion: Col H Sexo M o F
|
||||
SEXO_VALIDOS = {"M", "F"}
|
||||
|
||||
# Clarion: Col J Si o No (comparar en mayúsculas)
|
||||
MATERIAL_PELIGROSO_VALIDOS = {"SI", "NO"}
|
||||
|
||||
# Clarion: Col N y Col R tipo identificación
|
||||
FORMA_IDENTIFICACION_CLAVES = frozenset(
|
||||
{"ACW", "ALR", "BCP", "BCN", "CDN", "CON", "OTD", "REP", "RTP", "5J", "5K", "30"}
|
||||
)
|
||||
|
||||
# Mapeo clave CSV → valor guardado en BD (Clarion QueCSV:ColumnaN/R)
|
||||
FORMA_IDENTIFICACION_MAP = {
|
||||
"ACW": "ACW-Pasaporte",
|
||||
"ALR": "ALR-Residencia",
|
||||
"BCP": "BCP-Permiso Cruce",
|
||||
"BCN": "BCN-Acta Nacimiento",
|
||||
"CDN": "CDN-Ciudadania",
|
||||
"CON": "CON-CertificadoNaturalizacion",
|
||||
"OTD": "OTD-Otra Identificación",
|
||||
"REP": "REP-Permiso Reentrada",
|
||||
"RTP": "RTP-Permiso de Viaje",
|
||||
"5J": "5J - Licencia",
|
||||
"5K": "5K -Licencia",
|
||||
"30": "30 -Visa de EU",
|
||||
}
|
||||
|
||||
|
||||
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
|
||||
|
||||
def check_max_length(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
max_len: int,
|
||||
line_num: int,
|
||||
required: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
if required:
|
||||
return {"line": line_num, "col": col, "msg": "Requerido"}
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Maximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_int(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d+$", s):
|
||||
return int(s)
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def check_int_positive(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
v = parse_int(row.get(col))
|
||||
if v is None:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser numerico"}
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": col, "msg": "Debe ser mayor a 0"}
|
||||
return None
|
||||
|
||||
|
||||
def parse_birth_date(val: Any) -> Optional[int]:
|
||||
if val is None or (isinstance(val, str) and not val.strip()):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if re.match(r"^\d{8}$", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
if re.match(r"^\d+\.0+$", s):
|
||||
try:
|
||||
return int(float(s))
|
||||
except ValueError:
|
||||
return None
|
||||
for sep in ["/", "-", "."]:
|
||||
if sep in s:
|
||||
parts = s.split(sep)
|
||||
if len(parts) == 3:
|
||||
try:
|
||||
a, b, c = [p.strip() for p in parts]
|
||||
if len(c) == 4 and len(a) <= 2 and len(b) <= 2:
|
||||
return int(c) * 10000 + int(b) * 100 + int(a)
|
||||
if len(a) == 4 and len(b) <= 2 and len(c) <= 2:
|
||||
return int(a) * 10000 + int(b) * 100 + int(c)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_birth_date(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = row.get(col)
|
||||
if val is None or not str(val).strip():
|
||||
return None
|
||||
if parse_birth_date(val) is None:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": "Formato de fecha invalido (use YYYYMMDD o DD/MM/YYYY)",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def check_transportista_catalog(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_transporter_keys: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col A: Si TRANSPORTISTA no vacío, debe existir en catálogo (GTransportista)."""
|
||||
val = (row.get("TRANSPORTISTA") or "").strip()
|
||||
if not val or valid_transporter_keys is None:
|
||||
return None
|
||||
if val.upper() in valid_transporter_keys:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TRANSPORTISTA",
|
||||
"msg": f"Error: (Col. A) La Clave de Transportista: {val} es incorrecto.",
|
||||
"solution": "Capturar en columna A un Transportista existente en catalogo.",
|
||||
}
|
||||
|
||||
|
||||
def check_sexo_m_f(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col H: Si SEXO no vacío, debe ser M o F (Clarion)."""
|
||||
val = (row.get("SEXO") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if val.upper() in SEXO_VALIDOS:
|
||||
return None
|
||||
conductor = (row.get("CLAVE CONDUCTOR") or "").strip() or "(Conductor)"
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "SEXO",
|
||||
"msg": f"Error: (Col. H) El Sexo: {val} del Conductor: {conductor} es incorrecto.",
|
||||
"solution": "Capturar en columna H el sexo correcto (M o F).",
|
||||
}
|
||||
|
||||
|
||||
def check_pais_catalog_drivers(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
valid_country_ame: Optional[Set[str]] = None,
|
||||
col_letter: str = "",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Si col (PAIS NACIMIENTO, PAIS, PAIS 2) no vacío, debe ser clave americana en catálogo (GPaises.Pais_Ame)."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val or valid_country_ame is None:
|
||||
return None
|
||||
val_upper = val.upper()
|
||||
if len(val) > 3:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": f"Error: ({col_letter}) El Pais: {val} es incorrecto.",
|
||||
"solution": "Capturar un Pais en clave americana (US, MX, etc.).",
|
||||
}
|
||||
if val_upper in valid_country_ame:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": f"Error: ({col_letter}) El Pais: {val} es incorrecto.",
|
||||
"solution": "Capturar en columna un Pais en clave americana.",
|
||||
}
|
||||
|
||||
|
||||
def check_material_peligroso_si_no(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Col J: TRANSPORTA MAT. PELIGROSO? debe ser Si o No (Clarion; comparar en mayúsculas)."""
|
||||
val = (row.get("TRANSPORTA MAT. PELIGROSO?") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if val.upper() in MATERIAL_PELIGROSO_VALIDOS:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "TRANSPORTA MAT. PELIGROSO?",
|
||||
"msg": "Error: (Col. J) La Autorizacion para el Manejo de Material Peligroso es incorrecta.",
|
||||
"solution": "Capturar en columna J Si o No la autorizacion.",
|
||||
}
|
||||
|
||||
|
||||
def check_tipo_identificacion(
|
||||
row: Dict[str, Any],
|
||||
col: str,
|
||||
line_num: int,
|
||||
col_letter: str = "",
|
||||
primera_o_segunda: str = "Primera",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Col N o R: Si FORMA IDENTIFICACION no vacío, debe ser clave válida (ACW, ALR, ...)."""
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
val_upper = val.upper()
|
||||
if val_upper in FORMA_IDENTIFICACION_CLAVES:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": col,
|
||||
"msg": f"Error: ({col_letter}) El Tipo de Identificacion: {val} de la {primera_o_segunda} Identificacion es incorrecto.",
|
||||
"solution": f"Capturar en columna {col_letter} una clave de Identificacion valida (ACW, ALR, BCP, BCN, CDN, CON, OTD, REP, RTP, 5J, 5K, 30).",
|
||||
}
|
||||
|
||||
|
||||
def check_desfase_drivers(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Si COL_EXTRA (Col V) tiene valor -> advertencia de desfase (Clarion, no bloqueante)."""
|
||||
val = (row.get("COL_EXTRA") or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "COL_EXTRA",
|
||||
"msg": "Advertencia: Podria existir un desfase en esta linea.",
|
||||
"solution": "Revisar esta linea del archivo CSV y verificar cada campo este en la posicion correcta.",
|
||||
"warning": True,
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
Carga de conjuntos FK para validación de import CSV de conductores.
|
||||
Clarion: GTransportista (ClaveTrans), GPaises (Pais_Ame).
|
||||
"""
|
||||
from typing import Set, Tuple, Optional, Dict
|
||||
import logging
|
||||
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_drivers_fk_sets(
|
||||
tenant_id: Optional[int] = None,
|
||||
company_id: Optional[int] = None,
|
||||
) -> Tuple[Set[str], Set[str], Dict[str, str]]:
|
||||
"""
|
||||
Carga conjuntos para validación CSV de conductores (paridad Clarion).
|
||||
Devuelve:
|
||||
- valid_transporter_keys: todas las claves de transportistas en mayúsculas (a76.transporter)
|
||||
- valid_country_ame: claves americana de países (GPaises.Pais_Ame / Country.ame_key), mayúsculas
|
||||
- transporter_key_actual: dict clave_upper -> clave real en BD (para insert con mismo caso que en transporter)
|
||||
"""
|
||||
valid_transporter_keys: Set[str] = set()
|
||||
valid_country_ame: Set[str] = set()
|
||||
transporter_key_actual: Dict[str, str] = {}
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
from api.v1.modules.a76.transportation.transporters.models import Transporter
|
||||
from api.v1.modules.public.reference_data.countries.models import Country
|
||||
|
||||
# Solo transportistas del tenant/company del upload (paridad con Driver.tenant_id/company_id)
|
||||
q = session.query(Transporter.transporter_key).filter(
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
)
|
||||
for row in q.all():
|
||||
if row[0]:
|
||||
raw = (row[0] or "").strip()
|
||||
upper = raw.upper()
|
||||
valid_transporter_keys.add(upper)
|
||||
transporter_key_actual[upper] = raw
|
||||
|
||||
for row in session.query(Country.ame_key).all():
|
||||
if row[0]:
|
||||
valid_country_ame.add((row[0] or "").strip().upper())
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: could not load FK sets: %s", e)
|
||||
|
||||
logger.info(
|
||||
"Drivers import FK: %d transportistas (claves: %s), %d paises",
|
||||
len(valid_transporter_keys),
|
||||
sorted(valid_transporter_keys)[:20] if len(valid_transporter_keys) <= 20 else sorted(valid_transporter_keys)[:10] + ["..."],
|
||||
len(valid_country_ame),
|
||||
)
|
||||
return (valid_transporter_keys, valid_country_ame, transporter_key_actual)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para Driver (conductores).
|
||||
Clarion: FORMA IDENTIFICACION 1/2 se guardan expandidas (ACW → ACW-Pasaporte, etc.).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
MAX_LEN,
|
||||
parse_int,
|
||||
parse_birth_date,
|
||||
FORMA_IDENTIFICACION_MAP,
|
||||
)
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def _forma_identificacion_or_raw(val: Any, max_len: int) -> Optional[str]:
|
||||
"""Si el valor es una clave Clarion (ACW, ALR, ...), devuelve el valor expandido; si no, el valor truncado."""
|
||||
s = _str_or_none(val, max_len)
|
||||
if not s:
|
||||
return None
|
||||
expanded = FORMA_IDENTIFICACION_MAP.get(s.upper())
|
||||
if expanded:
|
||||
return expanded[:max_len] if len(expanded) > max_len else expanded
|
||||
return s
|
||||
|
||||
|
||||
def row_to_driver_data(
|
||||
row_norm: Dict[str, Any],
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""Mapea una fila normalizada del CSV a un diccionario para DriverCreateDTO."""
|
||||
transporter_key = _str_or_none(row_norm.get("TRANSPORTISTA"), MAX_LEN["transporter_key"])
|
||||
line = parse_int(row_norm.get("LINEA"))
|
||||
if not transporter_key or line is None:
|
||||
return {}
|
||||
return {
|
||||
"transporter_key": transporter_key,
|
||||
"line": line,
|
||||
"driver_name": _str_or_none(row_norm.get("CLAVE CONDUCTOR"), MAX_LEN["driver_name"]),
|
||||
"license_number": _str_or_none(row_norm.get("LICENCIA"), MAX_LEN["license_number"]),
|
||||
"express_line_id": _str_or_none(row_norm.get("PERMISO LINEA EXPRESS"), MAX_LEN["express_line_id"]),
|
||||
"ace_id": _str_or_none(row_norm.get("IDENTIFICACION ACE"), MAX_LEN["ace_id"]),
|
||||
"birth_date": parse_birth_date(row_norm.get("FECHA NACIMIENTO")),
|
||||
"gender": _str_or_none(row_norm.get("SEXO"), MAX_LEN["gender"]),
|
||||
"birth_country": _str_or_none(row_norm.get("PAIS NACIMIENTO"), MAX_LEN["birth_country"]),
|
||||
"hazardous_material_auth": _str_or_none(
|
||||
row_norm.get("TRANSPORTA MAT. PELIGROSO?"), MAX_LEN["hazardous_material_auth"]
|
||||
),
|
||||
"hazardous_material_state": _str_or_none(
|
||||
row_norm.get("PERMISO MAT. PELIGROSO"), MAX_LEN["hazardous_material_state"]
|
||||
),
|
||||
"first_name": _str_or_none(row_norm.get("NOMBRE(S)"), MAX_LEN["first_name"]),
|
||||
"last_name": _str_or_none(row_norm.get("APELLIDO PATERNO"), MAX_LEN["last_name"]),
|
||||
"id_key1": _forma_identificacion_or_raw(
|
||||
row_norm.get("FORMA IDENTIFICACION 1"), MAX_LEN["id_key1"]
|
||||
),
|
||||
"id_number1": _str_or_none(row_norm.get("NUM. IDENTIFICACION 1"), MAX_LEN["id_number1"]),
|
||||
"id_state1": _str_or_none(row_norm.get("ESTADO"), MAX_LEN["id_state1"]),
|
||||
"id_country1": _str_or_none(row_norm.get("PAIS"), MAX_LEN["id_country1"]),
|
||||
"id_key2": _forma_identificacion_or_raw(
|
||||
row_norm.get("FORMA IDENTIFICACION 2"), MAX_LEN["id_key2"]
|
||||
),
|
||||
"id_number2": _str_or_none(row_norm.get("NUM. IDENTIFICACION 2"), MAX_LEN["id_number2"]),
|
||||
"id_state2": _str_or_none(row_norm.get("ESTADO 2"), MAX_LEN["id_state2"]),
|
||||
"id_country2": _str_or_none(row_norm.get("PAIS 2"), MAX_LEN["id_country2"]),
|
||||
"company_id": company_id,
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
@@ -15,11 +15,11 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
DRV_IMPORT_FILE_PREFIX,
|
||||
@@ -81,7 +81,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
@@ -90,8 +90,6 @@ async def upload_import_file(
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: local file save failed: {e}")
|
||||
|
||||
scan_file.apply_async(args=[job_id], task_id=job_id)
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
497
backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py
Normal file
497
backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py
Normal file
@@ -0,0 +1,497 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Conductores.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses); CSV con headers duplicados (dedupe) y clave de estado en Redis.
|
||||
Paridad Clarion: actualizar, existing_driver_keys, valid_transporter_keys, valid_country_ame.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List, Set, Tuple
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
from sqlalchemy import func
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_driver, validate_row_driver_desfase
|
||||
from .common.mappers import row_to_driver_data
|
||||
from .common.fk_loader import load_drivers_fk_sets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "drv"
|
||||
|
||||
# Para routes.py
|
||||
DRV_IMPORT_FILE_PREFIX = "drv_import_file:"
|
||||
DRV_IMPORT_META_PREFIX = "drv_import_meta:"
|
||||
DRV_IMPORT_ERROR_LINES_PREFIX = "drv_import_error_lines:"
|
||||
DRV_IMPORT_STATUS_PREFIX = "drv_import_status:"
|
||||
DRV_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
DRV_IMPORT_TRANSPORTER_MAP_PREFIX = "drv_import_transporter_map:"
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
return redis.Redis.from_url(url, decode_responses=False)
|
||||
|
||||
|
||||
def _dedupe_headers(headers: List[str]) -> List[str]:
|
||||
counts: Dict[str, int] = {}
|
||||
unique: List[str] = []
|
||||
for header in headers:
|
||||
name = str(header or "").strip() or "COL"
|
||||
count = counts.get(name, 0) + 1
|
||||
counts[name] = count
|
||||
unique.append(name if count == 1 else f"{name} {count}")
|
||||
return unique
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
total_rows = sum(1 for _ in f) - 1
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
actualizar = meta.get("actualizar", False)
|
||||
existing_driver_keys: Set[Tuple[str, int]] = set()
|
||||
if actualizar:
|
||||
try:
|
||||
from api.v1.modules.a76.transportation.drivers.models import Driver
|
||||
with CoreSessionLocal() as session:
|
||||
for row in (
|
||||
session.query(Driver.transporter_key, Driver.line)
|
||||
.filter(
|
||||
Driver.tenant_id == tenant_id,
|
||||
Driver.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if row[0] is not None and row[1] is not None:
|
||||
existing_driver_keys.add(
|
||||
((row[0] or "").strip().upper(), int(row[1]))
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: could not load existing_driver_keys for actualizar: %s", e)
|
||||
|
||||
valid_transporter_keys, valid_country_ame, transporter_key_actual = load_drivers_fk_sets(tenant_id, company_id)
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_TRANSPORTER_MAP_PREFIX}{job_id}",
|
||||
json.dumps(transporter_key_actual).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store transporter map in Redis: %s", e)
|
||||
|
||||
logger.info(
|
||||
"Drivers import scan: job_id=%s tenant_id=%s company_id=%s actualizar=%s transportistas=%d",
|
||||
job_id, tenant_id, company_id, actualizar, len(valid_transporter_keys),
|
||||
)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
|
||||
error_path, "w", encoding="utf-8"
|
||||
) as f_err:
|
||||
sample = f_in.read(2048)
|
||||
f_in.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f_in, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
_ = validate_row_driver_desfase(row_norm, i)
|
||||
err = validate_row_driver(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
existing_driver_keys=existing_driver_keys,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_country_ame=valid_country_ame,
|
||||
)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("Drivers import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
def run_scan_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_scan(job_id, progress_callback=None)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("Drivers import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
result = _do_scan(job_id, progress_callback=on_progress)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store scan status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Drivers import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Drivers import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path) or {}
|
||||
actualizar = meta.get("actualizar", False)
|
||||
existing_driver_keys: Set[Tuple[str, int]] = set()
|
||||
if actualizar:
|
||||
try:
|
||||
from api.v1.modules.a76.transportation.drivers.models import Driver
|
||||
with CoreSessionLocal() as session:
|
||||
for row in (
|
||||
session.query(Driver.transporter_key, Driver.line)
|
||||
.filter(
|
||||
Driver.tenant_id == tenant_id,
|
||||
Driver.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
if row[0] is not None and row[1] is not None:
|
||||
existing_driver_keys.add(
|
||||
((row[0] or "").strip().upper(), int(row[1]))
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: could not load existing_driver_keys for actualizar: %s", e)
|
||||
|
||||
valid_transporter_keys, valid_country_ame, transporter_key_actual = load_drivers_fk_sets(tenant_id, company_id)
|
||||
|
||||
transporter_map_from_redis: Optional[Dict[str, str]] = None
|
||||
try:
|
||||
r = _get_redis()
|
||||
map_key = f"{DRV_IMPORT_TRANSPORTER_MAP_PREFIX}{job_id}"
|
||||
raw = r.get(map_key)
|
||||
if raw:
|
||||
transporter_map_from_redis = json.loads(raw.decode("utf-8"))
|
||||
logger.info(
|
||||
"Drivers import commit: using transporter map from Redis (job_id=%s, keys=%d)",
|
||||
job_id, len(transporter_map_from_redis),
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Drivers import commit: no transporter map in Redis for job_id=%s, using DB fallback",
|
||||
job_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Drivers import: could not load transporter map from Redis (job_id=%s): %s",
|
||||
job_id, e,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Drivers import commit: job_id=%s tenant_id=%s company_id=%s transportistas=%d",
|
||||
job_id, tenant_id, company_id, len(valid_transporter_keys),
|
||||
)
|
||||
|
||||
from api.v1.modules.a76.transportation.drivers.services import DriverService
|
||||
from api.v1.modules.a76.transportation.drivers.dto import DriverCreateDTO
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_invalid = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
seen_keys_in_file: Dict[str, int] = {}
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
sample = f.read(2048)
|
||||
f.seek(0)
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||
except Exception:
|
||||
dialect = "excel"
|
||||
reader = csv.reader(f, dialect=dialect)
|
||||
try:
|
||||
headers = next(reader)
|
||||
except StopIteration:
|
||||
headers = []
|
||||
headers = _dedupe_headers(headers)
|
||||
dict_reader = csv.DictReader(f, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
err = validate_row_driver(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
existing_driver_keys=existing_driver_keys,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_country_ame=valid_country_ame,
|
||||
)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
driver_key = (row_norm.get("CLAVE CONDUCTOR") or "").strip()[:80] or "-"
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": driver_key,
|
||||
"invoice": driver_key,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_driver_data(row_norm, tenant_id, company_id)
|
||||
if not data or not data.get("transporter_key") or data.get("line") is None:
|
||||
skipped_invalid += 1
|
||||
continue
|
||||
|
||||
tk = (data["transporter_key"] or "").strip()
|
||||
# Usar mapa del scan (Redis) si existe; si no, resolver en la sesión del commit (fallback)
|
||||
if transporter_map_from_redis is not None:
|
||||
tk_upper = tk.upper()
|
||||
if tk_upper not in transporter_map_from_redis:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": f"{tk}:{data.get('line')}",
|
||||
"invoice": f"{tk}:{data.get('line')}",
|
||||
"reason": f"El transportista {tk} no existe en el catálogo.",
|
||||
})
|
||||
continue
|
||||
data["transporter_key"] = transporter_map_from_redis[tk_upper]
|
||||
else:
|
||||
from api.v1.modules.a76.transportation.transporters.models import Transporter
|
||||
transporter_row = (
|
||||
session.query(Transporter.transporter_key)
|
||||
.filter(
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
func.upper(Transporter.transporter_key) == tk.upper(),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not transporter_row or not transporter_row[0]:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": f"{tk}:{data.get('line')}",
|
||||
"invoice": f"{tk}:{data.get('line')}",
|
||||
"reason": f"El transportista {tk} no existe en el catálogo.",
|
||||
})
|
||||
continue
|
||||
data["transporter_key"] = (transporter_row[0] or "").strip()
|
||||
|
||||
key = f"{data['transporter_key']}:{data['line']}"
|
||||
if key in seen_keys_in_file:
|
||||
skipped_duplicate += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"driver_key": key,
|
||||
"invoice": key,
|
||||
"reason": "Clave duplicada en el archivo (se usa la primera)",
|
||||
})
|
||||
continue
|
||||
seen_keys_in_file[key] = i
|
||||
|
||||
existing = DriverService.get_driver_by_key_and_line(
|
||||
session, data["transporter_key"], data["line"], str(company_id), tenant_id
|
||||
)
|
||||
try:
|
||||
if existing:
|
||||
update_fields = {
|
||||
k: v for k, v in data.items()
|
||||
if k not in ("transporter_key", "line", "company_id", "tenant_id")
|
||||
}
|
||||
for field, value in update_fields.items():
|
||||
setattr(existing, field, value)
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
else:
|
||||
create_data = DriverCreateDTO(**data)
|
||||
DriverService.create_driver(session, create_data)
|
||||
inserted_count += 1
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
skipped_invalid += 1
|
||||
err_msg = str(db_err)
|
||||
if "ForeignKeyViolation" in err_msg or "foreign key constraint" in err_msg.lower() or "driver_transporter_key_fkey" in err_msg:
|
||||
err_msg = f"El transportista {data.get('transporter_key', '')} no existe en el catálogo."
|
||||
logger.warning(
|
||||
"Drivers import: FK violation linea %d transporter_key=%r (valid_transporter_keys tiene %d claves)",
|
||||
i, data.get("transporter_key"), len(valid_transporter_keys),
|
||||
)
|
||||
skipped_details.append({
|
||||
"line": i, "driver_key": key, "invoice": key, "reason": err_msg,
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("Drivers import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Drivers import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.delete(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
r.delete(f"{DRV_IMPORT_TRANSPORTER_MAP_PREFIX}{job_id}")
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to delete status key: %s", e)
|
||||
|
||||
total_ok = inserted_count + updated_count
|
||||
if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0:
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.",
|
||||
}
|
||||
if total_ok == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros validos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
def run_commit_sync(job_id: str) -> Dict[str, Any]:
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("Drivers import: starting commit for job %s", job_id)
|
||||
result = _do_commit(job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_STATUS_PREFIX}{job_id}",
|
||||
json.dumps(result).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Drivers import: failed to store commit status in Redis: %s", e)
|
||||
return result
|
||||
@@ -27,6 +27,7 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
{"canonical": "NUM. IDENTIFICACION 2", "aliases": ["NUM IDENTIFICACION 2", "ID NUMERO 2", "ID NUMBER 2"]},
|
||||
{"canonical": "ESTADO 2", "aliases": ["STATE 2"]},
|
||||
{"canonical": "PAIS 2", "aliases": ["COUNTRY 2"]},
|
||||
{"canonical": "COL_EXTRA", "aliases": ["COLUMNA V", "COL V"]},
|
||||
],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_driver, validate_row_driver_desfase
|
||||
|
||||
__all__ = ["validate_row_driver", "validate_row_driver_desfase"]
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de conductores.
|
||||
Paridad Clarion: VALIDACIONES_CONDUCTOR, VALIDA_TODA_CONDUCTOR, VALIDA_PARCIAL_CONDUCTOR.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from ..common.common_validators import (
|
||||
MAX_LEN,
|
||||
check_max_length,
|
||||
check_int_positive,
|
||||
check_optional_birth_date,
|
||||
check_transportista_catalog,
|
||||
check_sexo_m_f,
|
||||
check_pais_catalog_drivers,
|
||||
check_material_peligroso_si_no,
|
||||
check_tipo_identificacion,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_driver_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""Obligatorios Clarion: Col A (TRANSPORTISTA) y Col C (CLAVE CONDUCTOR). Si falta uno, no se ejecutan validaciones."""
|
||||
err = check_max_length(
|
||||
row, "TRANSPORTISTA", MAX_LEN["transporter_key"], line_num, required=True
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_max_length(
|
||||
row, "CLAVE CONDUCTOR", MAX_LEN["driver_name"], line_num, required=True
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_driver_required_full(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""VALIDA_TODA: obligatorios A, C y LINEA (Col B) para registro nuevo."""
|
||||
err = validate_row_driver_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return check_int_positive(row, "LINEA", line_num)
|
||||
|
||||
|
||||
def validate_row_driver_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
checks = [
|
||||
("CLAVE CONDUCTOR", MAX_LEN["driver_name"]),
|
||||
("LICENCIA", MAX_LEN["license_number"]),
|
||||
("PERMISO LINEA EXPRESS", MAX_LEN["express_line_id"]),
|
||||
("IDENTIFICACION ACE", MAX_LEN["ace_id"]),
|
||||
("SEXO", MAX_LEN["gender"]),
|
||||
("PAIS NACIMIENTO", MAX_LEN["birth_country"]),
|
||||
("TRANSPORTA MAT. PELIGROSO?", MAX_LEN["hazardous_material_auth"]),
|
||||
("PERMISO MAT. PELIGROSO", MAX_LEN["hazardous_material_state"]),
|
||||
("NOMBRE(S)", MAX_LEN["first_name"]),
|
||||
("APELLIDO PATERNO", MAX_LEN["last_name"]),
|
||||
("FORMA IDENTIFICACION 1", MAX_LEN["id_key1"]),
|
||||
("NUM. IDENTIFICACION 1", MAX_LEN["id_number1"]),
|
||||
("ESTADO", MAX_LEN["id_state1"]),
|
||||
("PAIS", MAX_LEN["id_country1"]),
|
||||
("FORMA IDENTIFICACION 2", MAX_LEN["id_key2"]),
|
||||
("NUM. IDENTIFICACION 2", MAX_LEN["id_number2"]),
|
||||
("ESTADO 2", MAX_LEN["id_state2"]),
|
||||
("PAIS 2", MAX_LEN["id_country2"]),
|
||||
]
|
||||
for col, max_len in checks:
|
||||
err = check_max_length(row, col, max_len, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_driver_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
return check_optional_birth_date(row, "FECHA NACIMIENTO", line_num)
|
||||
|
||||
|
||||
def validaciones_conductores(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_transporter_keys: Optional[Set[str]] = None,
|
||||
valid_country_ame: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
VALIDACIONES_CONDUCTOR: reglas compartidas (longitudes, fecha, transportista en catálogo,
|
||||
sexo M/F, país nacimiento, material peligroso Si/No, tipo ID 1/2, país ID1/ID2).
|
||||
"""
|
||||
err = validate_row_driver_lengths(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_driver_date(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_transportista_catalog(row, line_num, valid_transporter_keys)
|
||||
if err:
|
||||
return err
|
||||
err = check_sexo_m_f(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_pais_catalog_drivers(
|
||||
row, "PAIS NACIMIENTO", line_num, valid_country_ame, col_letter="Col. I"
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_material_peligroso_si_no(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_tipo_identificacion(
|
||||
row,
|
||||
"FORMA IDENTIFICACION 1",
|
||||
line_num,
|
||||
col_letter="Col. N",
|
||||
primera_o_segunda="Primera",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_pais_catalog_drivers(
|
||||
row, "PAIS", line_num, valid_country_ame, col_letter="Col. Q"
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_tipo_identificacion(
|
||||
row,
|
||||
"FORMA IDENTIFICACION 2",
|
||||
line_num,
|
||||
col_letter="Col. R",
|
||||
primera_o_segunda="Segunda",
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
err = check_pais_catalog_drivers(
|
||||
row, "PAIS 2", line_num, valid_country_ame, col_letter="Col. U"
|
||||
)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
|
||||
|
||||
def valida_toda_conductor(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_transporter_keys: Optional[Set[str]] = None,
|
||||
valid_country_ame: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""VALIDA_TODA_CONDUCTOR: obligatorios A, C y LINEA (B) + validaciones_conductores (registro nuevo)."""
|
||||
err = validate_row_driver_required_full(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
return validaciones_conductores(
|
||||
row, line_num,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_country_ame=valid_country_ame,
|
||||
)
|
||||
|
||||
|
||||
def valida_parcial_conductor(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
valid_transporter_keys: Optional[Set[str]] = None,
|
||||
valid_country_ame: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""VALIDA_PARCIAL_CONDUCTOR: solo validaciones_conductores (actualizar registro existente)."""
|
||||
return validaciones_conductores(
|
||||
row, line_num,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_country_ame=valid_country_ame,
|
||||
)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
Punto de entrada de validación para import de una fila conductor.
|
||||
Paridad Clarion: desfase (advertencia), obligatorios A y C, VALIDA_TODA vs VALIDA_PARCIAL según actualizar y clave existente.
|
||||
"""
|
||||
from typing import Dict, Any, Optional, Set, Tuple
|
||||
|
||||
from ..common.common_validators import parse_int, check_desfase_drivers
|
||||
from .common import (
|
||||
validate_row_driver_required,
|
||||
valida_toda_conductor,
|
||||
valida_parcial_conductor,
|
||||
)
|
||||
|
||||
|
||||
def validate_row_driver(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
actualizar: bool = False,
|
||||
existing_driver_keys: Optional[Set[Tuple[str, int]]] = None,
|
||||
valid_transporter_keys: Optional[Set[str]] = None,
|
||||
valid_country_ame: Optional[Set[str]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de conductores.
|
||||
1. Obligatorios A (TRANSPORTISTA) y C (CLAVE CONDUCTOR) vacíos → error.
|
||||
2. Si actualizar y (TRANSPORTISTA, LINEA) en existing_driver_keys → valida_parcial_conductor.
|
||||
3. Si no actualizar o conductor no existe → valida_toda_conductor (obligatorios A, C y LINEA + validaciones).
|
||||
Desfase (COL_EXTRA) no se valida aquí; el caller puede llamar validate_row_driver_desfase para advertencias no bloqueantes.
|
||||
"""
|
||||
err = validate_row_driver_required(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
|
||||
existing = existing_driver_keys or set()
|
||||
transporter_key = (row.get("TRANSPORTISTA") or "").strip().upper()
|
||||
line = parse_int(row.get("LINEA"))
|
||||
use_partial = (
|
||||
actualizar
|
||||
and bool(transporter_key and line is not None)
|
||||
and (transporter_key, line) in existing
|
||||
)
|
||||
|
||||
if use_partial:
|
||||
err = valida_parcial_conductor(
|
||||
row,
|
||||
line_num,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_country_ame=valid_country_ame,
|
||||
)
|
||||
else:
|
||||
err = valida_toda_conductor(
|
||||
row,
|
||||
line_num,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_country_ame=valid_country_ame,
|
||||
)
|
||||
return err
|
||||
|
||||
|
||||
def validate_row_driver_desfase(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Advertencia de desfase si COL_EXTRA (Col V) tiene valor. No bloqueante; el caller puede acumular en warnings.
|
||||
"""
|
||||
return check_desfase_drivers(row, line_num)
|
||||
@@ -0,0 +1 @@
|
||||
# common_validators, mappers (no fk_loader for exchange_rate)
|
||||
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
Validadores reutilizables para import CSV de tipos de cambio.
|
||||
"""
|
||||
from datetime import datetime, time
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
DATE_FORMATS: List[str] = ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"]
|
||||
|
||||
# Valores que envía el frontend (globalCsvParams dateFormat) -> formato strptime
|
||||
# Cuando el usuario elige un formato en "Parámetros globales", solo se aceptan fechas en ese formato.
|
||||
DATE_FORMAT_PREFERENCE_MAP: Dict[str, str] = {
|
||||
"dd/mm/yyyy": "%d/%m/%Y",
|
||||
"mm/dd/yyyy": "%m/%d/%Y",
|
||||
"yyyy-mm-dd": "%Y-%m-%d",
|
||||
}
|
||||
|
||||
CURRENCY_MAX = 7
|
||||
|
||||
|
||||
def parse_date(val: Optional[str], date_format_preference: Optional[str] = None) -> Optional[datetime]:
|
||||
"""
|
||||
Parsea fecha. Si date_format_preference está definido (formato elegido en el frontend),
|
||||
solo se acepta ese formato; si la cadena no coincide, se rechaza.
|
||||
Si no hay preferencia, se intentan todos los formatos (retrocompatibilidad).
|
||||
"""
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
raw = str(val).strip()
|
||||
if date_format_preference and date_format_preference in DATE_FORMAT_PREFERENCE_MAP:
|
||||
fmt = DATE_FORMAT_PREFERENCE_MAP[date_format_preference]
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
return None
|
||||
for fmt in DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(raw, fmt)
|
||||
return datetime.combine(parsed.date(), time.min)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def parse_decimal_positive(val: Optional[str]) -> Optional[Decimal]:
|
||||
if not val or not str(val).strip():
|
||||
return None
|
||||
try:
|
||||
v = float(str(val).strip().replace(",", "."))
|
||||
if v <= 0:
|
||||
return None
|
||||
return Decimal(str(round(v, 6)))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def check_required_date(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
fecha_raw = (row.get("FECHA") or "").strip()
|
||||
if not fecha_raw:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Requerido"}
|
||||
if parse_date(fecha_raw) is None:
|
||||
return {"line": line_num, "col": "FECHA", "msg": "Formato de fecha inválido (use YYYY-MM-DD o DD/MM/YYYY)"}
|
||||
return None
|
||||
|
||||
|
||||
def check_required_value_positive(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
valor_raw = (row.get("VALOR") or "").strip()
|
||||
if not valor_raw:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Requerido"}
|
||||
try:
|
||||
v = float(valor_raw.replace(",", "."))
|
||||
if v <= 0:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser mayor que cero"}
|
||||
except ValueError:
|
||||
return {"line": line_num, "col": "VALOR", "msg": "Debe ser un número"}
|
||||
return None
|
||||
|
||||
|
||||
def check_optional_max_length(row: Dict[str, Any], col: str, max_len: int, line_num: int) -> Optional[Dict[str, Any]]:
|
||||
val = (row.get(col) or "").strip()
|
||||
if not val:
|
||||
return None
|
||||
if len(val) > max_len:
|
||||
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
|
||||
return None
|
||||
|
||||
|
||||
FECHA_MAX_LEN = 10
|
||||
MSG_FECHA_LONGITUD = "Error: (Col. A) La Fecha: {fecha} supera la longitud de caracteres."
|
||||
MSG_FECHA_LONGITUD_SOLUCION = "Capturar en la columna A el campo Fecha con este formato ##/##/####."
|
||||
MSG_FECHA_DIA_INVALIDO = "Error: (Col. A) El día {dia} de la Fecha: {fecha} no es válido para el mes."
|
||||
MSG_FECHA_DIA_SOLUCION = "Capturar correctamente en la columna A el día del campo Fecha, con este formato ##/##/####. (Día/Mes/Año)"
|
||||
MSG_FECHA_MES_INVALIDO = "Error: (Col. A) El mes {mes} de la Fecha: {fecha} es mayor a 12 esto no es valido."
|
||||
MSG_FECHA_MES_SOLUCION = "Capturar correctamente en la columna A el mes del campo Fecha, con este formato ##/##/####. (Día/Mes/Año)"
|
||||
|
||||
# Etiquetas para mensaje cuando no coincide con el formato elegido
|
||||
DATE_FORMAT_LABELS: Dict[str, str] = {
|
||||
"dd/mm/yyyy": "DD/MM/YYYY (Día/Mes/Año)",
|
||||
"mm/dd/yyyy": "MM/DD/YYYY (Mes/Día/Año)",
|
||||
"yyyy-mm-dd": "YYYY-MM-DD (Año-Mes-Día)",
|
||||
}
|
||||
|
||||
|
||||
def validate_fecha_clarion(
|
||||
raw_fecha: str, line_num: int, date_format_preference: Optional[str] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida la fecha según reglas Clarion: longitud ≤ 10, día válido para el mes, mes ≤ 12.
|
||||
Sin límite de año. Si date_format_preference está definido (ej. dd/mm/yyyy), se prioriza ese formato.
|
||||
"""
|
||||
if not raw_fecha or not str(raw_fecha).strip():
|
||||
return None
|
||||
raw = str(raw_fecha).strip()
|
||||
if len(raw) > FECHA_MAX_LEN:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA",
|
||||
"msg": f"{MSG_FECHA_LONGITUD.format(fecha=raw)} {MSG_FECHA_LONGITUD_SOLUCION}",
|
||||
}
|
||||
parsed = parse_date(raw, date_format_preference)
|
||||
if parsed is None:
|
||||
format_label = (
|
||||
DATE_FORMAT_LABELS.get(date_format_preference, "##/##/####")
|
||||
if date_format_preference
|
||||
else "##/##/####"
|
||||
)
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA",
|
||||
"msg": f"Error: (Col. A) La Fecha: {raw} no coincide con el formato elegido ({format_label}). {MSG_FECHA_LONGITUD_SOLUCION}",
|
||||
}
|
||||
d = parsed.date()
|
||||
if d.month > 12:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA",
|
||||
"msg": f"{MSG_FECHA_MES_INVALIDO.format(mes=d.month, fecha=raw)} {MSG_FECHA_MES_SOLUCION}",
|
||||
}
|
||||
return None
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Mapeo fila CSV → datos para ExchangeRate.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .common_validators import (
|
||||
parse_date,
|
||||
parse_decimal_positive,
|
||||
)
|
||||
|
||||
CURRENCY_MAX = 7
|
||||
|
||||
|
||||
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if not s:
|
||||
return None
|
||||
if max_len and len(s) > max_len:
|
||||
return s[:max_len]
|
||||
return s
|
||||
|
||||
|
||||
def row_to_exchange_rate_data(
|
||||
row_norm: Dict[str, Any], tenant_id: int, company_id: int, date_format_preference: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Build dict for ExchangeRate model. Returns {} if FECHA or VALOR invalid."""
|
||||
parsed_date = parse_date(row_norm.get("FECHA"), date_format_preference)
|
||||
value_decimal = parse_decimal_positive(row_norm.get("VALOR"))
|
||||
if not parsed_date or value_decimal is None:
|
||||
return {}
|
||||
local = _str_or_none(row_norm.get("MONEDA_LOCAL"), CURRENCY_MAX)
|
||||
if local:
|
||||
local = local.upper()
|
||||
foreign = _str_or_none(row_norm.get("MONEDA_EXTRANJERA"), CURRENCY_MAX)
|
||||
if foreign:
|
||||
foreign = foreign.upper()
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
"date": parsed_date,
|
||||
"value": value_decimal,
|
||||
"local_currency": local,
|
||||
"foreign_currency": foreign,
|
||||
}
|
||||
@@ -10,10 +10,11 @@ from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Dict, Any
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
@@ -39,11 +40,20 @@ def _get_redis():
|
||||
async def upload_import_file(
|
||||
file: UploadFile = File(...),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
reemplazar_sin_preguntar: bool = Query(
|
||||
True,
|
||||
description="Si True, reemplaza tipos de cambio existentes para la misma fecha; si False, solo agrega nuevos (omite fechas ya existentes)",
|
||||
),
|
||||
date_format: Optional[str] = Query(
|
||||
None,
|
||||
description="Formato de fecha del CSV: dd/mm/yyyy, mm/dd/yyyy o yyyy-mm-dd. Si no se envía, se intentan todos.",
|
||||
),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Fase 1: Subir CSV, guardar en Redis, encolar tarea de escaneo.
|
||||
Parámetros globales de carga: reemplazar_sin_preguntar (Modo Reemplazar vs Actualizar), date_format (Formato de Fecha).
|
||||
"""
|
||||
try:
|
||||
tenant_id = validate_access_to_resource(db, company_id, current_user)
|
||||
@@ -62,6 +72,8 @@ async def upload_import_file(
|
||||
"company_id": company_id,
|
||||
"user_id": current_user.get("id"),
|
||||
"template_id": "exchange_rates",
|
||||
"reemplazar_sin_preguntar": reemplazar_sin_preguntar,
|
||||
"date_format": date_format,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -81,7 +93,7 @@ async def upload_import_file(
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
241
backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py
Normal file
241
backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py
Normal file
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
Tareas Celery para importación CSV de Tipos de Cambio.
|
||||
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
||||
Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import CoreSessionLocal
|
||||
|
||||
from ..common import storage as common_storage
|
||||
from ..common import normalize as common_normalize
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from ..common import csv_reader as common_csv_reader
|
||||
from .template_config import row_from_template
|
||||
from .validators import validate_row_exchange_rate
|
||||
from .common.mappers import row_to_exchange_rate_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JOB_TYPE = "er"
|
||||
TEMPLATE_ID = "exchange_rates"
|
||||
|
||||
# Para routes.py
|
||||
ER_IMPORT_FILE_PREFIX = "er_import_file:"
|
||||
ER_IMPORT_META_PREFIX = "er_import_meta:"
|
||||
ER_IMPORT_ERROR_LINES_PREFIX = "er_import_error_lines:"
|
||||
ER_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
||||
|
||||
|
||||
def _norm_row(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return row_from_template(row, common_normalize.normalize_header, TEMPLATE_ID)
|
||||
|
||||
|
||||
def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import")
|
||||
if not file_path:
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
|
||||
try:
|
||||
total_rows = common_csv_reader.count_csv_rows(file_path)
|
||||
except Exception as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
# Formato activo del selector del frontend; si no viene, mismo default que el front (dd/mm/yyyy)
|
||||
date_format_preference = meta.get("date_format") or meta.get("dateFormat") or "dd/mm/yyyy"
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_exchange_rate(row_norm, i, raw_row=row, date_format_preference=date_format_preference)
|
||||
if err:
|
||||
error_count += 1
|
||||
error_lines_list.append(err["line"])
|
||||
f_err.write(json.dumps(err) + "\n")
|
||||
if len(errors_detail) < 500:
|
||||
errors_detail.append({
|
||||
"line": err["line"],
|
||||
"col": err.get("col", ""),
|
||||
"msg": err.get("msg", ""),
|
||||
})
|
||||
processed_rows += 1
|
||||
|
||||
if error_lines_list:
|
||||
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
||||
except Exception as e:
|
||||
logger.error("ER import scan failed: %s", e)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def scan_file(self, job_id: str, config: str = None):
|
||||
logger.info("ER import: starting scan for job %s", job_id)
|
||||
|
||||
def on_progress(current: int, total: int, errors: int) -> None:
|
||||
self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors})
|
||||
|
||||
return _do_scan(job_id, progress_callback=on_progress)
|
||||
|
||||
|
||||
def _do_commit(job_id: str) -> Dict[str, Any]:
|
||||
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "ER import")
|
||||
if not file_path:
|
||||
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
if not os.path.exists(alt_path):
|
||||
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
||||
file_path = alt_path
|
||||
else:
|
||||
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "ER import")
|
||||
|
||||
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
||||
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
||||
|
||||
try:
|
||||
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
||||
except ValueError as e:
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
meta = common_meta.load_meta(file_path)
|
||||
reemplazar_sin_preguntar = meta.get("reemplazar_sin_preguntar", True)
|
||||
date_format_preference = meta.get("date_format") or meta.get("dateFormat")
|
||||
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
|
||||
inserted_count = 0
|
||||
updated_count = 0
|
||||
skipped_duplicate = 0
|
||||
skipped_invalid = 0
|
||||
skipped_details: List[Dict[str, Any]] = []
|
||||
meta_path = common_meta.get_meta_path(file_path)
|
||||
|
||||
try:
|
||||
with CoreSessionLocal() as session:
|
||||
existing_by_date: Dict[tuple, ExchangeRate] = {}
|
||||
for er in (
|
||||
session.query(ExchangeRate)
|
||||
.filter(
|
||||
ExchangeRate.tenant_id == tenant_id,
|
||||
ExchangeRate.company_id == company_id,
|
||||
)
|
||||
.all()
|
||||
):
|
||||
d = er.date.date() if hasattr(er.date, "date") else er.date
|
||||
existing_by_date[(tenant_id, company_id, d)] = er
|
||||
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if i in error_lines:
|
||||
continue
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
err = validate_row_exchange_rate(row_norm, i, raw_row=row, date_format_preference=date_format_preference)
|
||||
if err:
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({
|
||||
"line": i,
|
||||
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
||||
})
|
||||
continue
|
||||
|
||||
data = row_to_exchange_rate_data(row_norm, tenant_id, company_id, date_format_preference)
|
||||
if not data or not data.get("date"):
|
||||
skipped_invalid += 1
|
||||
skipped_details.append({"line": i, "reason": "FECHA o VALOR no válidos"})
|
||||
continue
|
||||
|
||||
key_date = data["date"].date() if hasattr(data["date"], "date") else data["date"]
|
||||
existing = existing_by_date.get((tenant_id, company_id, key_date))
|
||||
if existing:
|
||||
if not reemplazar_sin_preguntar:
|
||||
skipped_duplicate += 1
|
||||
continue
|
||||
existing.value = data["value"]
|
||||
existing.local_currency = data.get("local_currency")
|
||||
existing.foreign_currency = data.get("foreign_currency")
|
||||
session.add(existing)
|
||||
updated_count += 1
|
||||
else:
|
||||
new_er = ExchangeRate(**data)
|
||||
session.add(new_er)
|
||||
existing_by_date[(tenant_id, company_id, key_date)] = new_er
|
||||
inserted_count += 1
|
||||
|
||||
try:
|
||||
session.commit()
|
||||
except Exception as db_err:
|
||||
session.rollback()
|
||||
logger.error("ER import DB error: %s", db_err)
|
||||
return {"status": "failed", "error": str(db_err)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("ER import task failed")
|
||||
return {"status": "failed", "error": str(e)}
|
||||
|
||||
common_storage.cleanup_import_job(
|
||||
JOB_TYPE, job_id,
|
||||
file_path=file_path,
|
||||
error_path=error_path,
|
||||
meta_path=meta_path,
|
||||
)
|
||||
|
||||
if inserted_count == 0 and updated_count == 0 and (skipped_invalid > 0 or skipped_duplicate > 0):
|
||||
return {
|
||||
"status": "warning",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
"message": f"No se insertaron registros. {skipped_invalid} rechazados, {skipped_duplicate} omitidos por fecha existente.",
|
||||
}
|
||||
if inserted_count == 0 and updated_count == 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": "No hay registros válidos en el archivo CSV",
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
return {
|
||||
"status": "finished",
|
||||
"inserted": inserted_count,
|
||||
"updated": updated_count,
|
||||
"skipped_invalid": skipped_invalid,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"skipped_missing_fk": 0,
|
||||
"skipped_details": skipped_details,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def insert_valid_rows(self, job_id: str):
|
||||
logger.info("ER import: starting commit for job %s", job_id)
|
||||
return _do_commit(job_id)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .create import validate_row_exchange_rate
|
||||
|
||||
__all__ = ["validate_row_exchange_rate"]
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Validaciones comunes de fila para import CSV de tipos de cambio.
|
||||
Paridad Clarion: desfase (Col C vacía), obligatorios (Col A Fecha, Col B Tipo de Cambio),
|
||||
validación fecha (longitud, día acorde al mes, mes ≤ 12; sin límite de año).
|
||||
"""
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..common.common_validators import (
|
||||
check_required_value_positive,
|
||||
check_optional_max_length,
|
||||
validate_fecha_clarion,
|
||||
CURRENCY_MAX,
|
||||
)
|
||||
|
||||
|
||||
MSG_DESFASE = "Error: Existe un desfase en esta línea."
|
||||
MSG_DESFASE_SOLUCION = "Revisar esta línea del archivo CSV y verificar cada campo este en la posicion correcta."
|
||||
|
||||
MSG_OBLIGATORIOS = "Existen campos vacios que son obligatorios, es la (Col.A) Fecha , (Col.B) Tipo de Cambio."
|
||||
MSG_OBLIGATORIOS_SOLUCION = "Revisar la línea del archivo y capturar los campos con la información correcta."
|
||||
|
||||
|
||||
def validate_row_desfase(raw_row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Si la fila tiene 3 o más columnas y la 3ª tiene valor, error de desfase (Clarion ColumnaC <> '').
|
||||
"""
|
||||
values_ordered = list(raw_row.values()) if raw_row else []
|
||||
if len(values_ordered) >= 3 and (values_ordered[2] or "").strip():
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "",
|
||||
"msg": f"{MSG_DESFASE} {MSG_DESFASE_SOLUCION}",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_required_exchange_rate(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
|
||||
"""FECHA y VALOR obligatorios con mensaje Clarion."""
|
||||
fecha = (row.get("FECHA") or "").strip()
|
||||
valor = (row.get("VALOR") or "").strip()
|
||||
if not fecha or not valor:
|
||||
return {
|
||||
"line": line_num,
|
||||
"col": "FECHA" if not fecha else "VALOR",
|
||||
"msg": f"{MSG_OBLIGATORIOS} {MSG_OBLIGATORIOS_SOLUCION}",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_row_fecha_clarion(
|
||||
row: Dict[str, Any], line_num: int, date_format_preference: Optional[str] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Longitud ≤ 10 y fecha válida (día acorde al mes, mes ≤ 12; sin límite de año)."""
|
||||
raw_fecha = (row.get("FECHA") or "").strip()
|
||||
if not raw_fecha:
|
||||
return None
|
||||
return validate_fecha_clarion(raw_fecha, line_num, date_format_preference)
|
||||
|
||||
|
||||
def validate_row_exchange_rate(
|
||||
row: Dict[str, Any],
|
||||
line_num: int,
|
||||
raw_row: Optional[Dict[str, Any]] = None,
|
||||
date_format_preference: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Valida una fila de CSV de tipos de cambio.
|
||||
Orden: desfase (si raw_row) → obligatorios → fecha Clarion → VALOR > 0 → MONEDA opc (max 7).
|
||||
date_format_preference: valor del parámetro global (ej. dd/mm/yyyy, mm/dd/yyyy, yyyy-mm-dd).
|
||||
"""
|
||||
if raw_row is not None:
|
||||
err = validate_row_desfase(raw_row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_required_exchange_rate(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = validate_row_fecha_clarion(row, line_num, date_format_preference)
|
||||
if err:
|
||||
return err
|
||||
err = check_required_value_positive(row, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "MONEDA_LOCAL", CURRENCY_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
err = check_optional_max_length(row, "MONEDA_EXTRANJERA", CURRENCY_MAX, line_num)
|
||||
if err:
|
||||
return err
|
||||
return None
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user