Mezlca de migraciones mas recientes
This commit is contained in:
@@ -1,19 +1,184 @@
|
||||
"""
|
||||
Audit Log Router
|
||||
"""
|
||||
from typing import List, Optional
|
||||
from datetime import date
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import or_, desc, distinct
|
||||
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user # Assuming this exists
|
||||
from core.security import get_current_user, get_tenant_from_token
|
||||
from core.storage_s3 import get_object_bytes, list_objects_tree, should_ensure_s3_bucket
|
||||
from .models import AuditLog
|
||||
from .schemas import AuditLogListResponse, AuditLogResponse, AuditLogDetailResponse
|
||||
from .schemas import (
|
||||
AuditFileBreadcrumb,
|
||||
AuditFileBrowserResponse,
|
||||
AuditFileFolderItem,
|
||||
AuditFileObjectItem,
|
||||
AuditLogDetailResponse,
|
||||
AuditLogListResponse,
|
||||
)
|
||||
from api.v1.modules.a76.general_catalogs.company.models import Company
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_SEGMENT_LABELS = {
|
||||
"tenants": "Espacio",
|
||||
"companies": "Companias",
|
||||
"users": "Usuarios",
|
||||
"imports": "Importaciones",
|
||||
"csv": "Archivos CSV",
|
||||
"branding": "Logotipos",
|
||||
"certificates": "Certificados",
|
||||
"customs_brokers": "Agentes aduanales",
|
||||
"keys": "Llaves",
|
||||
"cove": "COVE",
|
||||
"doda": "DODA",
|
||||
"system": "Sistema",
|
||||
"help": "Ayuda",
|
||||
}
|
||||
|
||||
|
||||
def _tenant_id_from_user(current_user: Dict[str, Any]) -> int:
|
||||
tenant_id = get_tenant_from_token(current_user) or current_user.get("tenant_id")
|
||||
if not tenant_id:
|
||||
raise HTTPException(status_code=401, detail="User context is invalid")
|
||||
return int(tenant_id)
|
||||
|
||||
|
||||
def _normalize_relative_path(raw: Optional[str]) -> str:
|
||||
if not raw:
|
||||
return ""
|
||||
val = raw.strip().strip("/")
|
||||
if not val:
|
||||
return ""
|
||||
if ".." in val or "\\" in val:
|
||||
raise HTTPException(status_code=400, detail="Invalid path")
|
||||
parts = [p for p in val.split("/") if p]
|
||||
for part in parts:
|
||||
if part in (".", ".."):
|
||||
raise HTTPException(status_code=400, detail="Invalid path segment")
|
||||
return "/".join(parts)
|
||||
|
||||
|
||||
def _tenant_prefix(tenant_id: int) -> str:
|
||||
return f"tenants/{tenant_id}/"
|
||||
|
||||
|
||||
def _relative_from_tenant_prefix(key: str, tenant_prefix: str) -> str:
|
||||
if not key.startswith(tenant_prefix):
|
||||
raise HTTPException(status_code=403, detail="Access denied to object key")
|
||||
return key[len(tenant_prefix) :].strip("/")
|
||||
|
||||
|
||||
def _companies_map(db: Session, tenant_id: int) -> Dict[str, str]:
|
||||
rows = (
|
||||
db.query(Company.id, Company.name)
|
||||
.filter(Company.tenant_id == tenant_id, Company.deleted_at.is_(None))
|
||||
.all()
|
||||
)
|
||||
out: Dict[str, str] = {}
|
||||
for company_id, company_name in rows:
|
||||
if company_id is None:
|
||||
continue
|
||||
safe_name = (company_name or "").strip()
|
||||
out[str(company_id)] = safe_name or "Compania"
|
||||
return out
|
||||
|
||||
|
||||
def _display_segment(
|
||||
part: str,
|
||||
prev_part: Optional[str],
|
||||
company_names: Dict[str, str],
|
||||
current_user_id: Optional[str] = None,
|
||||
current_user_label: Optional[str] = None,
|
||||
) -> str:
|
||||
if prev_part == "companies":
|
||||
return company_names.get(part, "Compania")
|
||||
if prev_part == "users":
|
||||
# Para carpetas de usuarios, mostrar un nombre amigable:
|
||||
# - Si es el propio usuario actual, usar preferred_username/email/nombre.
|
||||
# - Para otros IDs (UUIDs) mostrar un label genérico.
|
||||
if current_user_id and part == str(current_user_id):
|
||||
return (current_user_label or "").strip() or "Usuario"
|
||||
return "Usuario"
|
||||
if part in _SEGMENT_LABELS:
|
||||
return _SEGMENT_LABELS[part]
|
||||
# Evita exponer IDs puros en UI.
|
||||
if part.isdigit():
|
||||
return "Elemento"
|
||||
return part.replace("_", " ").strip().title() or "Elemento"
|
||||
|
||||
|
||||
def _display_path(
|
||||
rel_path: str,
|
||||
company_names: Dict[str, str],
|
||||
current_user_id: Optional[str] = None,
|
||||
current_user_label: Optional[str] = None,
|
||||
) -> str:
|
||||
if not rel_path:
|
||||
return "Raiz de archivos"
|
||||
parts = [p for p in rel_path.split("/") if p]
|
||||
labels: List[str] = []
|
||||
prev: Optional[str] = None
|
||||
for part in parts:
|
||||
labels.append(
|
||||
_display_segment(
|
||||
part,
|
||||
prev,
|
||||
company_names,
|
||||
current_user_id=current_user_id,
|
||||
current_user_label=current_user_label,
|
||||
)
|
||||
)
|
||||
prev = part
|
||||
return " / ".join(labels)
|
||||
|
||||
|
||||
def _display_file_name(filename: str) -> str:
|
||||
stem, dot, ext = filename.rpartition(".")
|
||||
if not dot:
|
||||
stem = filename
|
||||
ext = ""
|
||||
if stem.isdigit():
|
||||
return f"Archivo{f'.{ext}' if ext else ''}"
|
||||
return filename
|
||||
|
||||
|
||||
def _build_breadcrumbs(
|
||||
rel_path: str,
|
||||
company_names: Dict[str, str],
|
||||
current_user_id: Optional[str] = None,
|
||||
current_user_label: Optional[str] = None,
|
||||
) -> List[AuditFileBreadcrumb]:
|
||||
breadcrumbs: List[AuditFileBreadcrumb] = [
|
||||
AuditFileBreadcrumb(path="", display_name="Raiz de archivos")
|
||||
]
|
||||
if not rel_path:
|
||||
return breadcrumbs
|
||||
parts = [p for p in rel_path.split("/") if p]
|
||||
prev: Optional[str] = None
|
||||
acc: List[str] = []
|
||||
for part in parts:
|
||||
acc.append(part)
|
||||
breadcrumbs.append(
|
||||
AuditFileBreadcrumb(
|
||||
path="/".join(acc),
|
||||
display_name=_display_segment(
|
||||
part,
|
||||
prev,
|
||||
company_names,
|
||||
current_user_id=current_user_id,
|
||||
current_user_label=current_user_label,
|
||||
),
|
||||
)
|
||||
)
|
||||
prev = part
|
||||
return breadcrumbs
|
||||
|
||||
@router.get("/bitacora", response_model=AuditLogListResponse)
|
||||
async def get_bitacora(
|
||||
page: int = Query(1, ge=1),
|
||||
@@ -91,3 +256,127 @@ async def get_audit_detail(spec_id: int, db: Session = Depends(get_core_db)):
|
||||
if not log:
|
||||
raise HTTPException(status_code=404, detail="Log entry not found")
|
||||
return log
|
||||
|
||||
|
||||
@router.get("/files", response_model=AuditFileBrowserResponse)
|
||||
async def list_tenant_files(
|
||||
path: Optional[str] = Query(default="", description="Ruta relativa de navegación."),
|
||||
continuation_token: Optional[str] = Query(default=None),
|
||||
max_keys: int = Query(default=100, ge=1, le=500),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Explorador de archivos de solo lectura para Auditoría.
|
||||
"""
|
||||
if not should_ensure_s3_bucket():
|
||||
raise HTTPException(status_code=400, detail="S3 storage is disabled")
|
||||
|
||||
tenant_id = _tenant_id_from_user(current_user)
|
||||
tenant_prefix = _tenant_prefix(tenant_id)
|
||||
rel_path = _normalize_relative_path(path)
|
||||
list_prefix = f"{tenant_prefix}{rel_path}/" if rel_path else tenant_prefix
|
||||
|
||||
# Datos del usuario actual para etiquetas amigables bajo /users/{id}/...
|
||||
current_user_id = str(current_user.get("sub") or "")
|
||||
current_user_label = (
|
||||
(current_user.get("preferred_username") or "").strip()
|
||||
or (current_user.get("name") or "").strip()
|
||||
or (current_user.get("email") or "").strip()
|
||||
or "Usuario"
|
||||
)
|
||||
|
||||
data = list_objects_tree(
|
||||
prefix=list_prefix,
|
||||
delimiter="/",
|
||||
max_keys=max_keys,
|
||||
continuation_token=continuation_token,
|
||||
)
|
||||
|
||||
company_names = _companies_map(db, tenant_id)
|
||||
folders: List[AuditFileFolderItem] = []
|
||||
for prefix in data.get("prefixes", []):
|
||||
rel = _relative_from_tenant_prefix(prefix, tenant_prefix)
|
||||
folders.append(
|
||||
AuditFileFolderItem(
|
||||
path=rel,
|
||||
# Para el gestor de archivos mostramos el nombre amigable del último segmento
|
||||
# (empresa, usuario actual, etc.), no el ID bruto.
|
||||
display_name=_display_path(
|
||||
rel,
|
||||
company_names,
|
||||
current_user_id=current_user_id,
|
||||
current_user_label=current_user_label,
|
||||
).split(" / ")[-1],
|
||||
)
|
||||
)
|
||||
|
||||
files: List[AuditFileObjectItem] = []
|
||||
for obj in data.get("objects", []):
|
||||
key = obj.get("key")
|
||||
if not key:
|
||||
continue
|
||||
rel = _relative_from_tenant_prefix(key, tenant_prefix)
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
files.append(
|
||||
AuditFileObjectItem(
|
||||
path=rel,
|
||||
display_name=_display_file_name(name),
|
||||
size=int(obj.get("size", 0) or 0),
|
||||
last_modified=obj.get("last_modified"),
|
||||
)
|
||||
)
|
||||
|
||||
return AuditFileBrowserResponse(
|
||||
current_path=rel_path,
|
||||
display_path=_display_path(
|
||||
rel_path,
|
||||
company_names,
|
||||
current_user_id=current_user_id,
|
||||
current_user_label=current_user_label,
|
||||
),
|
||||
breadcrumbs=_build_breadcrumbs(
|
||||
rel_path,
|
||||
company_names,
|
||||
current_user_id=current_user_id,
|
||||
current_user_label=current_user_label,
|
||||
),
|
||||
folders=sorted(folders, key=lambda x: x.display_name.lower()),
|
||||
files=sorted(files, key=lambda x: x.display_name.lower()),
|
||||
next_token=data.get("next_continuation_token"),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/files/download")
|
||||
async def download_tenant_file(
|
||||
path: str = Query(..., description="Ruta relativa del archivo a descargar."),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Descarga segura (backend streaming) de archivos autorizados.
|
||||
"""
|
||||
if not should_ensure_s3_bucket():
|
||||
raise HTTPException(status_code=400, detail="S3 storage is disabled")
|
||||
|
||||
tenant_id = _tenant_id_from_user(current_user)
|
||||
tenant_prefix = _tenant_prefix(tenant_id)
|
||||
rel_path = _normalize_relative_path(path)
|
||||
if not rel_path or rel_path.endswith("/"):
|
||||
raise HTTPException(status_code=400, detail="A file path is required")
|
||||
|
||||
object_key = f"{tenant_prefix}{rel_path}"
|
||||
if not object_key.startswith(tenant_prefix):
|
||||
raise HTTPException(status_code=403, detail="Access denied to object key")
|
||||
|
||||
try:
|
||||
body = get_object_bytes(object_key)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {e}") from e
|
||||
|
||||
filename = rel_path.rsplit("/", 1)[-1]
|
||||
headers = {"Content-Disposition": f'attachment; filename="{filename}"'}
|
||||
return StreamingResponse(
|
||||
iter([body]),
|
||||
media_type="application/octet-stream",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
@@ -50,3 +50,29 @@ class AuditLogListResponse(BaseModel):
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
|
||||
|
||||
class AuditFileBreadcrumb(BaseModel):
|
||||
path: str = Field(default="")
|
||||
display_name: str
|
||||
|
||||
|
||||
class AuditFileFolderItem(BaseModel):
|
||||
path: str = Field(description="Ruta relativa interna del archivo, para navegación.")
|
||||
display_name: str
|
||||
|
||||
|
||||
class AuditFileObjectItem(BaseModel):
|
||||
path: str = Field(description="Ruta relativa interna del archivo, para descarga.")
|
||||
display_name: str
|
||||
size: int
|
||||
last_modified: Optional[datetime] = None
|
||||
|
||||
|
||||
class AuditFileBrowserResponse(BaseModel):
|
||||
current_path: str = Field(default="")
|
||||
display_path: str = Field(default="")
|
||||
breadcrumbs: List[AuditFileBreadcrumb]
|
||||
folders: List[AuditFileFolderItem]
|
||||
files: List[AuditFileObjectItem]
|
||||
next_token: Optional[str] = None
|
||||
|
||||
@@ -1,16 +1,69 @@
|
||||
from typing import Dict, Any
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from core.s3_keys import (
|
||||
customs_broker_vu_certificate_key,
|
||||
customs_broker_vu_cove_key,
|
||||
customs_broker_vu_doda_certificate_key,
|
||||
customs_broker_vu_doda_cove_key,
|
||||
customs_broker_vu_doda_private_key_key,
|
||||
customs_broker_vu_private_key_key,
|
||||
)
|
||||
from core.security import get_current_user, get_tenant_from_token, validate_access_to_resource
|
||||
from core.storage_s3 import delete_object_if_exists, put_object_bytes
|
||||
|
||||
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from . import dto, services
|
||||
from ..layouts_csv.customs_brokers.routes import router as imports_router
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
MAX_VU_CER_KEY_BYTES = 5 * 1024 * 1024 # 5 MB
|
||||
MAX_COVE_BYTES = 15 * 1024 * 1024 # 15 MB (xml/zip)
|
||||
|
||||
|
||||
def _resolve_tenant_id_int(current_user: dict) -> int:
|
||||
tid = get_tenant_from_token(current_user)
|
||||
if tid is not None:
|
||||
return int(tid)
|
||||
raw = current_user.get("tenant_id")
|
||||
if isinstance(raw, list) and raw:
|
||||
raw = raw[0]
|
||||
if raw is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Tenant ID not found in user data",
|
||||
)
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invalid tenant ID in token",
|
||||
)
|
||||
|
||||
|
||||
def _remove_stored_vu_path(ref: Optional[str]) -> None:
|
||||
if not ref:
|
||||
return
|
||||
if ref.startswith("tenants/"):
|
||||
delete_object_if_exists(ref)
|
||||
elif os.path.isfile(ref):
|
||||
try:
|
||||
os.remove(ref)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# CSV import (mismo flujo que a76.imports: upload → scan → commit)
|
||||
router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"])
|
||||
|
||||
@@ -112,4 +165,246 @@ def update_customs_broker_personnel(
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Customs Broker Personnel not found"
|
||||
)
|
||||
return updated_personnel
|
||||
return updated_personnel
|
||||
|
||||
|
||||
@router.post(
|
||||
"/customs-brokers/{broker_key}/vu/upload",
|
||||
summary="Sube VU/DODA (CER, KEY, COVE) al bucket bajo tenants/.../customs_brokers/{id}/...",
|
||||
)
|
||||
async def upload_customs_broker_vu_file(
|
||||
broker_key: str,
|
||||
file_kind: str = Query(
|
||||
...,
|
||||
description=(
|
||||
"certificate (.cer), key (.key), cove (xml/zip/txt/pdf/json), "
|
||||
"doda_certificate (.cer), doda_key (.key), doda_cove (xml/zip/txt/pdf/json)"
|
||||
),
|
||||
),
|
||||
company_id: int = Query(..., description="Company ID"),
|
||||
file: UploadFile = File(...),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Persiste el archivo bajo la misma jerarquía que logos/avatares (tenant/company/...).
|
||||
Guarda la clave S3 o ruta local en:
|
||||
- VU: certificate_path, key_path, xml_files_path
|
||||
- DODA: doda_certificate_path, doda_key_path, doda_xml_files_path
|
||||
"""
|
||||
validate_access_to_resource(db, company_id, current_user)
|
||||
tenant_id = _resolve_tenant_id_int(current_user)
|
||||
|
||||
broker = services.CustomsBrokerService.get_by_id(db, broker_key, tenant_id, company_id)
|
||||
if not broker:
|
||||
raise HTTPException(status_code=404, detail="Customs Broker not found")
|
||||
|
||||
fk = file_kind.lower().strip()
|
||||
if fk not in (
|
||||
"certificate",
|
||||
"key",
|
||||
"cove",
|
||||
"doda_certificate",
|
||||
"doda_key",
|
||||
"doda_cove",
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"file_kind must be certificate, key, cove, "
|
||||
"doda_certificate, doda_key, or doda_cove"
|
||||
),
|
||||
)
|
||||
|
||||
content = await file.read()
|
||||
max_bytes = MAX_COVE_BYTES if fk in ("cove", "doda_cove") else MAX_VU_CER_KEY_BYTES
|
||||
if len(content) > max_bytes:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"File too large (max {max_bytes // (1024 * 1024)} MB)",
|
||||
)
|
||||
|
||||
file_ext = os.path.splitext(file.filename or "")[1].lower()
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
vu = services.CustomsBrokerVUService.ensure_vu_for_broker(db, broker)
|
||||
broker_id = broker.id
|
||||
|
||||
field_name: str
|
||||
stored: str
|
||||
|
||||
try:
|
||||
if settings.use_s3_object_storage:
|
||||
if fk == "certificate":
|
||||
if file_ext != ".cer":
|
||||
raise HTTPException(status_code=400, detail="certificate must be .cer")
|
||||
key = customs_broker_vu_certificate_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
ct = "application/x-x509-ca-cert"
|
||||
field_name = "certificate_path"
|
||||
elif fk == "key":
|
||||
if file_ext != ".key":
|
||||
raise HTTPException(status_code=400, detail="key must be .key")
|
||||
key = customs_broker_vu_private_key_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
ct = "application/pkcs8"
|
||||
field_name = "key_path"
|
||||
elif fk == "cove":
|
||||
key = customs_broker_vu_cove_key(
|
||||
tenant_id,
|
||||
company_id,
|
||||
broker_id,
|
||||
timestamp,
|
||||
file.filename or "cove.xml",
|
||||
)
|
||||
ct = (
|
||||
file.content_type
|
||||
or mimetypes.guess_type(file.filename or "")[0]
|
||||
or "application/octet-stream"
|
||||
)
|
||||
field_name = "xml_files_path"
|
||||
elif fk == "doda_certificate":
|
||||
if file_ext != ".cer":
|
||||
raise HTTPException(status_code=400, detail="doda_certificate must be .cer")
|
||||
key = customs_broker_vu_doda_certificate_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
ct = "application/x-x509-ca-cert"
|
||||
field_name = "doda_certificate_path"
|
||||
elif fk == "doda_key":
|
||||
if file_ext != ".key":
|
||||
raise HTTPException(status_code=400, detail="doda_key must be .key")
|
||||
key = customs_broker_vu_doda_private_key_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
ct = "application/pkcs8"
|
||||
field_name = "doda_key_path"
|
||||
else:
|
||||
key = customs_broker_vu_doda_cove_key(
|
||||
tenant_id,
|
||||
company_id,
|
||||
broker_id,
|
||||
timestamp,
|
||||
file.filename or "doda.xml",
|
||||
)
|
||||
ct = (
|
||||
file.content_type
|
||||
or mimetypes.guess_type(file.filename or "")[0]
|
||||
or "application/octet-stream"
|
||||
)
|
||||
field_name = "doda_xml_files_path"
|
||||
|
||||
old = getattr(vu, field_name)
|
||||
_remove_stored_vu_path(old)
|
||||
put_object_bytes(key, content, content_type=ct)
|
||||
logger.info(
|
||||
"Customs broker VU upload kind=%s key=%s bytes=%s",
|
||||
fk,
|
||||
key,
|
||||
len(content),
|
||||
)
|
||||
stored = key
|
||||
else:
|
||||
base = os.path.join(
|
||||
"uploads", "customs_brokers", str(company_id), str(broker_id)
|
||||
)
|
||||
if fk == "certificate":
|
||||
if file_ext != ".cer":
|
||||
raise HTTPException(status_code=400, detail="certificate must be .cer")
|
||||
key = customs_broker_vu_certificate_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
field_name = "certificate_path"
|
||||
subdir = "certificates"
|
||||
elif fk == "key":
|
||||
if file_ext != ".key":
|
||||
raise HTTPException(status_code=400, detail="key must be .key")
|
||||
key = customs_broker_vu_private_key_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
field_name = "key_path"
|
||||
subdir = "keys"
|
||||
elif fk == "cove":
|
||||
try:
|
||||
key = customs_broker_vu_cove_key(
|
||||
tenant_id,
|
||||
company_id,
|
||||
broker_id,
|
||||
timestamp,
|
||||
file.filename or "cove.xml",
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
field_name = "xml_files_path"
|
||||
subdir = "cove"
|
||||
elif fk == "doda_certificate":
|
||||
if file_ext != ".cer":
|
||||
raise HTTPException(status_code=400, detail="doda_certificate must be .cer")
|
||||
key = customs_broker_vu_doda_certificate_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
field_name = "doda_certificate_path"
|
||||
subdir = "doda/certificates"
|
||||
elif fk == "doda_key":
|
||||
if file_ext != ".key":
|
||||
raise HTTPException(status_code=400, detail="doda_key must be .key")
|
||||
key = customs_broker_vu_doda_private_key_key(
|
||||
tenant_id, company_id, broker_id, timestamp, file_ext
|
||||
)
|
||||
field_name = "doda_key_path"
|
||||
subdir = "doda/keys"
|
||||
else:
|
||||
try:
|
||||
key = customs_broker_vu_doda_cove_key(
|
||||
tenant_id,
|
||||
company_id,
|
||||
broker_id,
|
||||
timestamp,
|
||||
file.filename or "doda.xml",
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
field_name = "doda_xml_files_path"
|
||||
subdir = "doda/cove"
|
||||
|
||||
fname = key.rsplit("/", 1)[-1]
|
||||
dest_dir = os.path.join(base, subdir)
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
path = os.path.join(dest_dir, fname)
|
||||
old = getattr(vu, field_name)
|
||||
_remove_stored_vu_path(old)
|
||||
with open(path, "wb") as f:
|
||||
f.write(content)
|
||||
logger.info(
|
||||
"Customs broker VU upload kind=%s path=%s bytes=%s",
|
||||
fk,
|
||||
path,
|
||||
len(content),
|
||||
)
|
||||
stored = path
|
||||
|
||||
setattr(vu, field_name, stored)
|
||||
db.add(vu)
|
||||
db.commit()
|
||||
db.refresh(vu)
|
||||
except HTTPException:
|
||||
db.rollback()
|
||||
raise
|
||||
except ValueError as e:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error saving file: {str(e)}"
|
||||
) from e
|
||||
|
||||
return {
|
||||
"message": "File uploaded successfully",
|
||||
"file_kind": fk,
|
||||
"field": field_name,
|
||||
"path": stored,
|
||||
"broker_key": broker_key,
|
||||
"company_id": company_id,
|
||||
}
|
||||
@@ -160,6 +160,26 @@ class CustomsBrokerVUService:
|
||||
db.commit()
|
||||
return vu
|
||||
|
||||
@staticmethod
|
||||
def ensure_vu_for_broker(db: Session, broker: models.CustomsBroker) -> models.CustomsBrokerVU:
|
||||
"""Crea fila VU vacía si no existe (p. ej. antes de subir CER/KEY/COVE al bucket)."""
|
||||
vu = (
|
||||
db.query(models.CustomsBrokerVU)
|
||||
.filter(models.CustomsBrokerVU.customs_broker_id == broker.id)
|
||||
.first()
|
||||
)
|
||||
if vu:
|
||||
return vu
|
||||
vu = models.CustomsBrokerVU(
|
||||
customs_broker_id=broker.id,
|
||||
tenant_id=broker.tenant_id,
|
||||
company_id=broker.company_id,
|
||||
)
|
||||
db.add(vu)
|
||||
db.commit()
|
||||
db.refresh(vu)
|
||||
return vu
|
||||
|
||||
|
||||
class CustomsBrokerPersonnelService:
|
||||
@staticmethod
|
||||
|
||||
@@ -57,7 +57,7 @@ class CompanyCreateDTO(BaseModel):
|
||||
)
|
||||
|
||||
# Configuration
|
||||
logo: Optional[str] = Field(None, max_length=255, description="Company logo")
|
||||
logo: Optional[str] = Field(None, max_length=512, description="Company logo (path local o clave S3)")
|
||||
has_express_line: Optional[bool] = Field(None, description="Has express line")
|
||||
order_format_type: Optional[str] = Field(
|
||||
None, max_length=19, description="Order format type"
|
||||
|
||||
@@ -60,7 +60,7 @@ class Company(Base, TimestampMixin):
|
||||
position: Mapped[Optional[str]] = mapped_column(String(30))
|
||||
|
||||
# Configuración básica
|
||||
logo: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
logo: Mapped[Optional[str]] = mapped_column(String(512))
|
||||
has_express_line: Mapped[Optional[bool]] = mapped_column(Boolean, default=False, server_default="false")
|
||||
order_format_type: Mapped[Optional[str]] = mapped_column(String(19))
|
||||
is_service_company: Mapped[Optional[bool]] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
@@ -2,20 +2,23 @@
|
||||
Rutas para gestión de empresa
|
||||
"""
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
from typing import List, Optional
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, File, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from core.s3_keys import company_certificate_key, company_logo_key
|
||||
from core.storage_s3 import delete_object_if_exists, get_object_bytes, put_object_bytes
|
||||
from core.security import get_current_user, get_tenant_from_token, validate_access_to_resource
|
||||
from .....common.tenant_crud_routes import TenantCRUDRoutes
|
||||
from .dto import CompanyCreateDTO, CompanyResponseDTO, CompanyUpdateDTO
|
||||
from .models import Company
|
||||
@@ -26,6 +29,44 @@ UPLOAD_DIR = "uploads/companies"
|
||||
ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
|
||||
MAX_FILE_SIZE = 5 * 1024 * 1024 # 5MB
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_tenant_id_int(current_user: dict) -> int:
|
||||
"""Misma lógica que validate_access_to_resource: entero estable para BD y claves S3."""
|
||||
tid = get_tenant_from_token(current_user)
|
||||
if tid is not None:
|
||||
return int(tid)
|
||||
raw = current_user.get("tenant_id")
|
||||
if isinstance(raw, list) and raw:
|
||||
raw = raw[0]
|
||||
if raw is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Tenant ID not found in user data",
|
||||
)
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invalid tenant ID in token",
|
||||
)
|
||||
|
||||
|
||||
def _is_s3_object_key(ref: Optional[str]) -> bool:
|
||||
return bool(ref and ref.startswith("tenants/"))
|
||||
|
||||
|
||||
def _remove_stored_file(ref: str) -> None:
|
||||
if _is_s3_object_key(ref):
|
||||
delete_object_if_exists(ref)
|
||||
elif ref and os.path.isfile(ref):
|
||||
try:
|
||||
os.remove(ref)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Main router that includes base CRUD
|
||||
router = APIRouter(prefix="/company")
|
||||
|
||||
@@ -219,14 +260,20 @@ async def get_company_logo_image(
|
||||
if not company or not company.logo:
|
||||
raise HTTPException(status_code=404, detail="Logo not found")
|
||||
|
||||
if _is_s3_object_key(company.logo):
|
||||
try:
|
||||
data = get_object_bytes(company.logo)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=404, detail="Logo file not found on server")
|
||||
media = mimetypes.guess_type(company.logo)[0] or "image/jpeg"
|
||||
return Response(content=data, media_type=media)
|
||||
|
||||
file_path = Path(company.logo)
|
||||
if not file_path.exists():
|
||||
# Fallback for old paths or moved files
|
||||
# Check if it exists in the 'standard' location even if DB thinks otherwise
|
||||
standard_path = Path(f"app_data/logos/{company_id}") / file_path.name
|
||||
if standard_path.exists():
|
||||
return FileResponse(standard_path)
|
||||
|
||||
|
||||
raise HTTPException(status_code=404, detail="Logo file not found on server")
|
||||
|
||||
return FileResponse(file_path)
|
||||
@@ -272,12 +319,7 @@ async def upload_company_logo(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Upload a logo for a company"""
|
||||
tenant_id = current_user.get("tenant_id")
|
||||
if not tenant_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Tenant ID not found in user data",
|
||||
)
|
||||
tenant_id = _resolve_tenant_id_int(current_user)
|
||||
|
||||
# Validar que la empresa existe
|
||||
company = CompanyService.get_by_id(db, company_id, tenant_id, 0)
|
||||
@@ -303,35 +345,34 @@ async def upload_company_logo(
|
||||
detail=f"File too large. Maximum size: {MAX_FILE_SIZE / 1024 / 1024}MB",
|
||||
)
|
||||
|
||||
# Crear directorio si no existe
|
||||
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
||||
|
||||
# Eliminar logo anterior si existe
|
||||
if company.logo:
|
||||
old_logo_path = company.logo
|
||||
if os.path.exists(old_logo_path):
|
||||
try:
|
||||
os.remove(old_logo_path)
|
||||
except Exception:
|
||||
pass # No es crítico si falla
|
||||
_remove_stored_file(company.logo)
|
||||
|
||||
# Generar nombre único
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"company_{company_id}_{timestamp}{file_ext}"
|
||||
file_path = os.path.join(UPLOAD_DIR, filename)
|
||||
filename = f"logo_{company_id}_{timestamp}{file_ext}"
|
||||
|
||||
# Guardar archivo
|
||||
try:
|
||||
await file.seek(0)
|
||||
with open(file_path, "wb") as buffer:
|
||||
shutil.copyfileobj(file.file, buffer)
|
||||
if settings.use_s3_object_storage:
|
||||
ct = mimetypes.guess_type(filename)[0] or "image/jpeg"
|
||||
key = company_logo_key(tenant_id, company_id, filename)
|
||||
put_object_bytes(key, content, content_type=ct)
|
||||
logger.info(
|
||||
"Company logo stored in S3 key=%s bytes=%s",
|
||||
key,
|
||||
len(content),
|
||||
)
|
||||
file_path = key
|
||||
else:
|
||||
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
||||
file_path = os.path.join(UPLOAD_DIR, filename)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(content)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error saving file: {str(e)}",
|
||||
)
|
||||
|
||||
# Actualizar la empresa con la ruta del logo
|
||||
update_data = CompanyUpdateDTO(logo=file_path)
|
||||
service = CompanyService(db)
|
||||
updated_company = service.update(db, company_id, tenant_id, 0, update_data)
|
||||
@@ -359,12 +400,7 @@ async def upload_company_certificate(
|
||||
Upload a certificate for a company
|
||||
certificate_type: fiel_cer, fiel_key, cfdi_cert_cer, cfdi_cert_key, cancel_cer, cancel_key
|
||||
"""
|
||||
tenant_id = current_user.get("tenant_id")
|
||||
if not tenant_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Tenant ID not found in user data",
|
||||
)
|
||||
tenant_id = _resolve_tenant_id_int(current_user)
|
||||
|
||||
# Validar que la empresa existe
|
||||
service = CompanyService(db)
|
||||
@@ -410,27 +446,38 @@ async def upload_company_certificate(
|
||||
detail=f"File too large. Maximum size: {MAX_FILE_SIZE / 1024 / 1024}MB",
|
||||
)
|
||||
|
||||
# Crear directorio si no existe
|
||||
certs_dir = os.path.join(UPLOAD_DIR, str(company_id), "certificates")
|
||||
os.makedirs(certs_dir, exist_ok=True)
|
||||
|
||||
# Generar nombre único
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"{certificate_type}_{timestamp}{file_ext}"
|
||||
file_path = os.path.join(certs_dir, filename)
|
||||
|
||||
# Guardar archivo
|
||||
try:
|
||||
await file.seek(0)
|
||||
with open(file_path, "wb") as buffer:
|
||||
shutil.copyfileobj(file.file, buffer)
|
||||
if settings.use_s3_object_storage:
|
||||
key = company_certificate_key(
|
||||
tenant_id, company_id, certificate_type, timestamp, file_ext
|
||||
)
|
||||
ct = (
|
||||
"application/x-x509-ca-cert"
|
||||
if file_ext == ".cer"
|
||||
else "application/pkcs8"
|
||||
)
|
||||
put_object_bytes(key, content, content_type=ct)
|
||||
logger.info(
|
||||
"Company certificate stored in S3 key=%s bytes=%s",
|
||||
key,
|
||||
len(content),
|
||||
)
|
||||
file_path = key
|
||||
else:
|
||||
certs_dir = os.path.join(UPLOAD_DIR, str(company_id), "certificates")
|
||||
os.makedirs(certs_dir, exist_ok=True)
|
||||
filename = f"{certificate_type}_{timestamp}{file_ext}"
|
||||
file_path = os.path.join(certs_dir, filename)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(content)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error saving file: {str(e)}",
|
||||
)
|
||||
|
||||
# Actualizar la base de datos
|
||||
service.upload_certificate(company_id, certificate_type, file_path, tenant_id)
|
||||
|
||||
return {
|
||||
|
||||
@@ -10,6 +10,10 @@ from fastapi import HTTPException
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.config import settings
|
||||
from core.s3_keys import tenant_company_prefix
|
||||
from core.storage_s3 import delete_objects_with_prefix
|
||||
|
||||
from .dto import CompanyCreateDTO, CompanyResponseDTO, CompanyUpdateDTO
|
||||
from .models import Company
|
||||
from ...audit_log.services.service import AuditService
|
||||
@@ -594,9 +598,24 @@ class CompanyService:
|
||||
# ----------------------
|
||||
|
||||
try:
|
||||
# 1) Borrado lógico en base de datos
|
||||
company.deleted_at = datetime.utcnow()
|
||||
|
||||
db.flush()
|
||||
|
||||
# 2) Limpieza de objetos S3/MinIO asociados a la compañía
|
||||
if settings.use_s3_object_storage:
|
||||
try:
|
||||
prefix = tenant_company_prefix(tenant_id, company_id)
|
||||
delete_objects_with_prefix(prefix)
|
||||
except Exception as e:
|
||||
# No bloquear la eliminación lógica si falla la limpieza de objetos
|
||||
logger.error(
|
||||
"Error deleting S3 objects for company %s (tenant %s): %s",
|
||||
company_id,
|
||||
tenant_id,
|
||||
e,
|
||||
)
|
||||
|
||||
db.commit()
|
||||
|
||||
# --- Audit Log ---
|
||||
@@ -678,6 +697,11 @@ class CompanyService:
|
||||
|
||||
try:
|
||||
if target_cert:
|
||||
old_path = getattr(target_cert, field_to_update, None)
|
||||
if old_path and str(old_path).startswith("tenants/"):
|
||||
from core.storage_s3 import delete_object_if_exists
|
||||
|
||||
delete_object_if_exists(str(old_path))
|
||||
# Si existe, actualizamos
|
||||
setattr(target_cert, field_to_update, file_path)
|
||||
else:
|
||||
|
||||
@@ -13,12 +13,19 @@ from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRat
|
||||
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
|
||||
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
|
||||
from api.v1.modules.a76.transportation.transporters.models import Transporter
|
||||
from api.v1.modules.a76.transportation.vehicles.models import Vehicle
|
||||
from api.v1.modules.a76.transportation.trailers.models import Trailer
|
||||
from api.v1.modules.a76.transportation.drivers.models import Driver
|
||||
from api.v1.modules.a76.manifests.manifest.models import Manifest
|
||||
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
|
||||
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
|
||||
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
|
||||
|
||||
|
||||
def _logistics_str_nonempty(value) -> bool:
|
||||
return value is not None and str(value).strip() != ""
|
||||
|
||||
|
||||
def _normalize_invoice_currency_value(value) -> str:
|
||||
"""Lowercase currency code (foreign/local/manual) for comparisons."""
|
||||
if value is None or value == "":
|
||||
@@ -344,25 +351,40 @@ def validate_common(
|
||||
code="INVALID_VALUE",
|
||||
value=invoice.compliance_mx.remesa,
|
||||
)
|
||||
else:
|
||||
# Misma remesa puede repetirse en distintos pedimentos; solo debe ser única
|
||||
# por (pedimento_id, empresa, tenant). Creación: existing_invoice es None.
|
||||
current_invoice_id = None
|
||||
if existing_invoice is not None:
|
||||
current_invoice_id = existing_invoice.id
|
||||
else:
|
||||
current_invoice_id = getattr(invoice, "id", None)
|
||||
|
||||
duplicated_remesa = (
|
||||
db.query(InvoiceComplianceMx)
|
||||
.filter(
|
||||
InvoiceComplianceMx.remesa == invoice.compliance_mx.remesa,
|
||||
InvoiceComplianceMx.tenant_id == tenant_id,
|
||||
InvoiceComplianceMx.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if duplicated_remesa and hasattr(invoice, "id"):
|
||||
if invoice.id != duplicated_remesa.invoice_id:
|
||||
errors.add_error(
|
||||
field="compliance_mx.remesa",
|
||||
message="El valor de Remesa ya está asociado a otro Pedimento.",
|
||||
solution=["Proporciona un valor único para Remesa"],
|
||||
code="DUPLICATE_VALUE",
|
||||
value=invoice.compliance_mx.remesa,
|
||||
duplicated_remesa = (
|
||||
db.query(InvoiceComplianceMx)
|
||||
.filter(
|
||||
InvoiceComplianceMx.remesa == invoice.compliance_mx.remesa,
|
||||
InvoiceComplianceMx.pedimento_id
|
||||
== invoice.compliance_mx.pedimento_id,
|
||||
InvoiceComplianceMx.tenant_id == tenant_id,
|
||||
InvoiceComplianceMx.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if duplicated_remesa and (
|
||||
current_invoice_id is None
|
||||
or duplicated_remesa.invoice_id != current_invoice_id
|
||||
):
|
||||
errors.add_error(
|
||||
field="compliance_mx.remesa",
|
||||
message=(
|
||||
"El valor de Remesa ya está registrado para este mismo Pedimento "
|
||||
"en otra factura."
|
||||
),
|
||||
solution=["Usa otra remesa o revisa la factura que ya la tiene capturada"],
|
||||
code="DUPLICATE_VALUE",
|
||||
value=invoice.compliance_mx.remesa,
|
||||
)
|
||||
|
||||
# Financials checks (if provided)
|
||||
if invoice.financials:
|
||||
@@ -511,16 +533,31 @@ def validate_common(
|
||||
)
|
||||
|
||||
if invoice.logistics:
|
||||
carrier_exists = None
|
||||
if invoice.logistics.carrier_id:
|
||||
carrier_exists = (
|
||||
db.query(Transporter)
|
||||
.filter(
|
||||
Transporter.id == invoice.logistics.carrier_id,
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
# `carrier_id` is the frontend/export key (string). Prefer validating by the internal int
|
||||
# when present, otherwise validate by `transporter_key`.
|
||||
if getattr(invoice.logistics, "carrier_int_id", None) is not None:
|
||||
carrier_exists = (
|
||||
db.query(Transporter)
|
||||
.filter(
|
||||
Transporter.transporter_id
|
||||
== invoice.logistics.carrier_int_id,
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
carrier_exists = (
|
||||
db.query(Transporter)
|
||||
.filter(
|
||||
Transporter.transporter_key == invoice.logistics.carrier_id,
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not carrier_exists:
|
||||
errors.add_error(
|
||||
field="logistics.carrier_id",
|
||||
@@ -528,8 +565,36 @@ def validate_common(
|
||||
solution=["Verifica el ID del Transportista", "Revisa el catálogo"],
|
||||
code="NOT_FOUND",
|
||||
value=invoice.logistics.carrier_id,
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
if carrier_exists and _logistics_str_nonempty(
|
||||
getattr(invoice.logistics, "driver_name", None)
|
||||
):
|
||||
driver_row = (
|
||||
db.query(Driver)
|
||||
.filter(
|
||||
Driver.transporter_key == str(invoice.logistics.carrier_id).strip(),
|
||||
Driver.driver_name
|
||||
== str(invoice.logistics.driver_name).strip(),
|
||||
Driver.tenant_id == tenant_id,
|
||||
Driver.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not driver_row:
|
||||
errors.add_error(
|
||||
field="logistics.driver_name",
|
||||
message=(
|
||||
"El conductor no existe en el catálogo de conductores "
|
||||
"para el transportista indicado."
|
||||
),
|
||||
solution=[
|
||||
"Registra el conductor en el catálogo o elige uno de la lista"
|
||||
],
|
||||
code="NOT_FOUND",
|
||||
value=invoice.logistics.driver_name,
|
||||
)
|
||||
|
||||
if invoice.logistics.transport_type not in [t.value for t in TransportType]:
|
||||
errors.add_error(
|
||||
field="logistics.transport_type",
|
||||
@@ -541,31 +606,138 @@ def validate_common(
|
||||
value=invoice.logistics.transport_type,
|
||||
)
|
||||
else:
|
||||
if (
|
||||
invoice.logistics.transport_type == "none"
|
||||
and invoice.logistics.transport_num
|
||||
):
|
||||
errors.add_error(
|
||||
field="logistics.transport_num",
|
||||
message="El Número de Transporte no debe proporcionarse cuando el Tipo de Transporte es 'none'.",
|
||||
solution=[
|
||||
"Elimina el Número de Transporte o selecciona un Tipo de Transporte válido"
|
||||
],
|
||||
code="INVALID_VALUE",
|
||||
value=invoice.logistics.transport_num,
|
||||
)
|
||||
else:
|
||||
if (
|
||||
not invoice.logistics.transport_num
|
||||
and invoice.logistics.transport_type != "none"
|
||||
):
|
||||
errors.add_error(
|
||||
field="logistics.transport_num",
|
||||
message="El Número de Transporte es obligatorio cuando se proporciona un Tipo de Transporte distinto de 'none'.",
|
||||
solution=["Proporciona un Número de Transporte válido"],
|
||||
code="REQUIRED_FIELD",
|
||||
value=invoice.logistics.transport_num,
|
||||
# Vehículo: solo catálogo si viene informado (cualquier tipo de transporte).
|
||||
tid = getattr(invoice.logistics, "transport_int_id", None)
|
||||
tcode = invoice.logistics.transport_id
|
||||
if tid is not None:
|
||||
v_by_int = (
|
||||
db.query(Vehicle)
|
||||
.filter(
|
||||
Vehicle.vehicle_id == tid,
|
||||
Vehicle.tenant_id == tenant_id,
|
||||
Vehicle.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not v_by_int:
|
||||
errors.add_error(
|
||||
field="logistics.transport_int_id",
|
||||
message="El Vehículo (transporte) no existe en el catálogo.",
|
||||
solution=[
|
||||
"Verifica el ID interno del vehículo o usa la clave de transporte"
|
||||
],
|
||||
code="NOT_FOUND",
|
||||
value=tid,
|
||||
)
|
||||
elif _logistics_str_nonempty(tcode):
|
||||
v_by_key = (
|
||||
db.query(Vehicle)
|
||||
.filter(
|
||||
func.upper(Vehicle.vehicle_key)
|
||||
== str(tcode).strip().upper(),
|
||||
Vehicle.tenant_id == tenant_id,
|
||||
Vehicle.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not v_by_key:
|
||||
errors.add_error(
|
||||
field="logistics.transport_id",
|
||||
message="El Vehículo (clave transporte) no existe en el catálogo.",
|
||||
solution=["Verifica la clave en el catálogo de vehículos"],
|
||||
code="NOT_FOUND",
|
||||
value=tcode,
|
||||
)
|
||||
|
||||
# Remolque: obligatorio si tipo != 'none'; con 'none' solo se valida catálogo si hay dato.
|
||||
tr_int = getattr(invoice.logistics, "trailer_int_id", None)
|
||||
tr_num = invoice.logistics.trailer_num
|
||||
if invoice.logistics.transport_type != "none":
|
||||
if tr_int is not None:
|
||||
tr = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
Trailer.trailer_id == tr_int,
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not tr:
|
||||
errors.add_error(
|
||||
field="logistics.trailer_int_id",
|
||||
message="El Remolque no existe en el catálogo.",
|
||||
solution=["Verifica el ID interno del remolque"],
|
||||
code="NOT_FOUND",
|
||||
value=tr_int,
|
||||
)
|
||||
elif not _logistics_str_nonempty(tr_num):
|
||||
errors.add_error(
|
||||
field="logistics.trailer_num",
|
||||
message=(
|
||||
"El Remolque es obligatorio cuando el Tipo de Transporte es distinto de 'none'."
|
||||
),
|
||||
solution=["Selecciona un remolque del catálogo"],
|
||||
code="REQUIRED_FIELD",
|
||||
value=tr_num,
|
||||
)
|
||||
else:
|
||||
tr = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
func.upper(Trailer.trailer_number)
|
||||
== str(tr_num).strip().upper(),
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not tr:
|
||||
errors.add_error(
|
||||
field="logistics.trailer_num",
|
||||
message="El Remolque no existe en el catálogo.",
|
||||
solution=["Verifica el número de remolque"],
|
||||
code="NOT_FOUND",
|
||||
value=tr_num,
|
||||
)
|
||||
else:
|
||||
if tr_int is not None:
|
||||
tr = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
Trailer.trailer_id == tr_int,
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not tr:
|
||||
errors.add_error(
|
||||
field="logistics.trailer_int_id",
|
||||
message="El Remolque no existe en el catálogo.",
|
||||
solution=["Verifica el ID interno del remolque"],
|
||||
code="NOT_FOUND",
|
||||
value=tr_int,
|
||||
)
|
||||
elif _logistics_str_nonempty(tr_num):
|
||||
tr = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
func.upper(Trailer.trailer_number)
|
||||
== str(tr_num).strip().upper(),
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not tr:
|
||||
errors.add_error(
|
||||
field="logistics.trailer_num",
|
||||
message="El Remolque no existe en el catálogo.",
|
||||
solution=["Verifica el número de remolque"],
|
||||
code="NOT_FOUND",
|
||||
value=tr_num,
|
||||
)
|
||||
|
||||
if invoice.financials:
|
||||
submitted = invoice.financials.currency
|
||||
|
||||
@@ -563,9 +563,15 @@ class InvoiceLogistics(Base, TenantScopedMixin, TimestampMixin):
|
||||
carrier_id: Mapped[Optional[str]] = mapped_column(
|
||||
String(10)
|
||||
) # TRANSPORTISTA / Transportista
|
||||
carrier_int_id: Mapped[Optional[int]] = mapped_column(
|
||||
BigInteger, ForeignKey("a76.transporter.transporter_id"), nullable=True
|
||||
) # Internal surrogate ID for Transporter (by transporter_key)
|
||||
transport_id: Mapped[Optional[str]] = mapped_column(
|
||||
String(10)
|
||||
) # NUMTRAILER / Transportista
|
||||
transport_int_id: Mapped[Optional[int]] = mapped_column(
|
||||
BigInteger, ForeignKey("a76.vehicle.vehicle_id"), nullable=True
|
||||
) # Internal surrogate ID for Vehicle (by vehicle_key)
|
||||
transport_us_id: Mapped[Optional[str]] = mapped_column(
|
||||
String(10)
|
||||
) # TRANSPORTISTAAME / Transportista americano
|
||||
@@ -601,6 +607,9 @@ class InvoiceLogistics(Base, TenantScopedMixin, TimestampMixin):
|
||||
trailer_num: Mapped[Optional[str]] = mapped_column(
|
||||
String(20)
|
||||
) # NUMTRAILER / Número de trailer
|
||||
trailer_int_id: Mapped[Optional[int]] = mapped_column(
|
||||
BigInteger, ForeignKey("a76.trailer.trailer_id"), nullable=True
|
||||
) # Internal surrogate ID for Trailer (by trailer_number)
|
||||
seal_number: Mapped[Optional[str]] = mapped_column(
|
||||
String(15)
|
||||
) # PRECINTO / Precinto
|
||||
|
||||
@@ -95,8 +95,11 @@ def list_invoices(
|
||||
invoice_type: str = Query(None, description="Filter by invoice type"),
|
||||
manifest_number: str = Query(None, description="Filter by manifest number"),
|
||||
pedimento: str = Query(None, description="Filter by pedimento"),
|
||||
invoice_number: str = Query(None, description="Filter by invoice number"),
|
||||
project_number: str = Query(None, description="Filter by project number"),
|
||||
year: str = Query(None, description="Filter by year"),
|
||||
sort_by: Optional[str] = Query(None, description="Column to sort by"),
|
||||
sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"),
|
||||
sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
@@ -109,12 +112,14 @@ def list_invoices(
|
||||
|
||||
skip = (page - 1) * page_size
|
||||
filters = {
|
||||
"invoice_number": search,
|
||||
"invoice_number": invoice_number or search,
|
||||
"status": status,
|
||||
"operation_type": operation_type,
|
||||
"invoice_type": invoice_type,
|
||||
"manifest_number": manifest_number,
|
||||
"pedimento": pedimento,
|
||||
"project_number": project_number,
|
||||
"year": year,
|
||||
}
|
||||
|
||||
# Remove None values
|
||||
|
||||
@@ -299,9 +299,14 @@ class InvoiceFinancialsBase(BaseModel):
|
||||
|
||||
class InvoiceLogisticsBase(BaseModel):
|
||||
"""Base fields for Logistics"""
|
||||
|
||||
carrier_id: Optional[str] = Field(None, max_length=10, description="Carrier ID")
|
||||
carrier_id: Optional[str] = Field(None, max_length=10, description="Carrier code (front-end key)")
|
||||
carrier_int_id: Optional[int] = Field(
|
||||
None, description="Internal carrier integer ID (mapped from carrier_id key)"
|
||||
)
|
||||
transport_id: Optional[str] = Field(None, max_length=10, description="Transport ID")
|
||||
transport_int_id: Optional[int] = Field(
|
||||
None, description="Internal vehicle integer ID (mapped from transport_id key)"
|
||||
)
|
||||
transport_us_id: Optional[str] = Field(
|
||||
None, max_length=10, description="US transport ID"
|
||||
)
|
||||
@@ -329,6 +334,9 @@ class InvoiceLogisticsBase(BaseModel):
|
||||
trailer_num: Optional[str] = Field(
|
||||
None, max_length=20, description="Trailer number"
|
||||
)
|
||||
trailer_int_id: Optional[int] = Field(
|
||||
None, description="Internal trailer integer ID (mapped from trailer_num key)"
|
||||
)
|
||||
seal_number: Optional[str] = Field(None, max_length=15, description="Seal number")
|
||||
guide_number: Optional[str] = Field(None, max_length=20, description="Guide number")
|
||||
bill_number: Optional[str] = Field(None, max_length=15, description="Bill number")
|
||||
|
||||
@@ -11,10 +11,142 @@ from .exports.validators.create import validate_create as validate_create_export
|
||||
from .exports.validators.update import validate_update as validate_update_export
|
||||
from .common.common_validators import invoice_exists
|
||||
from api.v1.modules.a76.items.models import LineItem
|
||||
|
||||
from . import models, schemas
|
||||
|
||||
|
||||
def _autofill_transport_int_ids(
|
||||
db: Session,
|
||||
logistics_target,
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
) -> None:
|
||||
"""
|
||||
Ensures invoice logistics internal int IDs are populated when catalog rows exist.
|
||||
|
||||
String keys (carrier_id, transport_id, trailer_num) are matched to existing catalog
|
||||
rows for the tenant/company; no placeholder rows are created. Invalid references must
|
||||
be rejected by validate_common before persist.
|
||||
"""
|
||||
|
||||
# Local imports to avoid circular dependencies.
|
||||
from api.v1.modules.a76.transportation.transporters.models import Transporter
|
||||
from api.v1.modules.a76.transportation.vehicles.models import Vehicle
|
||||
from api.v1.modules.a76.transportation.trailers.models import Trailer
|
||||
|
||||
def _get(field: str):
|
||||
if isinstance(logistics_target, dict):
|
||||
return logistics_target.get(field)
|
||||
return getattr(logistics_target, field, None)
|
||||
|
||||
def _set(field: str, value):
|
||||
if isinstance(logistics_target, dict):
|
||||
logistics_target[field] = value
|
||||
else:
|
||||
setattr(logistics_target, field, value)
|
||||
|
||||
carrier_code = _get("carrier_id")
|
||||
carrier_int_id = _get("carrier_int_id")
|
||||
if carrier_code and carrier_int_id is None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(
|
||||
Transporter.transporter_key == carrier_code,
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if transporter_obj:
|
||||
_set("carrier_int_id", transporter_obj.transporter_id)
|
||||
|
||||
transport_code = _get("transport_id")
|
||||
transport_int_id = _get("transport_int_id")
|
||||
if transport_code and transport_int_id is None:
|
||||
vehicle_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(
|
||||
func.upper(Vehicle.vehicle_key) == str(transport_code).strip().upper(),
|
||||
Vehicle.tenant_id == tenant_id,
|
||||
Vehicle.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if vehicle_obj:
|
||||
_set("transport_int_id", vehicle_obj.vehicle_id)
|
||||
|
||||
trailer_code = _get("trailer_num")
|
||||
trailer_int_id = _get("trailer_int_id")
|
||||
if trailer_code and trailer_int_id is None:
|
||||
trailer_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
func.upper(Trailer.trailer_number)
|
||||
== str(trailer_code).strip().upper(),
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if trailer_obj:
|
||||
_set("trailer_int_id", trailer_obj.trailer_id)
|
||||
|
||||
# Placa: si no vino valor, tomar del vehículo y si no del remolque (catálogo)
|
||||
lp = _get("license_plate")
|
||||
if lp is None or str(lp).strip() == "":
|
||||
veh = None
|
||||
tv_id = _get("transport_int_id")
|
||||
tv_code = _get("transport_id")
|
||||
if tv_id is not None:
|
||||
veh = (
|
||||
db.query(Vehicle)
|
||||
.filter(
|
||||
Vehicle.vehicle_id == tv_id,
|
||||
Vehicle.tenant_id == tenant_id,
|
||||
Vehicle.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
elif tv_code:
|
||||
veh = (
|
||||
db.query(Vehicle)
|
||||
.filter(
|
||||
func.upper(Vehicle.vehicle_key) == str(tv_code).strip().upper(),
|
||||
Vehicle.tenant_id == tenant_id,
|
||||
Vehicle.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if veh and getattr(veh, "plate_number", None):
|
||||
_set("license_plate", veh.plate_number)
|
||||
else:
|
||||
tr = None
|
||||
tr_id = _get("trailer_int_id")
|
||||
tr_code = _get("trailer_num")
|
||||
if tr_id is not None:
|
||||
tr = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
Trailer.trailer_id == tr_id,
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
elif tr_code:
|
||||
tr = (
|
||||
db.query(Trailer)
|
||||
.filter(
|
||||
func.upper(Trailer.trailer_number)
|
||||
== str(tr_code).strip().upper(),
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if tr and getattr(tr, "plate_number", None):
|
||||
_set("license_plate", tr.plate_number)
|
||||
|
||||
|
||||
def _get_current_username() -> str:
|
||||
"""Helper to get current username from context or fallback to System"""
|
||||
try:
|
||||
@@ -126,46 +258,70 @@ class InvoiceService:
|
||||
|
||||
# Apply filters if provided
|
||||
if filters:
|
||||
# Join compliance_mx if needed for filters
|
||||
needs_compliance_join = any(k in filters for k in ["pedimento", "manifest_number"])
|
||||
if needs_compliance_join:
|
||||
query = query.join(models.InvoiceComplianceMx)
|
||||
|
||||
if filters.get("status") is not None:
|
||||
status = models.InvoiceStatus.PROCESSED if filters["status"] == True else models.InvoiceStatus.PENDING
|
||||
|
||||
status = (
|
||||
models.InvoiceStatus.PROCESSED
|
||||
if filters["status"] == True
|
||||
else models.InvoiceStatus.PENDING
|
||||
)
|
||||
query = query.filter(models.InvoiceHeader.status == status)
|
||||
|
||||
if filters.get("operation_type"):
|
||||
ot = filters["operation_type"]
|
||||
ot_val = ot.value if hasattr(ot, "value") else ot
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.operation_type == ot_val
|
||||
)
|
||||
query = query.filter(models.InvoiceHeader.operation_type == ot_val)
|
||||
|
||||
if filters.get("invoice_type"):
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.invoice_type == filters["invoice_type"]
|
||||
)
|
||||
|
||||
if filters.get("invoice_number"):
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.invoice_number.ilike(
|
||||
f"%{filters['invoice_number']}%"
|
||||
)
|
||||
)
|
||||
if filters.get("pedimento"):
|
||||
query = query.join(models.InvoiceComplianceMx).filter(
|
||||
models.InvoiceComplianceMx.pedimento.ilike(
|
||||
f"%{filters['pedimento']}%"
|
||||
|
||||
if filters.get("project_number"):
|
||||
query = query.filter(
|
||||
models.InvoiceHeader.project_number.ilike(
|
||||
f"%{filters['project_number']}%"
|
||||
)
|
||||
)
|
||||
|
||||
if filters.get("year"):
|
||||
try:
|
||||
year_val = int(filters["year"])
|
||||
query = query.filter(
|
||||
func.extract("year", models.InvoiceHeader.invoice_date) == year_val
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if filters.get("pedimento"):
|
||||
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
|
||||
query = query.join(models.InvoiceComplianceMx.pedimento).filter(
|
||||
Pedimentos.pedimento_number.ilike(f"%{filters['pedimento']}%")
|
||||
)
|
||||
|
||||
if filters.get("manifest_number"):
|
||||
query = query.filter(
|
||||
models.InvoiceComplianceMx.manifest_number.ilike(
|
||||
f"%{filters['manifest_number']}%"
|
||||
)
|
||||
)
|
||||
|
||||
# Special case for exports: exclude REPAR if no invoice_type specified
|
||||
ot_exp = filters.get("operation_type")
|
||||
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
|
||||
if not filters.get("invoice_type") and ot_exp_val == "exp":
|
||||
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
|
||||
|
||||
if filters.get("manifest_number"):
|
||||
# Avoid duplicate joins if pedimento filter was also applied (though rare in this context)
|
||||
# For safety, we can just use the relationship attribute directly if mapped,
|
||||
# but explicit join is clearer given the previous pattern.
|
||||
# Assuming SQLAlchemy handles the join overlap or we just accept it for now.
|
||||
# To be safe and consistent with previous 'pedimento' block:
|
||||
query = query.join(models.InvoiceComplianceMx).filter(
|
||||
models.InvoiceComplianceMx.manifest_number.ilike(f"%{filters['manifest_number']}%")
|
||||
)
|
||||
query = query.filter(models.InvoiceHeader.invoice_type != "REPAR")
|
||||
|
||||
# Apply sorting
|
||||
if sort_by:
|
||||
@@ -298,6 +454,12 @@ class InvoiceService:
|
||||
logistics_dict["invoice_id"] = new_invoice.id
|
||||
logistics_dict["tenant_id"] = tenant_id
|
||||
logistics_dict["company_id"] = company_id
|
||||
_autofill_transport_int_ids(
|
||||
db,
|
||||
logistics_dict,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
)
|
||||
new_logistics = models.InvoiceLogistics(**logistics_dict)
|
||||
db.add(new_logistics)
|
||||
|
||||
@@ -460,11 +622,23 @@ class InvoiceService:
|
||||
if value == "":
|
||||
value = None
|
||||
setattr(invoice.logistics, key, value)
|
||||
_autofill_transport_int_ids(
|
||||
db,
|
||||
invoice.logistics,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
)
|
||||
else:
|
||||
logistics_dict = invoice_data.logistics.model_dump()
|
||||
logistics_dict["invoice_id"] = invoice.id
|
||||
logistics_dict["tenant_id"] = tenant_id
|
||||
logistics_dict["company_id"] = company_id
|
||||
_autofill_transport_int_ids(
|
||||
db,
|
||||
logistics_dict,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
)
|
||||
new_logistics = models.InvoiceLogistics(**logistics_dict)
|
||||
db.add(new_logistics)
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ async def list_items(
|
||||
search: Optional[str] = Query(
|
||||
None, description="Search term for invoice number, reference, order, or guide"),
|
||||
sort_by: Optional[str] = Query(None, description="Column to sort by"),
|
||||
sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"),
|
||||
sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
@@ -178,7 +178,7 @@ async def list_items_by_invoice(
|
||||
skip: int = Query(0, ge=0, description="Number of records to skip"),
|
||||
limit: int = Query(100, ge=1, le=1000, description="Maximum records to return"),
|
||||
sort_by: Optional[str] = Query(None, description="Column to sort by"),
|
||||
sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"),
|
||||
sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"),
|
||||
db: Session = Depends(get_core_db),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user),
|
||||
):
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para BOMs.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,7 +13,6 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -22,12 +20,14 @@ from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
BOM_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
BOM_IMPORT_META_PREFIX,
|
||||
BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -66,31 +66,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{BOM_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=BOM_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=BOM_IMPORT_REDIS_TTL,
|
||||
log_label="BOMs import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"BOMs import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"BOMs import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"BOMs import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -127,12 +119,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -62,8 +62,7 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Cambio de régimen y Regularización (encabezado y partidas).
|
||||
Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,7 +13,6 @@ from typing import Literal, Optional, Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -22,6 +20,7 @@ from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from .schemas import ImportJobResponse, CommitRequest
|
||||
from .tasks import scan_file, insert_valid_rows, JOB_TYPE, CRREG_IMPORT_REDIS_TTL
|
||||
from ..common import storage as common_storage
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
|
||||
router = APIRouter()
|
||||
@@ -64,7 +63,6 @@ async def upload_import_file(
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id)
|
||||
meta_data = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
@@ -75,25 +73,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(file_key, base64.b64encode(contents), ex=CRREG_IMPORT_REDIS_TTL)
|
||||
r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=CRREG_IMPORT_REDIS_TTL)
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=CRREG_IMPORT_REDIS_TTL,
|
||||
log_label="Cambio régimen/Regularización import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error("Cambio régimen/Regularización import: store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error("Cambio régimen/Regularización import: Redis store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
with open(csv_path, "wb") as f:
|
||||
f.write(contents)
|
||||
meta_path = csv_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning("Cambio régimen/Regularización import: local file save failed: %s", e)
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -133,11 +129,11 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"):
|
||||
return task_result.result
|
||||
return normalize_commit_status_payload(task_result.result)
|
||||
|
||||
logger.warning("Cambio régimen/Regularización import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -75,8 +75,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
|
||||
try:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None):
|
||||
if i % 500 == 0:
|
||||
on_progress(i, total_rows)
|
||||
on_progress(i, total_rows)
|
||||
_norm_row(row, template_id)
|
||||
processed_rows += 1
|
||||
except Exception as e:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Clases de Materiales.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,19 +13,20 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CLS_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
CLS_IMPORT_META_PREFIX,
|
||||
CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
|
||||
router = APIRouter()
|
||||
@@ -70,31 +70,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CLS_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CLS_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=CLS_IMPORT_REDIS_TTL,
|
||||
log_label="Classes import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Classes import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"Classes import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Classes import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -133,12 +125,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
# Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica
|
||||
if isinstance(result, str):
|
||||
@@ -150,7 +142,7 @@ async def get_import_status(job_id: str):
|
||||
):
|
||||
return parsed
|
||||
if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"):
|
||||
return parsed
|
||||
return normalize_commit_status_payload(parsed)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -84,8 +84,7 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Clientes y Proveedores.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,19 +13,20 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CP_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
CP_IMPORT_META_PREFIX,
|
||||
CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
|
||||
router = APIRouter()
|
||||
@@ -69,31 +69,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CP_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CP_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=CP_IMPORT_REDIS_TTL,
|
||||
log_label="CP import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"CP import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"CP import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CP import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -135,13 +127,13 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("CP import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -77,7 +77,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
|
||||
@@ -72,3 +72,36 @@ def commit_result(
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
|
||||
|
||||
def normalize_commit_status_payload(d: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Homogeneiza respuestas de commit CSV (finished / warning) al contrato usado por pedimentos:
|
||||
inserted, updated, skipped_*, skipped_details, skipped_summary, critical_reference_gaps,
|
||||
reference_state_ready. No modifica waiting_confirmation ni otros status.
|
||||
"""
|
||||
if not isinstance(d, dict):
|
||||
return d
|
||||
st = d.get("status")
|
||||
if st not in ("finished", "warning"):
|
||||
return d
|
||||
out = dict(d)
|
||||
out.setdefault("inserted", 0)
|
||||
out.setdefault("updated", 0)
|
||||
out.setdefault("skipped_invalid", 0)
|
||||
out.setdefault("skipped_missing_fk", 0)
|
||||
out.setdefault("skipped_duplicate", 0)
|
||||
out.setdefault("skipped_missing_invoice", 0)
|
||||
details = out.get("skipped_details")
|
||||
if not isinstance(details, list):
|
||||
out["skipped_details"] = []
|
||||
else:
|
||||
out["skipped_details"] = details
|
||||
out.setdefault("critical_reference_gaps", 0)
|
||||
if "reference_state_ready" not in out:
|
||||
out["reference_state_ready"] = out.get("critical_reference_gaps", 0) == 0
|
||||
if "skipped_summary" not in out:
|
||||
out["skipped_summary"] = (
|
||||
_summarize_reasons(out["skipped_details"], "reason") if out["skipped_details"] else []
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -15,6 +15,12 @@ logger = logging.getLogger(__name__)
|
||||
IMPORT_REDIS_TTL = 3600
|
||||
|
||||
|
||||
class ImportStoreError(Exception):
|
||||
"""Fallo al guardar CSV en Redis/MinIO."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def _get_redis():
|
||||
import redis
|
||||
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
|
||||
@@ -60,23 +66,103 @@ def error_path_for_job(job_type: str, job_id: str) -> str:
|
||||
return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl")
|
||||
|
||||
|
||||
def store_import_file(
|
||||
job_type: str,
|
||||
job_id: str,
|
||||
raw_bytes: bytes,
|
||||
meta_dict: dict,
|
||||
tenant_id,
|
||||
company_id: int,
|
||||
ttl: int = IMPORT_REDIS_TTL,
|
||||
log_label: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Guarda CSV y meta en Redis. Si CSV_IMPORT_STORAGE=minio, sube el CSV a S3 y en Redis
|
||||
guarda JSON {"v":2,"s3_key":...}; si no, base64 en Redis (comportamiento anterior).
|
||||
En modo redis opcionalmente escribe copia local bajo layouts/imports/temp (debug).
|
||||
"""
|
||||
from core.config import settings
|
||||
|
||||
file_key, meta_key, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
meta_bytes = json.dumps(meta_dict).encode("utf-8")
|
||||
|
||||
if settings.CSV_IMPORT_STORAGE == "minio":
|
||||
from core.storage_s3 import put_csv_object, s3_key_for_csv_import
|
||||
|
||||
key = s3_key_for_csv_import(tenant_id, company_id, job_type, job_id)
|
||||
try:
|
||||
put_csv_object(key, raw_bytes)
|
||||
except Exception as e:
|
||||
logger.exception("%s MinIO put failed: %s", log_label or job_type, e)
|
||||
raise ImportStoreError(str(e)) from e
|
||||
payload = json.dumps({"v": 2, "s3_key": key}).encode("utf-8")
|
||||
r.set(file_key, payload, ex=ttl)
|
||||
else:
|
||||
r.set(file_key, base64.b64encode(raw_bytes), ex=ttl)
|
||||
|
||||
r.set(meta_key, meta_bytes, ex=ttl)
|
||||
|
||||
if settings.CSV_IMPORT_STORAGE != "minio":
|
||||
try:
|
||||
path = file_path_for_job(job_type, job_id)
|
||||
os.makedirs(upload_dir(), exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(raw_bytes)
|
||||
meta_path = path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta_dict, f)
|
||||
except Exception as e:
|
||||
logger.warning("%s local file save failed: %s", log_label or job_type, e)
|
||||
|
||||
|
||||
def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]:
|
||||
"""
|
||||
Descarga contenido del CSV desde Redis y lo escribe en disco.
|
||||
Obtiene el CSV desde Redis (referencia MinIO o base64 legacy) y lo escribe en disco.
|
||||
Devuelve la ruta del archivo o None si no hay datos o falla.
|
||||
"""
|
||||
from core.config import settings
|
||||
|
||||
file_key, _, _ = storage_keys(job_type, job_id)
|
||||
r = _get_redis()
|
||||
data = r.get(file_key)
|
||||
if not data:
|
||||
return None
|
||||
|
||||
path = file_path_for_job(job_type, job_id)
|
||||
os.makedirs(upload_dir(), exist_ok=True)
|
||||
|
||||
if data.startswith(b"{"):
|
||||
try:
|
||||
obj = json.loads(data.decode("utf-8"))
|
||||
if isinstance(obj, dict) and obj.get("v") == 2 and obj.get("s3_key"):
|
||||
if settings.CSV_IMPORT_STORAGE != "minio":
|
||||
logger.warning(
|
||||
"%s Redis has MinIO ref but CSV_IMPORT_STORAGE=%s",
|
||||
log_prefix or job_type,
|
||||
settings.CSV_IMPORT_STORAGE,
|
||||
)
|
||||
from core.storage_s3 import get_object_bytes
|
||||
|
||||
try:
|
||||
raw = get_object_bytes(obj["s3_key"])
|
||||
except Exception as e:
|
||||
logger.warning("%s MinIO get failed: %s", log_prefix or job_type, e)
|
||||
return None
|
||||
with open(path, "wb") as f:
|
||||
f.write(raw)
|
||||
return path
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to parse MinIO ref from Redis: %s", log_prefix or job_type, e)
|
||||
return None
|
||||
|
||||
try:
|
||||
raw = base64.b64decode(data)
|
||||
except Exception as e:
|
||||
logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e)
|
||||
return None
|
||||
path = file_path_for_job(job_type, job_id)
|
||||
os.makedirs(upload_dir(), exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(raw)
|
||||
return path
|
||||
@@ -152,7 +238,31 @@ def cleanup_import_job(
|
||||
error_path: Optional[str] = None,
|
||||
meta_path: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Elimina archivos locales y claves Redis del job."""
|
||||
"""Elimina archivos locales, objeto MinIO si aplica, y claves Redis del job."""
|
||||
from core.config import settings
|
||||
|
||||
if settings.CSV_IMPORT_STORAGE == "minio":
|
||||
from core.storage_s3 import delete_object_if_exists
|
||||
from core.s3_keys import legacy_csv_import_key
|
||||
|
||||
file_key, _, _ = storage_keys(job_type, job_id)
|
||||
try:
|
||||
r = _get_redis()
|
||||
data = r.get(file_key)
|
||||
deleted = False
|
||||
if data and data.startswith(b"{"):
|
||||
try:
|
||||
obj = json.loads(data.decode("utf-8"))
|
||||
if isinstance(obj, dict) and obj.get("v") == 2 and obj.get("s3_key"):
|
||||
delete_object_if_exists(obj["s3_key"])
|
||||
deleted = True
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: parse Redis file ref: %s", e)
|
||||
if not deleted:
|
||||
delete_object_if_exists(legacy_csv_import_key(job_type, job_id))
|
||||
except Exception as e:
|
||||
logger.warning("Cleanup: MinIO delete failed: %s", e)
|
||||
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Agentes Aduanales.
|
||||
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,19 +13,20 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
CB_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
CB_IMPORT_META_PREFIX,
|
||||
CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
|
||||
router = APIRouter()
|
||||
@@ -69,31 +69,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{CB_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=CB_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=CB_IMPORT_REDIS_TTL,
|
||||
log_label="CB import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"CB import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"CB import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"CB import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -135,13 +127,13 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("CB import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -88,7 +88,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER,
|
||||
headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL,
|
||||
):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importacion CSV para Conductores.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -15,19 +14,19 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
DRV_IMPORT_FILE_PREFIX,
|
||||
DRV_IMPORT_META_PREFIX,
|
||||
JOB_TYPE,
|
||||
DRV_IMPORT_STATUS_PREFIX,
|
||||
DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -66,31 +65,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{DRV_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=DRV_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=DRV_IMPORT_REDIS_TTL,
|
||||
log_label="Drivers import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Drivers import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"Drivers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Drivers import: local file save failed: {e}")
|
||||
|
||||
def run_scan_background():
|
||||
try:
|
||||
run_scan_sync(job_id)
|
||||
@@ -113,6 +104,8 @@ async def get_import_status(job_id: str):
|
||||
raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
|
||||
return normalize_commit_status_payload(data)
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Drivers import: could not read status from Redis: {e}")
|
||||
@@ -131,12 +124,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -135,7 +135,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Tipos de Cambio.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,19 +13,20 @@ from typing import Dict, Any, Optional
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
ER_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
ER_IMPORT_META_PREFIX,
|
||||
ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
|
||||
router = APIRouter()
|
||||
@@ -80,31 +80,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{ER_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=ER_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=ER_IMPORT_REDIS_TTL,
|
||||
log_label="ER import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"ER import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"ER import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"ER import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -146,12 +138,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("ER import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -66,7 +66,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Exportación (encabezado y partidas).
|
||||
Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -17,7 +16,6 @@ from typing import Literal, Optional, Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -25,6 +23,7 @@ from .schemas import ImportJobResponse, CommitRequest
|
||||
from .tasks import scan_file, insert_valid_rows, JOB_TYPE, EXP_IMPORT_REDIS_TTL
|
||||
from ..common import storage as common_storage
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -60,7 +59,6 @@ async def upload_import_file(
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id)
|
||||
default_template = (
|
||||
"exp_def_header" if model_target == "invoice_header"
|
||||
else "exp_def_series" if model_target == "invoice_series"
|
||||
@@ -82,25 +80,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(file_key, base64.b64encode(contents), ex=EXP_IMPORT_REDIS_TTL)
|
||||
r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=EXP_IMPORT_REDIS_TTL)
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=EXP_IMPORT_REDIS_TTL,
|
||||
log_label="Exportación import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error("Exportación import: store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error("Exportación import: Redis store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
||||
with open(csv_path, "wb") as f:
|
||||
f.write(contents)
|
||||
meta_path = csv_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning("Exportación import: local file save failed: %s", e)
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -138,11 +134,11 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"):
|
||||
return task_result.result
|
||||
return normalize_commit_status_payload(task_result.result)
|
||||
|
||||
logger.warning("Exportación import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -74,8 +74,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
|
||||
processed_rows = 0
|
||||
try:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None):
|
||||
if i % 500 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows})
|
||||
_norm_row(row, template_id)
|
||||
processed_rows += 1
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
import base64
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
@@ -14,19 +13,19 @@ from typing import Optional, Literal, Dict, Any
|
||||
from core.celery_app import celery_app
|
||||
from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
IMPORT_FILE_KEY_PREFIX,
|
||||
JOB_TYPE,
|
||||
IMPORT_META_KEY_PREFIX,
|
||||
IMPORT_REDIS_TTL,
|
||||
)
|
||||
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
|
||||
from ..common import storage as common_storage
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
|
||||
router = APIRouter()
|
||||
@@ -82,37 +81,25 @@ async def upload_import_file(
|
||||
"template_id": template_id,
|
||||
}
|
||||
|
||||
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
|
||||
try:
|
||||
redis_client = _get_redis()
|
||||
redis_client.set(
|
||||
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
)
|
||||
redis_client.set(
|
||||
f"{IMPORT_META_KEY_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=IMPORT_REDIS_TTL,
|
||||
log_label="Facturas import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Import store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
|
||||
except Exception as e:
|
||||
logger.error(f"Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
|
||||
|
||||
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
file_path = os.path.join(upload_dir, f"{job_id}.csv")
|
||||
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(contents)
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Local file save failed (worker will use Redis): {e}")
|
||||
|
||||
# Trigger Celery Task (Async). Worker loads file from Redis.
|
||||
# Trigger Celery Task (Async). Worker loads file from Redis / MinIO.
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -150,7 +137,7 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
|
||||
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)
|
||||
|
||||
@@ -27,10 +27,20 @@ from ..common import storage as common_storage
|
||||
from ..common import meta as common_meta
|
||||
from ..common import responses as common_responses
|
||||
from .template_config import row_from_template
|
||||
from .validators.encabezados_impo_temp import csv_tipo_moneda_es_me_mn_mc
|
||||
# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _merge_unique_invoice_scan_error_lines(
|
||||
precheck_lines: Set[int],
|
||||
row_validation_lines: List[int],
|
||||
) -> List[int]:
|
||||
"""Unión deduplicada: líneas marcadas en precheck + líneas con error en validación por fila (imp_temp_header)."""
|
||||
return sorted(set(precheck_lines) | set(row_validation_lines))
|
||||
|
||||
|
||||
# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py)
|
||||
JOB_TYPE = ""
|
||||
|
||||
@@ -618,8 +628,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
errors_detail = []
|
||||
error_lines_list: List[int] = []
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
row_norm = row_from_template(row, "imp_def_series", normalize_header)
|
||||
warnings_list: List[Dict[str, Any]] = []
|
||||
row_errors = validate_row_series_impo_def(
|
||||
@@ -812,8 +821,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
errors_detail = []
|
||||
error_lines_list: List[int] = []
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
row_norm = row_from_template(row, "exp_def_series", normalize_header)
|
||||
warnings_list: List[Dict[str, Any]] = []
|
||||
row_errors = validate_row_series_expo(
|
||||
@@ -994,8 +1002,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
errors_detail = []
|
||||
error_lines_list: List[int] = []
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
row_norm = row_from_template(row, "cmex_series", normalize_header)
|
||||
warnings_list: List[Dict[str, Any]] = []
|
||||
row_errors = validate_row_series_impo_def(
|
||||
@@ -1149,8 +1156,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
error_lines_list: List[int] = []
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
|
||||
row_norm = row_from_template(row, "imp_temp_series", normalize_header)
|
||||
warnings_list: List[Dict[str, Any]] = []
|
||||
row_errors = validate_row_series_impo_temp(
|
||||
@@ -1485,8 +1491,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "imp_temp_details", normalize_header)
|
||||
row_errors = validate_row_partidas_impo_temp(
|
||||
row_norm,
|
||||
@@ -1796,8 +1801,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "imp_def_details", normalize_header)
|
||||
row_errors = validate_row_partidas_impo_def(
|
||||
row_norm,
|
||||
@@ -2363,8 +2367,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "cmex_details", normalize_header)
|
||||
row_errors = validate_row_partidas_impo_def(
|
||||
row_norm,
|
||||
@@ -2453,6 +2456,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
parse_pedimento_col_a,
|
||||
_pedimento_key_from_parsed,
|
||||
)
|
||||
from .validators.transport_catalog import logistics_scan_row_errors
|
||||
|
||||
def _ped_key_from_row(ped_str: str) -> Optional[str]:
|
||||
parsed = parse_pedimento_col_a(ped_str)
|
||||
@@ -2781,7 +2785,31 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
}
|
||||
)
|
||||
|
||||
# No return temprano: el commit siempre valida por fila (logística, etc.). Si solo se
|
||||
# devolvía el precheck, valid_rows del scan quedaba inflado vs inserted/skipped del commit.
|
||||
scan_precheck_message: Optional[str] = None
|
||||
if precheck_errors:
|
||||
scan_precheck_message = (
|
||||
"Precheck de referencias falló. Corrige catálogos/pedimentos según los errores indicados. "
|
||||
"Se aplicó también validación completa por fila (logística, etc.) para paridad con el commit."
|
||||
)
|
||||
|
||||
processed_rows = 0
|
||||
error_lines_list: List[int] = []
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
for e in precheck_errors:
|
||||
if len(errors_detail) < 5000:
|
||||
errors_detail.append(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
"warning": False,
|
||||
}
|
||||
)
|
||||
|
||||
with CoreSessionLocal() as session:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for e in precheck_errors:
|
||||
f_err.write(
|
||||
@@ -2795,96 +2823,95 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
unique_precheck_lines = sorted(precheck_lines)
|
||||
common_storage.store_error_lines(effective_job_type, job_id, unique_precheck_lines)
|
||||
return common_responses.scan_result(
|
||||
job_id,
|
||||
len(rows_list),
|
||||
len(unique_precheck_lines),
|
||||
precheck_errors,
|
||||
total_rows_in_file=total_rows,
|
||||
message="Precheck de referencias falló. Corrige catálogos/pedimentos antes de confirmar importación.",
|
||||
)
|
||||
|
||||
error_count = 0
|
||||
processed_rows = 0
|
||||
error_lines_list: List[int] = []
|
||||
errors_detail: List[Dict[str, Any]] = []
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "imp_temp_header", normalize_header)
|
||||
warnings_row: List[Dict[str, Any]] = []
|
||||
row_errors = validate_row_encabezados_impo_temp(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
invoice_exists_by_number=invoice_exists_by_number,
|
||||
invoice_processed_by_number=invoice_processed_by_number,
|
||||
pedimento_rows=pedimento_rows,
|
||||
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
|
||||
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
|
||||
valid_provider_ids=valid_provider_ids,
|
||||
valid_sold_to_ids=valid_sold_to_ids,
|
||||
valid_shipped_to_ids=valid_shipped_to_ids,
|
||||
valid_provider_short_names=valid_provider_short_names,
|
||||
valid_sold_to_short_names=valid_sold_to_short_names,
|
||||
valid_shipped_to_short_names=valid_shipped_to_short_names,
|
||||
valid_broker_ids=valid_broker_ids,
|
||||
valid_broker_claves=valid_broker_claves,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_incoterms=valid_incoterms,
|
||||
valid_aduana_codes=valid_aduana_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
exchange_rate_by_date=exchange_rate_by_date,
|
||||
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
|
||||
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
|
||||
autonumerar_remesas=autonumerar_remesas,
|
||||
control_remesa=control_remesa,
|
||||
remesa_inicio=remesa_inicio,
|
||||
remesa_fin=remesa_fin,
|
||||
date_format=date_format,
|
||||
parse_date_fn=parse_date,
|
||||
warnings=warnings_row,
|
||||
)
|
||||
blocking = [e for e in row_errors if not e.get("warning")]
|
||||
if blocking:
|
||||
error_count += 1
|
||||
error_lines_list.append(i)
|
||||
for e in blocking:
|
||||
f_err.write(
|
||||
json.dumps(
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={
|
||||
"current": i,
|
||||
"total": total_rows,
|
||||
"errors": len(
|
||||
_merge_unique_invoice_scan_error_lines(
|
||||
precheck_lines, error_lines_list
|
||||
)
|
||||
),
|
||||
},
|
||||
)
|
||||
row_norm = row_from_template(row, "imp_temp_header", normalize_header)
|
||||
warnings_row: List[Dict[str, Any]] = []
|
||||
row_errors = validate_row_encabezados_impo_temp(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
invoice_exists_by_number=invoice_exists_by_number,
|
||||
invoice_processed_by_number=invoice_processed_by_number,
|
||||
pedimento_rows=pedimento_rows,
|
||||
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
|
||||
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
|
||||
valid_provider_ids=valid_provider_ids,
|
||||
valid_sold_to_ids=valid_sold_to_ids,
|
||||
valid_shipped_to_ids=valid_shipped_to_ids,
|
||||
valid_provider_short_names=valid_provider_short_names,
|
||||
valid_sold_to_short_names=valid_sold_to_short_names,
|
||||
valid_shipped_to_short_names=valid_shipped_to_short_names,
|
||||
valid_broker_ids=valid_broker_ids,
|
||||
valid_broker_claves=valid_broker_claves,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_incoterms=valid_incoterms,
|
||||
valid_aduana_codes=valid_aduana_codes,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
exchange_rate_by_date=exchange_rate_by_date,
|
||||
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
|
||||
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
|
||||
autonumerar_remesas=autonumerar_remesas,
|
||||
control_remesa=control_remesa,
|
||||
remesa_inicio=remesa_inicio,
|
||||
remesa_fin=remesa_fin,
|
||||
date_format=date_format,
|
||||
parse_date_fn=parse_date,
|
||||
warnings=warnings_row,
|
||||
)
|
||||
row_errors.extend(
|
||||
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
|
||||
)
|
||||
blocking = [e for e in row_errors if not e.get("warning")]
|
||||
if blocking:
|
||||
error_lines_list.append(i)
|
||||
for e in blocking:
|
||||
f_err.write(
|
||||
json.dumps(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
for e in row_errors + warnings_row:
|
||||
if len(errors_detail) < 5000:
|
||||
errors_detail.append(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
"warning": bool(e.get("warning", False)),
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
for e in row_errors + warnings_row:
|
||||
if len(errors_detail) < 5000:
|
||||
errors_detail.append(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
"warning": bool(e.get("warning", False)),
|
||||
}
|
||||
)
|
||||
processed_rows += 1
|
||||
processed_rows += 1
|
||||
|
||||
# El commit usa líneas únicas para omitir filas; el resumen preliminar
|
||||
# debe usar la misma base para evitar discrepancias de válidos/errores.
|
||||
unique_error_lines = sorted(set(error_lines_list))
|
||||
# El commit usa líneas únicas para omitir filas; unir precheck + validación por fila.
|
||||
unique_error_lines = _merge_unique_invoice_scan_error_lines(precheck_lines, error_lines_list)
|
||||
error_count = len(unique_error_lines)
|
||||
common_storage.store_error_lines(effective_job_type, job_id, unique_error_lines)
|
||||
return common_responses.scan_result(
|
||||
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
|
||||
job_id,
|
||||
processed_rows,
|
||||
error_count,
|
||||
errors_detail,
|
||||
total_rows_in_file=total_rows,
|
||||
message=scan_precheck_message,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Encabezados importación temporal scan failed: %s", e)
|
||||
@@ -2909,6 +2936,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
parse_pedimento_col_a_impo_def,
|
||||
)
|
||||
from .validators.encabezados_impo_temp import _pedimento_key_from_parsed
|
||||
from .validators.transport_catalog import logistics_scan_row_errors
|
||||
|
||||
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF")
|
||||
|
||||
@@ -3169,8 +3197,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "imp_def_header", normalize_header)
|
||||
warnings_row = []
|
||||
row_errors = validate_row_encabezados_impo_def(
|
||||
@@ -3205,6 +3232,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
parse_date_fn=parse_date,
|
||||
warnings=warnings_row,
|
||||
)
|
||||
row_errors.extend(
|
||||
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
|
||||
)
|
||||
blocking = [e for e in row_errors if not e.get("warning")]
|
||||
if blocking:
|
||||
error_count += 1
|
||||
@@ -3260,6 +3290,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
from .validators.encabezados_expo import validate_row_encabezados_expo
|
||||
from .validators.encabezados_impo_temp import _pedimento_key_from_parsed
|
||||
from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def
|
||||
from .validators.transport_catalog import logistics_scan_row_errors
|
||||
|
||||
_fc = parse_footer_config(meta.get("footer_config"))
|
||||
actualizar = meta.get("actualizar", False)
|
||||
@@ -3542,8 +3573,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "exp_def_header", normalize_header)
|
||||
warnings_row = []
|
||||
row_errors = validate_row_encabezados_expo(
|
||||
@@ -3582,6 +3612,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
parse_date_fn=parse_date,
|
||||
warnings=warnings_row,
|
||||
)
|
||||
row_errors.extend(
|
||||
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
|
||||
)
|
||||
blocking = [e for e in row_errors if not e.get("warning")]
|
||||
if blocking:
|
||||
error_count += 1
|
||||
@@ -3630,6 +3663,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
|
||||
from api.v1.modules.a76.transportation.transporters.models import Transporter
|
||||
from .validators.encabezados_cmex import validate_row_encabezados_cmex
|
||||
from .validators.transport_catalog import logistics_scan_row_errors
|
||||
|
||||
_fc = parse_footer_config(meta.get("footer_config"))
|
||||
actualizar = meta.get("actualizar", False)
|
||||
@@ -3751,62 +3785,68 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
error_lines_list = []
|
||||
errors_detail = []
|
||||
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
|
||||
row_norm = row_from_template(row, "cmex_header", normalize_header)
|
||||
warnings_row = []
|
||||
row_errors = validate_row_encabezados_cmex(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
invoice_exists_by_number=invoice_exists_by_number,
|
||||
invoice_processed_by_number=invoice_processed_by_number,
|
||||
valid_provider_ids=valid_provider_ids,
|
||||
valid_sold_to_ids=valid_sold_to_ids,
|
||||
valid_shipped_to_ids=valid_shipped_to_ids,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_incoterms=valid_incoterms,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
exchange_rate_by_date=exchange_rate_by_date,
|
||||
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
|
||||
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
|
||||
valid_provider_short_names=valid_provider_short_names,
|
||||
valid_sold_to_short_names=valid_sold_to_short_names,
|
||||
valid_shipped_to_short_names=valid_shipped_to_short_names,
|
||||
date_format=date_format,
|
||||
parse_date_fn=parse_date,
|
||||
warnings=warnings_row,
|
||||
)
|
||||
blocking = [e for e in row_errors if not e.get("warning")]
|
||||
if blocking:
|
||||
error_count += 1
|
||||
error_lines_list.append(i)
|
||||
for e in blocking:
|
||||
f_err.write(
|
||||
json.dumps(
|
||||
with CoreSessionLocal() as session:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in enumerate(rows_list, start=1):
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
row_norm = row_from_template(row, "cmex_header", normalize_header)
|
||||
warnings_row = []
|
||||
row_errors = validate_row_encabezados_cmex(
|
||||
row_norm,
|
||||
i,
|
||||
actualizar=actualizar,
|
||||
invoice_exists_by_number=invoice_exists_by_number,
|
||||
invoice_processed_by_number=invoice_processed_by_number,
|
||||
valid_provider_ids=valid_provider_ids,
|
||||
valid_sold_to_ids=valid_sold_to_ids,
|
||||
valid_shipped_to_ids=valid_shipped_to_ids,
|
||||
valid_transporter_keys=valid_transporter_keys,
|
||||
valid_incoterms=valid_incoterms,
|
||||
valid_currency_codes=valid_currency_codes,
|
||||
exchange_rate_by_date=exchange_rate_by_date,
|
||||
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
|
||||
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
|
||||
valid_provider_short_names=valid_provider_short_names,
|
||||
valid_sold_to_short_names=valid_sold_to_short_names,
|
||||
valid_shipped_to_short_names=valid_shipped_to_short_names,
|
||||
date_format=date_format,
|
||||
parse_date_fn=parse_date,
|
||||
warnings=warnings_row,
|
||||
)
|
||||
row_errors.extend(
|
||||
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
|
||||
)
|
||||
blocking = [e for e in row_errors if not e.get("warning")]
|
||||
if blocking:
|
||||
error_count += 1
|
||||
error_lines_list.append(i)
|
||||
for e in blocking:
|
||||
f_err.write(
|
||||
json.dumps(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
for e in row_errors + warnings_row:
|
||||
if len(errors_detail) < 5000:
|
||||
errors_detail.append(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
"warning": bool(e.get("warning", False)),
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
for e in row_errors + warnings_row:
|
||||
if len(errors_detail) < 5000:
|
||||
errors_detail.append(
|
||||
{
|
||||
"line": e["line"],
|
||||
"col": e.get("col", ""),
|
||||
"msg": e.get("msg", ""),
|
||||
"solution": e.get("solution", ""),
|
||||
"warning": bool(e.get("warning", False)),
|
||||
}
|
||||
)
|
||||
processed_rows += 1
|
||||
processed_rows += 1
|
||||
|
||||
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
|
||||
return common_responses.scan_result(
|
||||
@@ -3898,12 +3938,11 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
|
||||
|
||||
for i, row in enumerate(reader, start=1):
|
||||
# Check for Progress Update
|
||||
if i % 1000 == 0:
|
||||
self.update_state(state='PROGRESS', meta={
|
||||
'current': i,
|
||||
'total': total_rows,
|
||||
'errors': error_count
|
||||
})
|
||||
self.update_state(state='PROGRESS', meta={
|
||||
'current': i,
|
||||
'total': total_rows,
|
||||
'errors': error_count
|
||||
})
|
||||
|
||||
# Solo columnas de la plantilla (respetar plantilla tal cual)
|
||||
row_norm = row_from_template(row, template_id, normalize_header)
|
||||
@@ -4016,6 +4055,8 @@ def validate_row_phase_1(
|
||||
def check_currency(col_name):
|
||||
val = row.get(col_name)
|
||||
if val and str(val).strip():
|
||||
if csv_tipo_moneda_es_me_mn_mc(str(val)):
|
||||
return None
|
||||
parsed_currency = parse_currency(val, None)
|
||||
val_norm = normalize_header(val)
|
||||
# parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid
|
||||
@@ -4223,6 +4264,18 @@ def validate_row_strict(
|
||||
if err:
|
||||
return err
|
||||
|
||||
from .validators.transport_catalog import logistics_scan_row_errors
|
||||
|
||||
log_errs = logistics_scan_row_errors(
|
||||
row,
|
||||
validator.session,
|
||||
validator.tenant_id,
|
||||
validator.company_id,
|
||||
line_num,
|
||||
)
|
||||
if log_errs:
|
||||
return log_errs[0]
|
||||
|
||||
elif target == "invoice_details":
|
||||
invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip()
|
||||
if not invoice_number:
|
||||
@@ -4398,6 +4451,13 @@ def parse_currency(value: Optional[str], currency_type: Optional[str]):
|
||||
return Currency.FOREIGN
|
||||
if "MANUAL" in normalized:
|
||||
return Currency.MANUAL
|
||||
# MC (moneda por clave): paridad con encabezados_impo_temp / InvoiceFinancials + CLAVE MONEDA
|
||||
if normalized.replace(" ", "") == "MC":
|
||||
if currency_type and str(currency_type).strip().upper() == "MXN":
|
||||
return Currency.LOCAL
|
||||
if currency_type:
|
||||
return Currency.FOREIGN
|
||||
return Currency.FOREIGN
|
||||
if currency_type and str(currency_type).strip().upper() == "MXN":
|
||||
return Currency.LOCAL
|
||||
if currency_type:
|
||||
@@ -5370,6 +5430,12 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt
|
||||
row_to_transport_type_clarion,
|
||||
_patente_from_agente_aduanal,
|
||||
)
|
||||
from .validators.transport_catalog import (
|
||||
normalize_transport_cell,
|
||||
resolve_logistics_transport_fields_for_commit,
|
||||
resolve_transport_csv_fields,
|
||||
validate_csv_invoice_logistics_transport,
|
||||
)
|
||||
from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def
|
||||
from .validators.pedimento_resolution import resolve_pedimento_candidates
|
||||
|
||||
@@ -5832,14 +5898,15 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt
|
||||
|
||||
currency_val = row_norm.get('TIPO MONEDA')
|
||||
if currency_val and str(currency_val).strip():
|
||||
parsed_currency = parse_currency(currency_val, None)
|
||||
val_norm = normalize_header(currency_val)
|
||||
if parsed_currency.value == "manual" and "MANUAL" not in val_norm:
|
||||
skipped_invalid += 1
|
||||
reason = "TIPO MONEDA: Moneda invalida"
|
||||
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
|
||||
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
|
||||
continue
|
||||
if not csv_tipo_moneda_es_me_mn_mc(str(currency_val)):
|
||||
parsed_currency = parse_currency(currency_val, None)
|
||||
val_norm = normalize_header(currency_val)
|
||||
if parsed_currency.value == "manual" and "MANUAL" not in val_norm:
|
||||
skipped_invalid += 1
|
||||
reason = "TIPO MONEDA: Moneda invalida"
|
||||
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
|
||||
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
|
||||
continue
|
||||
|
||||
# 2. Client/Provider and broker checks are handled above
|
||||
|
||||
@@ -6064,18 +6131,84 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt
|
||||
|
||||
weight_type = parse_weight_unit(row_norm.get('TIPO PESO'))
|
||||
logistics = None
|
||||
if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'):
|
||||
transport_key_csv, trailer_num_csv, transport_num_csv = resolve_transport_csv_fields(row_norm)
|
||||
has_logistics_data = bool(
|
||||
weight_type
|
||||
or row_norm.get('TIPO TRANSPORTE')
|
||||
or row_norm.get('NUMERO TRANSPORTE')
|
||||
or transport_key_csv
|
||||
or trailer_num_csv
|
||||
or row_norm.get('CLAVE TRANSPORTISTA')
|
||||
or row_norm.get('NOMBRE CONDUCTOR')
|
||||
)
|
||||
if has_logistics_data:
|
||||
raw_transport = row_norm.get('TIPO TRANSPORTE')
|
||||
transport_str = (row_to_transport_type_clarion(raw_transport) or str(raw_transport or "").strip().lower() or "none")
|
||||
try:
|
||||
transport_type = TransportType(transport_str)
|
||||
except ValueError:
|
||||
transport_type = TransportType.NONE
|
||||
err_transport = validate_csv_invoice_logistics_transport(
|
||||
row_norm=row_norm,
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
transport_type=transport_type,
|
||||
)
|
||||
if err_transport:
|
||||
skipped_invalid += 1
|
||||
reason = err_transport
|
||||
skipped_fk_details.append(
|
||||
{"line": i, "invoice": invoice_number, "reason": reason}
|
||||
)
|
||||
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
|
||||
continue
|
||||
|
||||
transport_num_effective = (transport_num_csv or None)
|
||||
if transport_type == TransportType.NONE:
|
||||
transport_num_effective = None
|
||||
|
||||
carrier_code = normalize_transport_cell(row_norm.get('CLAVE TRANSPORTISTA')) or None
|
||||
carrier_int_id = None
|
||||
if carrier_code:
|
||||
from api.v1.modules.a76.transportation.transporters.models import (
|
||||
Transporter,
|
||||
)
|
||||
|
||||
carrier_obj = (
|
||||
session.query(Transporter)
|
||||
.filter(
|
||||
Transporter.transporter_key == carrier_code,
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if carrier_obj:
|
||||
carrier_int_id = carrier_obj.transporter_id
|
||||
|
||||
(
|
||||
transport_id_value,
|
||||
trailer_num_value,
|
||||
transport_int_id,
|
||||
trailer_int_id,
|
||||
) = resolve_logistics_transport_fields_for_commit(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
clave_transporte_raw=transport_key_csv,
|
||||
numero_caja_raw=trailer_num_csv,
|
||||
)
|
||||
logistics = InvoiceLogistics(
|
||||
carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None),
|
||||
carrier_id=carrier_code,
|
||||
carrier_int_id=carrier_int_id,
|
||||
driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None),
|
||||
transport_type=transport_type,
|
||||
transport_num=(row_norm.get('NUMERO TRANSPORTE') or None),
|
||||
transport_num=transport_num_effective,
|
||||
transport_id=transport_id_value,
|
||||
trailer_num=trailer_num_value,
|
||||
transport_int_id=transport_int_id,
|
||||
trailer_int_id=trailer_int_id,
|
||||
weight_type=weight_type or WeightUnit.KGS,
|
||||
seal_number=(row_norm.get('PRECINTO') or None),
|
||||
incoterm=(row_norm.get('CLAVE INCOTERM') or None),
|
||||
|
||||
@@ -29,6 +29,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "CLAVE TRANSPORTE"},
|
||||
{"canonical": "NUMERO CAJA"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
@@ -66,6 +68,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "CLAVE TRANSPORTE"},
|
||||
{"canonical": "NUMERO CAJA"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
@@ -102,6 +106,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "CLAVE TRANSPORTE"},
|
||||
{"canonical": "NUMERO CAJA"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
@@ -135,6 +141,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
|
||||
{"canonical": "CLAVE TRANSPORTISTA"},
|
||||
{"canonical": "NOMBRE CONDUCTOR"},
|
||||
{"canonical": "TIPO TRANSPORTE"},
|
||||
{"canonical": "CLAVE TRANSPORTE"},
|
||||
{"canonical": "NUMERO CAJA"},
|
||||
{"canonical": "NUMERO TRANSPORTE"},
|
||||
{"canonical": "TIPO MONEDA"},
|
||||
{"canonical": "CLAVE MONEDA"},
|
||||
|
||||
@@ -51,18 +51,6 @@ def _validaciones_transporte_cmex(row: Dict[str, Any], line_num: int) -> Optiona
|
||||
"NUMERO TRANSPORTE",
|
||||
"Error: (Celda N{}) El Tipo de Transporte está vacío y está capturado un número de transporte.".format(line_num),
|
||||
)
|
||||
if m_raw and (m == "NINGUNO" or m_with_space == "NINGUNO") and n:
|
||||
return _err(
|
||||
line_num,
|
||||
"NUMERO TRANSPORTE",
|
||||
"Error: (Celda N{}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.".format(line_num),
|
||||
)
|
||||
if m_raw and m != "NINGUNO" and m_with_space != "NINGUNO" and not n:
|
||||
return _err(
|
||||
line_num,
|
||||
"NUMERO TRANSPORTE",
|
||||
"Error: (Celda N{}) El Tipo de Transporte es {} y no está capturado el número de transporte.".format(line_num, m_raw),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,16 @@ REGIMENES_VALIDOS = frozenset({"ITE", "ITR"})
|
||||
TIPOS_MONEDA_VALIDOS = frozenset({"ME", "MN", "MC"})
|
||||
TIPO_PESO_VALIDOS = frozenset({"KILOS", "LIBRAS"})
|
||||
|
||||
|
||||
def csv_tipo_moneda_es_me_mn_mc(value: Any) -> bool:
|
||||
"""
|
||||
ME / MN / MC con la misma regla que _validaciones_moneda (strip + upper).
|
||||
Compartido entre scan (encabezados) y commit / validate_row_strict en tasks.
|
||||
"""
|
||||
if value is None or not str(value).strip():
|
||||
return False
|
||||
return str(value).strip().upper() in TIPOS_MONEDA_VALIDOS
|
||||
|
||||
# Clarion Col M → valor normalizado (minúscula para TransportType enum)
|
||||
TIPO_TRANSPORTE_CLARION_TO_NORM = {
|
||||
"NINGUNO": "none",
|
||||
@@ -344,18 +354,6 @@ def _validaciones_transporte(row: Dict[str, Any], line_num: int) -> Optional[Dic
|
||||
"NUMERO TRANSPORTE",
|
||||
f"Error: (Celda N{line_num}) El Tipo de Transporte está vacío y está capturado un número de transporte.",
|
||||
)
|
||||
if m == "NINGUNO" and n:
|
||||
return _err(
|
||||
line_num,
|
||||
"NUMERO TRANSPORTE",
|
||||
f"Error: (Celda N{line_num}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.",
|
||||
)
|
||||
if m and m != "NINGUNO" and not n:
|
||||
return _err(
|
||||
line_num,
|
||||
"NUMERO TRANSPORTE",
|
||||
f"Error: (Celda N{line_num}) El Tipo de Transporte es {m} y no está capturado el número de transporte.",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Mapeo y validación de transporte para CSV (paridad con facturas manuales / validate_common)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from api.v1.modules.a76.invoices.models import TransportType
|
||||
|
||||
|
||||
def normalize_transport_cell(raw: Optional[str]) -> str:
|
||||
if raw is None:
|
||||
return ""
|
||||
return str(raw).replace("\u00a0", " ").strip()
|
||||
|
||||
|
||||
def resolve_transport_csv_fields(row: dict) -> Tuple[str, str, str]:
|
||||
"""
|
||||
Prioridad de columnas:
|
||||
1) CLAVE TRANSPORTE / NUMERO CAJA
|
||||
2) NUMERO TRANSPORTE como fallback legacy.
|
||||
"""
|
||||
legacy = normalize_transport_cell(row.get("NUMERO TRANSPORTE"))
|
||||
clave_transporte = normalize_transport_cell(row.get("CLAVE TRANSPORTE")) or legacy
|
||||
numero_caja = normalize_transport_cell(row.get("NUMERO CAJA")) or legacy
|
||||
numero_transporte = legacy
|
||||
return clave_transporte, numero_caja, numero_transporte
|
||||
|
||||
|
||||
def resolve_logistics_transport_fields_for_commit(
|
||||
session: Session,
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
clave_transporte_raw: str,
|
||||
numero_caja_raw: str,
|
||||
) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[int]]:
|
||||
"""
|
||||
Resuelve claves y IDs internos para logística:
|
||||
- transport_id / transport_int_id a partir de Vehicle.vehicle_key
|
||||
- trailer_num / trailer_int_id a partir de Trailer.trailer_number
|
||||
"""
|
||||
from api.v1.modules.a76.transportation.vehicles.models import Vehicle
|
||||
from api.v1.modules.a76.transportation.trailers.models import Trailer
|
||||
|
||||
clave_transporte = normalize_transport_cell(clave_transporte_raw)
|
||||
numero_caja = normalize_transport_cell(numero_caja_raw)
|
||||
|
||||
transport_id: Optional[str] = clave_transporte or None
|
||||
trailer_num: Optional[str] = numero_caja or None
|
||||
transport_int_id: Optional[int] = None
|
||||
trailer_int_id: Optional[int] = None
|
||||
|
||||
if transport_id:
|
||||
vehicle = (
|
||||
session.query(Vehicle)
|
||||
.filter(
|
||||
func.upper(Vehicle.vehicle_key) == transport_id.upper(),
|
||||
Vehicle.tenant_id == tenant_id,
|
||||
Vehicle.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if vehicle is not None:
|
||||
transport_int_id = vehicle.vehicle_id
|
||||
|
||||
if trailer_num:
|
||||
trailer = (
|
||||
session.query(Trailer)
|
||||
.filter(
|
||||
func.upper(Trailer.trailer_number) == trailer_num.upper(),
|
||||
Trailer.tenant_id == tenant_id,
|
||||
Trailer.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if trailer is not None:
|
||||
trailer_int_id = trailer.trailer_id
|
||||
|
||||
return transport_id, trailer_num, transport_int_id, trailer_int_id
|
||||
|
||||
|
||||
def validate_csv_invoice_logistics_transport(
|
||||
row_norm: dict,
|
||||
session: Session,
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
transport_type: TransportType,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Paridad con validate_common (factura manual):
|
||||
- Transportista y conductor: solo catálogo si vienen informados.
|
||||
- Vehículo: solo catálogo si viene CLAVE TRANSPORTE explícita y no resuelve.
|
||||
- Remolque: obligatorio (y catálogo) si tipo ≠ none; con none, solo catálogo si NUMERO CAJA explícito.
|
||||
"""
|
||||
from api.v1.modules.a76.transportation.transporters.models import Transporter
|
||||
from api.v1.modules.a76.transportation.drivers.models import Driver
|
||||
|
||||
carrier_code = normalize_transport_cell(row_norm.get("CLAVE TRANSPORTISTA"))
|
||||
carrier_ok = False
|
||||
if carrier_code:
|
||||
tr_obj = (
|
||||
session.query(Transporter)
|
||||
.filter(
|
||||
Transporter.transporter_key == carrier_code,
|
||||
Transporter.tenant_id == tenant_id,
|
||||
Transporter.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not tr_obj:
|
||||
return "CLAVE TRANSPORTISTA no existe en el catálogo"
|
||||
carrier_ok = True
|
||||
|
||||
driver_raw = normalize_transport_cell(row_norm.get("NOMBRE CONDUCTOR"))
|
||||
if carrier_ok and driver_raw:
|
||||
dr = (
|
||||
session.query(Driver)
|
||||
.filter(
|
||||
Driver.transporter_key == carrier_code,
|
||||
Driver.driver_name == driver_raw,
|
||||
Driver.tenant_id == tenant_id,
|
||||
Driver.company_id == company_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not dr:
|
||||
return (
|
||||
"NOMBRE CONDUCTOR: no existe en el catálogo de conductores "
|
||||
"para el transportista indicado"
|
||||
)
|
||||
|
||||
transport_key_csv, trailer_num_csv, _ = resolve_transport_csv_fields(row_norm)
|
||||
(
|
||||
_transport_id_value,
|
||||
trailer_num_resolved,
|
||||
transport_int_id,
|
||||
trailer_int_id,
|
||||
) = resolve_logistics_transport_fields_for_commit(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
clave_transporte_raw=transport_key_csv,
|
||||
numero_caja_raw=trailer_num_csv,
|
||||
)
|
||||
|
||||
explicit_clave = normalize_transport_cell(row_norm.get("CLAVE TRANSPORTE"))
|
||||
if explicit_clave and transport_int_id is None:
|
||||
return "CLAVE TRANSPORTE: vehículo no existe en el catálogo"
|
||||
|
||||
explicit_caja = normalize_transport_cell(row_norm.get("NUMERO CAJA"))
|
||||
if transport_type == TransportType.NONE:
|
||||
if explicit_caja and trailer_int_id is None:
|
||||
return "NUMERO CAJA: remolque no existe en el catálogo"
|
||||
return None
|
||||
|
||||
if trailer_int_id is None:
|
||||
if not (trailer_num_resolved and str(trailer_num_resolved).strip()):
|
||||
return (
|
||||
"NUMERO CAJA (remolque) obligatorio cuando TIPO TRANSPORTE es distinto de NINGUNO"
|
||||
)
|
||||
return "NUMERO CAJA: remolque no existe en el catálogo"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _csv_row_has_resolved_weight_unit(row_norm: dict) -> bool:
|
||||
"""
|
||||
Paridad con insert_valid_rows: has_logistics_data incluye parse_weight_unit(TIPO PESO).
|
||||
Import diferido para evitar ciclo al cargar tasks.
|
||||
"""
|
||||
from api.v1.modules.a76.layouts_csv.facturas.tasks import parse_weight_unit
|
||||
|
||||
return parse_weight_unit(row_norm.get("TIPO PESO")) is not None
|
||||
|
||||
|
||||
def logistics_scan_row_errors(
|
||||
row_norm: dict,
|
||||
session: Session,
|
||||
tenant_id: int,
|
||||
company_id: int,
|
||||
line_num: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Errores de scan (misma lógica que commit: validate_csv_invoice_logistics_transport).
|
||||
Solo corre si la fila trae algún dato de logística de transporte.
|
||||
"""
|
||||
from .encabezados_impo_temp import row_to_transport_type_clarion
|
||||
|
||||
transport_key_csv, trailer_num_csv, _ = resolve_transport_csv_fields(row_norm)
|
||||
|
||||
def _cell(key: str) -> str:
|
||||
return normalize_transport_cell(row_norm.get(key))
|
||||
|
||||
has_data = bool(
|
||||
_cell("TIPO TRANSPORTE")
|
||||
or _cell("NUMERO TRANSPORTE")
|
||||
or _cell("CLAVE TRANSPORTE")
|
||||
or transport_key_csv
|
||||
or trailer_num_csv
|
||||
or _cell("CLAVE TRANSPORTISTA")
|
||||
or _cell("NOMBRE CONDUCTOR")
|
||||
or _csv_row_has_resolved_weight_unit(row_norm)
|
||||
)
|
||||
if not has_data:
|
||||
return []
|
||||
|
||||
raw_transport = row_norm.get("TIPO TRANSPORTE")
|
||||
transport_str = (
|
||||
row_to_transport_type_clarion(raw_transport)
|
||||
or str(raw_transport or "").strip().lower()
|
||||
or "none"
|
||||
)
|
||||
try:
|
||||
tt = TransportType(transport_str)
|
||||
except ValueError:
|
||||
tt = TransportType.NONE
|
||||
|
||||
msg = validate_csv_invoice_logistics_transport(
|
||||
row_norm=row_norm,
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
company_id=company_id,
|
||||
transport_type=tt,
|
||||
)
|
||||
if not msg:
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"line": line_num,
|
||||
"col": "LOGISTICA TRANSPORTE",
|
||||
"msg": msg,
|
||||
"solution": (
|
||||
"Verifica CLAVE TRANSPORTISTA, NOMBRE CONDUCTOR, CLAVE TRANSPORTE, "
|
||||
"NUMERO CAJA y TIPO TRANSPORTE (remolque obligatorio si no es NINGUNO)."
|
||||
),
|
||||
}
|
||||
]
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Números de Parte.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,7 +13,6 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -22,12 +20,14 @@ from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
PART_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
PART_IMPORT_META_PREFIX,
|
||||
PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -70,31 +70,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{PART_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PART_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=PART_IMPORT_REDIS_TTL,
|
||||
log_label="Parts import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Parts import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"Parts import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Parts import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -131,12 +123,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
# Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica
|
||||
if isinstance(result, str):
|
||||
@@ -148,7 +140,7 @@ async def get_import_status(job_id: str):
|
||||
):
|
||||
return parsed
|
||||
if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"):
|
||||
return parsed
|
||||
return normalize_commit_status_payload(parsed)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -94,8 +94,7 @@ def scan_file(self, job_id: str, config: str = None):
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if i % 500 == 0:
|
||||
self.update_state(
|
||||
self.update_state(
|
||||
state="PROGRESS",
|
||||
meta={"current": i, "total": total_rows, "errors": error_count},
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Pedimentos.
|
||||
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,11 +13,11 @@ from typing import Dict, Any, Optional
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
@@ -63,7 +62,6 @@ async def upload_import_file(
|
||||
job_id = str(uuid4())
|
||||
contents = await file.read()
|
||||
|
||||
file_key, meta_key, _ = common_storage.storage_keys(PED_JOB_TYPE, job_id)
|
||||
meta_data: Dict[str, Any] = {
|
||||
"tenant_id": tenant_id,
|
||||
"company_id": company_id,
|
||||
@@ -75,33 +73,23 @@ async def upload_import_file(
|
||||
meta_data["dateFormat"] = dateFormat
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
file_key,
|
||||
base64.b64encode(contents),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
meta_key,
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=PED_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
PED_JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=PED_IMPORT_REDIS_TTL,
|
||||
log_label="Pedimentos import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Pedimentos import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"Pedimentos import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
csv_path = common_storage.file_path_for_job(PED_JOB_TYPE, job_id)
|
||||
with open(csv_path, "wb") as f:
|
||||
f.write(contents)
|
||||
meta_path = csv_path.replace(".csv", ".meta.json")
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Pedimentos import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -143,12 +131,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -99,7 +99,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = _norm_row(row)
|
||||
|
||||
@@ -25,8 +25,8 @@ def load_trailers_fk_sets(
|
||||
Devuelve:
|
||||
- valid_trailer_type_keys: códigos de trailer_type (GTipoTrailer)
|
||||
- valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas
|
||||
- state_descriptions_upper: descripciones de estados en mayúsculas (GEstados)
|
||||
- state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país"
|
||||
- state_descriptions_upper: descripciones y mex_key en mayúsculas (GEstados)
|
||||
- state_country_set: (ame_key_pais, token_upper) para description o mex_key
|
||||
- state_ame_to_description: dict clave_ame_upper -> description_upper (opcional; vacío si State no tiene ame_key)
|
||||
"""
|
||||
valid_trailer_type_keys: Set[str] = set()
|
||||
@@ -50,21 +50,35 @@ def load_trailers_fk_sets(
|
||||
valid_country_ame.add((row[0] or "").strip().upper())
|
||||
|
||||
for state in session.query(State).all():
|
||||
country = (
|
||||
session.query(Country)
|
||||
.filter(Country.m3_key == state.m3_key)
|
||||
.first()
|
||||
)
|
||||
ame_country = None
|
||||
if country and (country.ame_key or "").strip():
|
||||
ame_country = (country.ame_key or "").strip().upper()
|
||||
|
||||
desc = (state.description or "").strip()
|
||||
if desc:
|
||||
state_descriptions_upper.add(desc.upper())
|
||||
country = (
|
||||
session.query(Country)
|
||||
.filter(Country.m3_key == state.m3_key)
|
||||
.first()
|
||||
if ame_country:
|
||||
state_country_set.add((ame_country, desc.upper()))
|
||||
|
||||
mex_key = (state.mex_key or "").strip()
|
||||
if mex_key:
|
||||
mk = mex_key.upper()
|
||||
state_descriptions_upper.add(mk)
|
||||
if ame_country:
|
||||
state_country_set.add((ame_country, mk))
|
||||
if desc:
|
||||
state_ame_to_description[mk] = desc.upper()
|
||||
|
||||
ame_state = getattr(state, "ame_key", None)
|
||||
if ame_state and (ame_state or "").strip():
|
||||
state_ame_to_description[(ame_state or "").strip().upper()] = (
|
||||
desc.upper() if desc else (ame_state or "").strip().upper()
|
||||
)
|
||||
if country and (country.ame_key or "").strip():
|
||||
state_country_set.add(
|
||||
((country.ame_key or "").strip().upper(), desc.upper())
|
||||
)
|
||||
ame = getattr(state, "ame_key", None)
|
||||
if ame and (ame or "").strip():
|
||||
state_ame_to_description[(ame or "").strip().upper()] = desc.upper() if desc else (ame or "").strip().upper()
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Trailers import: could not load FK sets: %s", e)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Trailers y Cajas.
|
||||
Flujo: upload -> scan -> status (polling) -> commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -15,7 +14,6 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -24,12 +22,13 @@ from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
TRL_IMPORT_FILE_PREFIX,
|
||||
TRL_IMPORT_META_PREFIX,
|
||||
JOB_TYPE,
|
||||
TRL_IMPORT_STATUS_PREFIX,
|
||||
TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -70,31 +69,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{TRL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=TRL_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=TRL_IMPORT_REDIS_TTL,
|
||||
log_label="Trailers import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Trailers import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"Trailers import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Trailers import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -132,6 +123,8 @@ async def get_import_status(job_id: str):
|
||||
raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
|
||||
return normalize_commit_status_payload(data)
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Trailers import: could not read status from Redis: {e}")
|
||||
@@ -150,12 +143,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -121,7 +121,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
|
||||
|
||||
for i, row in enumerate(dict_reader, start=1):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
|
||||
@@ -24,8 +24,8 @@ def load_transportistas_fk_sets(
|
||||
Devuelve:
|
||||
- existing_transporter_keys: claves de transportistas existentes (tenant/company) en mayúsculas
|
||||
- valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas
|
||||
- state_descriptions_upper: descripciones de estados en mayúsculas (GEstados)
|
||||
- state_country_set: set de (ame_key_pais, description_estado_upper) para validar estado pertenece a país
|
||||
- state_descriptions_upper: descripciones y claves cortas (mex_key) de estados en mayúsculas (GEstados)
|
||||
- state_country_set: set de (ame_key_pais, token_upper) para description o mex_key
|
||||
"""
|
||||
existing_transporter_keys: Set[str] = set()
|
||||
valid_country_ame: Set[str] = set()
|
||||
@@ -55,18 +55,27 @@ def load_transportistas_fk_sets(
|
||||
valid_country_ame.add((row[0] or "").strip().upper())
|
||||
|
||||
for state in session.query(State).all():
|
||||
country = (
|
||||
session.query(Country)
|
||||
.filter(Country.m3_key == state.m3_key)
|
||||
.first()
|
||||
)
|
||||
ame = None
|
||||
if country and (country.ame_key or "").strip():
|
||||
ame = (country.ame_key or "").strip().upper()
|
||||
|
||||
desc = (state.description or "").strip()
|
||||
if desc:
|
||||
state_descriptions_upper.add(desc.upper())
|
||||
country = (
|
||||
session.query(Country)
|
||||
.filter(Country.m3_key == state.m3_key)
|
||||
.first()
|
||||
)
|
||||
if country and (country.ame_key or "").strip():
|
||||
state_country_set.add(
|
||||
((country.ame_key or "").strip().upper(), desc.upper())
|
||||
)
|
||||
if ame:
|
||||
state_country_set.add((ame, desc.upper()))
|
||||
|
||||
mex_key = (state.mex_key or "").strip()
|
||||
if mex_key:
|
||||
mk = mex_key.upper()
|
||||
state_descriptions_upper.add(mk)
|
||||
if ame:
|
||||
state_country_set.add((ame, mk))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Transportistas import: could not load FK sets: %s", e)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Transportistas.
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -15,7 +14,6 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -24,12 +22,13 @@ from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
TRP_IMPORT_FILE_PREFIX,
|
||||
TRP_IMPORT_META_PREFIX,
|
||||
JOB_TYPE,
|
||||
TRP_IMPORT_STATUS_PREFIX,
|
||||
TRP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -70,31 +69,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{TRP_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=TRP_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{TRP_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=TRP_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=TRP_IMPORT_REDIS_TTL,
|
||||
log_label="Transportistas import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error("Transportistas import: store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error("Transportistas import: Redis store error: %s", e)
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"trp_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"trp_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning("Transportistas import: local file save failed: %s", e)
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -132,6 +123,8 @@ async def get_import_status(job_id: str):
|
||||
raw = r.get(f"{TRP_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
|
||||
return normalize_commit_status_payload(data)
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug("Transportistas import: could not read status from Redis: %s", e)
|
||||
@@ -150,12 +143,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("Transportistas import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -79,7 +79,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Fracción Americana (US Tariff Fractions).
|
||||
Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -14,19 +13,20 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
from .schemas import ImportJobResponse
|
||||
from .tasks import (
|
||||
scan_file,
|
||||
insert_valid_rows,
|
||||
FA_IMPORT_FILE_PREFIX,
|
||||
JOB_TYPE,
|
||||
FA_IMPORT_META_PREFIX,
|
||||
FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
|
||||
router = APIRouter()
|
||||
@@ -72,31 +72,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{FA_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=FA_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=FA_IMPORT_REDIS_TTL,
|
||||
log_label="FA import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"FA import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"FA import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"FA import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -138,12 +130,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("FA import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -68,7 +68,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
warn = validate_row_desfase_fa(row, i)
|
||||
|
||||
@@ -24,8 +24,8 @@ def load_vehicles_fk_sets(
|
||||
Devuelve:
|
||||
- valid_transport_codes: códigos de transport_types (GTipoTransportes)
|
||||
- valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas
|
||||
- state_descriptions_upper: descripciones de estados en mayúsculas (GEstados), para "estado existe"
|
||||
- state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país"
|
||||
- state_descriptions_upper: descripciones y mex_key en mayúsculas (GEstados)
|
||||
- state_country_set: (ame_key_pais, token_upper) para description o mex_key
|
||||
"""
|
||||
valid_transport_codes: Set[str] = set()
|
||||
valid_country_ame: Set[str] = set()
|
||||
@@ -46,21 +46,28 @@ def load_vehicles_fk_sets(
|
||||
if row[0]:
|
||||
valid_country_ame.add((row[0] or "").strip().upper())
|
||||
|
||||
# States: description (GEstados.Descripcion); State.m3_key = Country.m3_key
|
||||
for state in session.query(State).all():
|
||||
country = (
|
||||
session.query(Country)
|
||||
.filter(Country.m3_key == state.m3_key)
|
||||
.first()
|
||||
)
|
||||
ame = None
|
||||
if country and (country.ame_key or "").strip():
|
||||
ame = (country.ame_key or "").strip().upper()
|
||||
|
||||
desc = (state.description or "").strip()
|
||||
if desc:
|
||||
state_descriptions_upper.add(desc.upper())
|
||||
# País para este estado vía m3_key
|
||||
country = (
|
||||
session.query(Country)
|
||||
.filter(Country.m3_key == state.m3_key)
|
||||
.first()
|
||||
)
|
||||
if country and (country.ame_key or "").strip():
|
||||
state_country_set.add(
|
||||
((country.ame_key or "").strip().upper(), desc.upper())
|
||||
)
|
||||
if ame:
|
||||
state_country_set.add((ame, desc.upper()))
|
||||
|
||||
mex_key = (state.mex_key or "").strip()
|
||||
if mex_key:
|
||||
mk = mex_key.upper()
|
||||
state_descriptions_upper.add(mk)
|
||||
if ame:
|
||||
state_country_set.add((ame, mk))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Vehicles import: could not load FK sets: %s", e)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Rutas de importación CSV para Vehículos (Transportes).
|
||||
Flujo: upload → scan → status (polling) → commit.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -15,7 +14,6 @@ from typing import Dict, Any
|
||||
|
||||
from core.celery_app import celery_app
|
||||
from core.database import get_core_db
|
||||
from core.paths import layout_path
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from api.v1.modules.core.tasks_tracking import track_and_dispatch
|
||||
|
||||
@@ -24,12 +22,13 @@ from .tasks import (
|
||||
scan_file,
|
||||
run_scan_sync,
|
||||
run_commit_sync,
|
||||
VEHL_IMPORT_FILE_PREFIX,
|
||||
VEHL_IMPORT_META_PREFIX,
|
||||
JOB_TYPE,
|
||||
VEHL_IMPORT_STATUS_PREFIX,
|
||||
VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
from ..common import storage as common_storage
|
||||
from ..common.error_csv import download_scan_errors_csv_stream
|
||||
from ..common.responses import normalize_commit_status_payload
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -70,31 +69,23 @@ async def upload_import_file(
|
||||
}
|
||||
|
||||
try:
|
||||
r = _get_redis()
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
|
||||
base64.b64encode(contents),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
)
|
||||
r.set(
|
||||
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
|
||||
json.dumps(meta_data).encode("utf-8"),
|
||||
ex=VEHL_IMPORT_REDIS_TTL,
|
||||
common_storage.store_import_file(
|
||||
JOB_TYPE,
|
||||
job_id,
|
||||
contents,
|
||||
meta_data,
|
||||
tenant_id=int(tenant_id),
|
||||
company_id=company_id,
|
||||
ttl=VEHL_IMPORT_REDIS_TTL,
|
||||
log_label="Vehicles import",
|
||||
)
|
||||
except common_storage.ImportStoreError as e:
|
||||
logger.error(f"Vehicles import: store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
except Exception as e:
|
||||
logger.error(f"Vehicles import: Redis store error: {e}")
|
||||
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
|
||||
|
||||
try:
|
||||
upload_dir = layout_path("imports", "temp")
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f:
|
||||
f.write(contents)
|
||||
with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f:
|
||||
json.dump(meta_data, f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Vehicles import: local file save failed: {e}")
|
||||
|
||||
track_and_dispatch(
|
||||
db=db,
|
||||
task=scan_file,
|
||||
@@ -131,6 +122,8 @@ async def get_import_status(job_id: str):
|
||||
raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
|
||||
if raw:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
|
||||
return normalize_commit_status_payload(data)
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Vehicles import: could not read status from Redis: {e}")
|
||||
@@ -149,12 +142,12 @@ async def get_import_status(job_id: str):
|
||||
if task_result.state == "SUCCESS":
|
||||
result = task_result.result
|
||||
if isinstance(result, dict) and "status" in result:
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
return {"status": "finished", "result": result}
|
||||
|
||||
result = getattr(task_result, "result", None)
|
||||
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
|
||||
return result
|
||||
return normalize_commit_status_payload(result)
|
||||
|
||||
logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state)
|
||||
err_msg = None
|
||||
|
||||
@@ -92,7 +92,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
|
||||
try:
|
||||
with open(error_path, "w", encoding="utf-8") as f_err:
|
||||
for i, row in common_csv_reader.iter_csv_rows(file_path):
|
||||
if progress_callback and i % 500 == 0:
|
||||
if progress_callback:
|
||||
progress_callback(i, total_rows, error_count)
|
||||
|
||||
row_norm = row_from_template(row, common_normalize.normalize_header)
|
||||
|
||||
@@ -298,7 +298,13 @@ class ConsolidadoImportacionMexService:
|
||||
|
||||
if logistics:
|
||||
# 1. Transporter (CAAT / SCAC)
|
||||
if logistics.carrier_id:
|
||||
if logistics.carrier_int_id is not None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_id == logistics.carrier_int_id)
|
||||
.first()
|
||||
)
|
||||
elif logistics.carrier_id:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_key == logistics.carrier_id)
|
||||
@@ -342,11 +348,18 @@ class ConsolidadoImportacionMexService:
|
||||
|
||||
# 2. Vehicle (Placas Tracto) - Try transport_id first
|
||||
if logistics.transport_id:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if logistics.transport_int_id is not None:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if veh_obj:
|
||||
placas_val = veh_obj.plate_number or placas_val
|
||||
elif (
|
||||
@@ -362,11 +375,18 @@ class ConsolidadoImportacionMexService:
|
||||
|
||||
# 3. Trailer (Placas Remolque)
|
||||
if logistics.trailer_num:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if logistics.trailer_int_id is not None:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_id == logistics.trailer_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if trl_obj:
|
||||
placas_remolque_val = trl_obj.plate_number or ""
|
||||
|
||||
|
||||
@@ -175,8 +175,16 @@ class ConsolidadoImportacionMexService:
|
||||
|
||||
if logistics:
|
||||
# 1. Transporter (CAAT / SCAC)
|
||||
if logistics.carrier_id:
|
||||
transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first()
|
||||
if logistics.carrier_int_id is not None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_id == logistics.carrier_int_id)
|
||||
.first()
|
||||
)
|
||||
elif logistics.carrier_id:
|
||||
transporter_obj = db.query(Transporter).filter(
|
||||
Transporter.transporter_key == logistics.carrier_id
|
||||
).first()
|
||||
if transporter_obj:
|
||||
caat_val = transporter_obj.caat_code or ""
|
||||
scac_val = transporter_obj.transport_code or "" # Mapping transport_code to SCAC
|
||||
@@ -213,7 +221,16 @@ class ConsolidadoImportacionMexService:
|
||||
|
||||
# 2. Vehicle (Placas Tracto) - Try transport_id first
|
||||
if logistics.transport_id:
|
||||
veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first()
|
||||
if logistics.transport_int_id is not None:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
veh_obj = db.query(Vehicle).filter(
|
||||
Vehicle.vehicle_key == logistics.transport_id
|
||||
).first()
|
||||
if veh_obj:
|
||||
placas_val = veh_obj.plate_number or placas_val
|
||||
elif logistics.vehicle_num: # Fallback to vehicle_num if populated and transport_id failed/empty
|
||||
@@ -223,7 +240,16 @@ class ConsolidadoImportacionMexService:
|
||||
|
||||
# 3. Trailer (Placas Remolque)
|
||||
if logistics.trailer_num:
|
||||
trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first()
|
||||
if logistics.trailer_int_id is not None:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_id == logistics.trailer_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
trl_obj = db.query(Trailer).filter(
|
||||
Trailer.trailer_number == logistics.trailer_num
|
||||
).first()
|
||||
if trl_obj:
|
||||
placas_remolque_val = trl_obj.plate_number or ""
|
||||
|
||||
|
||||
@@ -317,7 +317,17 @@ class FacturaImportacionMexService:
|
||||
|
||||
if logistics:
|
||||
# 1. Transporter (CAAT / SCAC)
|
||||
if logistics.carrier_id:
|
||||
if logistics.carrier_int_id is not None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_id == logistics.carrier_int_id)
|
||||
.first()
|
||||
)
|
||||
if transporter_obj:
|
||||
caat_val = transporter_obj.caat_code or ""
|
||||
scac_val = transporter_obj.transport_code or ""
|
||||
transportista_val = transporter_obj.name or logistics.carrier_id
|
||||
elif logistics.carrier_id:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_key == logistics.carrier_id)
|
||||
@@ -332,11 +342,18 @@ class FacturaImportacionMexService:
|
||||
|
||||
# 2. Vehicle (Placas Tracto) - Try transport_id first
|
||||
if logistics.transport_id:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if logistics.transport_int_id is not None:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if veh_obj:
|
||||
placas_val = veh_obj.plate_number or placas_val
|
||||
elif (
|
||||
@@ -352,11 +369,18 @@ class FacturaImportacionMexService:
|
||||
|
||||
# 3. Trailer (Placas Remolque)
|
||||
if logistics.trailer_num:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if logistics.trailer_int_id is not None:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_id == logistics.trailer_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if trl_obj:
|
||||
placas_remolque_val = trl_obj.plate_number or ""
|
||||
|
||||
|
||||
@@ -166,8 +166,16 @@ class FacturaImportacionMexService:
|
||||
|
||||
if logistics:
|
||||
# 1. Transporter (CAAT / SCAC)
|
||||
if logistics.carrier_id:
|
||||
transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first()
|
||||
if logistics.carrier_int_id is not None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_id == logistics.carrier_int_id)
|
||||
.first()
|
||||
)
|
||||
elif logistics.carrier_id:
|
||||
transporter_obj = db.query(Transporter).filter(
|
||||
Transporter.transporter_key == logistics.carrier_id
|
||||
).first()
|
||||
if transporter_obj:
|
||||
caat_val = transporter_obj.caat_code or ""
|
||||
scac_val = transporter_obj.transport_code or "" # Mapping transport_code to SCAC
|
||||
@@ -175,7 +183,16 @@ class FacturaImportacionMexService:
|
||||
|
||||
# 2. Vehicle (Placas Tracto) - Try transport_id first
|
||||
if logistics.transport_id:
|
||||
veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first()
|
||||
if logistics.transport_int_id is not None:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
veh_obj = db.query(Vehicle).filter(
|
||||
Vehicle.vehicle_key == logistics.transport_id
|
||||
).first()
|
||||
if veh_obj:
|
||||
placas_val = veh_obj.plate_number or placas_val
|
||||
elif logistics.vehicle_num: # Fallback to vehicle_num if populated and transport_id failed/empty
|
||||
@@ -185,7 +202,16 @@ class FacturaImportacionMexService:
|
||||
|
||||
# 3. Trailer (Placas Remolque)
|
||||
if logistics.trailer_num:
|
||||
trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first()
|
||||
if logistics.trailer_int_id is not None:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_id == logistics.trailer_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
trl_obj = db.query(Trailer).filter(
|
||||
Trailer.trailer_number == logistics.trailer_num
|
||||
).first()
|
||||
if trl_obj:
|
||||
placas_remolque_val = trl_obj.plate_number or ""
|
||||
|
||||
|
||||
@@ -317,7 +317,17 @@ class FacturaImportacionUsaService:
|
||||
|
||||
if logistics:
|
||||
# 1. Transporter (CAAT / SCAC)
|
||||
if logistics.carrier_id:
|
||||
if logistics.carrier_int_id is not None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_id == logistics.carrier_int_id)
|
||||
.first()
|
||||
)
|
||||
if transporter_obj:
|
||||
caat_val = transporter_obj.caat_code or ""
|
||||
scac_val = transporter_obj.transport_code or ""
|
||||
transportista_val = transporter_obj.name or logistics.carrier_id
|
||||
elif logistics.carrier_id:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_key == logistics.carrier_id)
|
||||
@@ -325,18 +335,23 @@ class FacturaImportacionUsaService:
|
||||
)
|
||||
if transporter_obj:
|
||||
caat_val = transporter_obj.caat_code or ""
|
||||
scac_val = (
|
||||
transporter_obj.transport_code or ""
|
||||
) # Mapping transport_code to SCAC
|
||||
scac_val = transporter_obj.transport_code or ""
|
||||
transportista_val = transporter_obj.name or logistics.carrier_id
|
||||
|
||||
# 2. Vehicle (Plates)
|
||||
if logistics.transport_id:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if logistics.transport_int_id is not None:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if veh_obj:
|
||||
placas_val = veh_obj.plate_number or placas_val
|
||||
elif logistics.vehicle_num:
|
||||
@@ -350,11 +365,18 @@ class FacturaImportacionUsaService:
|
||||
|
||||
# 3. Trailer
|
||||
if logistics.trailer_num:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if logistics.trailer_int_id is not None:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_id == logistics.trailer_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if trl_obj:
|
||||
placas_remolque_val = trl_obj.plate_number or ""
|
||||
|
||||
|
||||
@@ -186,8 +186,18 @@ class PackingListService:
|
||||
|
||||
if logistics:
|
||||
# 1. Transporter (CAAT / SCAC)
|
||||
if logistics.carrier_id:
|
||||
transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first()
|
||||
if logistics.carrier_int_id is not None:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_id == logistics.carrier_int_id)
|
||||
.first()
|
||||
)
|
||||
elif logistics.carrier_id:
|
||||
transporter_obj = (
|
||||
db.query(Transporter)
|
||||
.filter(Transporter.transporter_key == logistics.carrier_id)
|
||||
.first()
|
||||
)
|
||||
if transporter_obj:
|
||||
caat_val = transporter_obj.caat_code or ""
|
||||
scac_val = transporter_obj.transport_code or ""
|
||||
@@ -195,7 +205,18 @@ class PackingListService:
|
||||
|
||||
# 2. Vehicle (Placas Tracto)
|
||||
if logistics.transport_id:
|
||||
veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first()
|
||||
if logistics.transport_int_id is not None:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
veh_obj = (
|
||||
db.query(Vehicle)
|
||||
.filter(Vehicle.vehicle_key == logistics.transport_id)
|
||||
.first()
|
||||
)
|
||||
if veh_obj:
|
||||
placas_val = veh_obj.plate_number or placas_val
|
||||
elif logistics.vehicle_num:
|
||||
@@ -205,7 +226,18 @@ class PackingListService:
|
||||
|
||||
# 3. Trailer (Placas Remolque)
|
||||
if logistics.trailer_num:
|
||||
trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first()
|
||||
if logistics.trailer_int_id is not None:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_id == logistics.trailer_int_id)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
trl_obj = (
|
||||
db.query(Trailer)
|
||||
.filter(Trailer.trailer_number == logistics.trailer_num)
|
||||
.first()
|
||||
)
|
||||
if trl_obj:
|
||||
placas_remolque_val = trl_obj.plate_number or ""
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from pydantic import BaseModel
|
||||
|
||||
class DriverBaseDTO(BaseModel):
|
||||
transporter_key: str
|
||||
driver_id: Optional[int] = None
|
||||
line: int
|
||||
driver_name: Optional[str] = None
|
||||
license_number: Optional[str] = None
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
|
||||
from core.database import Base
|
||||
from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
|
||||
from sqlalchemy import BigInteger, Column, ForeignKey, ForeignKeyConstraint, Integer, String
|
||||
|
||||
|
||||
class Driver(Base, TenantScopedMixin, TimestampMixin):
|
||||
@@ -16,6 +16,8 @@ class Driver(Base, TenantScopedMixin, TimestampMixin):
|
||||
nullable=False,
|
||||
)
|
||||
line = Column(Integer, primary_key=True, nullable=False)
|
||||
# Internal integer surrogate ID. The frontend continues to use transporter_key/line.
|
||||
driver_id = Column(BigInteger, nullable=False, unique=True, index=True)
|
||||
driver_name = Column(String(80), nullable=True)
|
||||
license_number = Column(String(29), nullable=True)
|
||||
express_line_id = Column(String(17), nullable=True)
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from . import dto, models
|
||||
|
||||
DRIVER_ID_SEQ = "a76.driver_driver_id_seq"
|
||||
|
||||
|
||||
def allocate_driver_id(db: Session) -> int:
|
||||
"""Next surrogate driver_id (sequence from migration ca7d3c4e8b2a)."""
|
||||
return db.execute(text(f"SELECT nextval('{DRIVER_ID_SEQ}')")).scalar()
|
||||
|
||||
|
||||
class DriverService:
|
||||
@staticmethod
|
||||
@@ -34,7 +42,10 @@ class DriverService:
|
||||
|
||||
@staticmethod
|
||||
def create_driver(db: Session, driver_data: dto.DriverCreateDTO):
|
||||
new_driver = models.Driver(**driver_data.dict())
|
||||
data = driver_data.model_dump()
|
||||
if data.get("driver_id") is None:
|
||||
data["driver_id"] = allocate_driver_id(db)
|
||||
new_driver = models.Driver(**data)
|
||||
db.add(new_driver)
|
||||
db.commit()
|
||||
db.refresh(new_driver)
|
||||
|
||||
@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
class TrailerBaseDTO(BaseModel):
|
||||
trailer_number: str = Field(..., description="Trailer number (primary identifier)")
|
||||
trailer_id: Optional[int] = Field(None, description="Internal trailer integer ID")
|
||||
ace_trailer_number: Optional[str] = None
|
||||
trailer_type_key: Optional[str] = None
|
||||
seal: Optional[str] = None
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
|
||||
from core.database import Base
|
||||
from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, String
|
||||
from sqlalchemy import BigInteger, Column, ForeignKey, ForeignKeyConstraint, String
|
||||
|
||||
|
||||
class Trailer(Base, TenantScopedMixin, TimestampMixin):
|
||||
@@ -10,6 +10,8 @@ class Trailer(Base, TenantScopedMixin, TimestampMixin):
|
||||
)
|
||||
|
||||
trailer_number = Column(String(20), primary_key=True, nullable=False)
|
||||
# Internal integer identifier (surrogate). The frontend continues to use trailer_number.
|
||||
trailer_id = Column(BigInteger, nullable=False, unique=True, index=True)
|
||||
ace_trailer_number = Column(String(10), nullable=True)
|
||||
trailer_type_key = Column(
|
||||
String(2), ForeignKey("public.trailer_type.trailer_type_key"), nullable=True
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
from typing import Optional, Tuple, List, Dict, Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import text
|
||||
|
||||
from . import dto, models
|
||||
|
||||
TRAILER_ID_SEQ = "a76.trailer_trailer_id_seq"
|
||||
|
||||
|
||||
def allocate_trailer_id(db: Session) -> int:
|
||||
"""Next surrogate trailer_id (sequence from migration ca7d3c4e8b2a)."""
|
||||
return db.execute(text(f"SELECT nextval('{TRAILER_ID_SEQ}')")).scalar()
|
||||
|
||||
|
||||
class TrailerService:
|
||||
"""Service for Trailer CRUD operations with tenant support"""
|
||||
@@ -66,8 +74,11 @@ class TrailerService:
|
||||
company_id: int,
|
||||
) -> models.Trailer:
|
||||
"""Create a new trailer"""
|
||||
data = trailer_data.model_dump()
|
||||
if data.get("trailer_id") is None:
|
||||
data["trailer_id"] = allocate_trailer_id(db)
|
||||
new_trailer = models.Trailer(
|
||||
**trailer_data.model_dump(), tenant_id=tenant_id, company_id=company_id
|
||||
**data, tenant_id=tenant_id, company_id=company_id
|
||||
)
|
||||
db.add(new_trailer)
|
||||
db.commit()
|
||||
|
||||
@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
class TransporterBaseDTO(BaseModel):
|
||||
transporter_key: str = Field(..., description="Transporter key (primary identifier)")
|
||||
transporter_id: Optional[int] = Field(None, description="Internal transporter integer ID")
|
||||
name: Optional[str] = None
|
||||
short_name: Optional[str] = None
|
||||
responsible: Optional[str] = None
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
|
||||
from core.database import Base
|
||||
from sqlalchemy import Column, ForeignKeyConstraint, String
|
||||
from sqlalchemy import BigInteger, Column, ForeignKeyConstraint, String
|
||||
|
||||
|
||||
class Transporter(Base, TenantScopedMixin, TimestampMixin):
|
||||
@@ -10,6 +10,8 @@ class Transporter(Base, TenantScopedMixin, TimestampMixin):
|
||||
)
|
||||
|
||||
transporter_key = Column(String(23), primary_key=True, nullable=False)
|
||||
# Internal integer identifier (surrogate). The frontend continues to use transporter_key.
|
||||
transporter_id = Column(BigInteger, nullable=False, unique=True, index=True)
|
||||
name = Column(String(256), nullable=True)
|
||||
short_name = Column(String(10), nullable=True)
|
||||
responsible = Column(String(100), nullable=True)
|
||||
|
||||
@@ -2,12 +2,19 @@ from typing import Optional, Tuple, List, Dict, Any
|
||||
|
||||
import logging
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import func, text
|
||||
|
||||
from . import dto, models
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRANSPORTER_ID_SEQ = "a76.transporter_transporter_id_seq"
|
||||
|
||||
|
||||
def allocate_transporter_id(db: Session) -> int:
|
||||
"""Next surrogate transporter_id (sequence from migration ca7d3c4e8b2a)."""
|
||||
return db.execute(text(f"SELECT nextval('{TRANSPORTER_ID_SEQ}')")).scalar()
|
||||
|
||||
|
||||
class TransporterService:
|
||||
"""Service for Transporter CRUD operations with tenant support"""
|
||||
@@ -91,8 +98,11 @@ class TransporterService:
|
||||
company_id: int,
|
||||
) -> models.Transporter:
|
||||
"""Create a new transporter"""
|
||||
data = transporter_data.model_dump()
|
||||
if data.get("transporter_id") is None:
|
||||
data["transporter_id"] = allocate_transporter_id(db)
|
||||
new_transporter = models.Transporter(
|
||||
**transporter_data.model_dump(), tenant_id=tenant_id, company_id=company_id
|
||||
**data, tenant_id=tenant_id, company_id=company_id
|
||||
)
|
||||
db.add(new_transporter)
|
||||
db.commit()
|
||||
|
||||
@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
class VehicleBaseDTO(BaseModel):
|
||||
vehicle_key: str = Field(..., description="Vehicle key (primary identifier)")
|
||||
vehicle_id: Optional[int] = Field(None, description="Internal vehicle integer ID")
|
||||
ace_vehicle_key: Optional[str] = None
|
||||
transporter_key: Optional[str] = None
|
||||
transport_identifier: Optional[str] = None
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
|
||||
from core.database import Base
|
||||
from sqlalchemy import DECIMAL, Column, Integer, String
|
||||
from sqlalchemy import DECIMAL, Column, Integer, BigInteger, String
|
||||
|
||||
|
||||
class Vehicle(Base, TenantScopedMixin, TimestampMixin):
|
||||
@@ -10,6 +10,8 @@ class Vehicle(Base, TenantScopedMixin, TimestampMixin):
|
||||
)
|
||||
|
||||
vehicle_key = Column(String(14), primary_key=True, nullable=False)
|
||||
# Internal integer identifier (surrogate). The frontend continues to use vehicle_key.
|
||||
vehicle_id = Column(BigInteger, nullable=False, unique=True, index=True)
|
||||
ace_vehicle_key = Column(String(10), nullable=True)
|
||||
transporter_key = Column(String(23), nullable=True)
|
||||
transport_identifier = Column(String(30), nullable=True)
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
from typing import Optional, Tuple, List, Dict, Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import text
|
||||
|
||||
from . import dto, models
|
||||
|
||||
VEHICLE_ID_SEQ = "a76.vehicle_vehicle_id_seq"
|
||||
|
||||
|
||||
def allocate_vehicle_id(db: Session) -> int:
|
||||
"""Next surrogate vehicle_id (sequence from migration ca7d3c4e8b2a)."""
|
||||
return db.execute(text(f"SELECT nextval('{VEHICLE_ID_SEQ}')")).scalar()
|
||||
|
||||
|
||||
class VehicleService:
|
||||
"""Service for Vehicle CRUD operations with tenant support"""
|
||||
@@ -66,8 +74,11 @@ class VehicleService:
|
||||
company_id: int,
|
||||
) -> models.Vehicle:
|
||||
"""Create a new vehicle"""
|
||||
data = vehicle_data.model_dump()
|
||||
if data.get("vehicle_id") is None:
|
||||
data["vehicle_id"] = allocate_vehicle_id(db)
|
||||
new_vehicle = models.Vehicle(
|
||||
**vehicle_data.model_dump(), tenant_id=tenant_id, company_id=company_id
|
||||
**data, tenant_id=tenant_id, company_id=company_id
|
||||
)
|
||||
db.add(new_vehicle)
|
||||
db.commit()
|
||||
|
||||
@@ -1,13 +1,24 @@
|
||||
import shutil
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Dict, Any
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Header, status, UploadFile, File
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy.orm import Session
|
||||
from core.database import get_core_db
|
||||
|
||||
from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.s3_keys import (
|
||||
help_asset_key,
|
||||
help_public_api_path,
|
||||
help_s3_key_to_public_relative_path,
|
||||
system_help_object_key,
|
||||
)
|
||||
from core.storage_s3 import get_object_bytes, put_object_bytes
|
||||
from core.security import get_current_user, has_role
|
||||
from .schemas import HelpArticleInDB, HelpArticleCreate, HelpArticleUpdate, HelpSyncRequest, HelpSyncResponse
|
||||
from .services import HelpCenterService
|
||||
@@ -80,60 +91,95 @@ def sync_help_article(sync_data: HelpSyncRequest, db: Session = Depends(get_core
|
||||
|
||||
return result
|
||||
|
||||
@router.get("/files/{file_path:path}")
|
||||
def serve_help_file(file_path: str):
|
||||
"""Sirve un objeto bajo system/help/ (público vía middleware)."""
|
||||
if ".." in file_path or file_path.startswith("/"):
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
try:
|
||||
key = system_help_object_key(file_path)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
if not settings.use_s3_object_storage:
|
||||
legacy = os.path.join("uploads", "help", file_path)
|
||||
if not os.path.isfile(legacy):
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
with open(legacy, "rb") as f:
|
||||
data = f.read()
|
||||
media = mimetypes.guess_type(file_path)[0] or "application/octet-stream"
|
||||
return Response(content=data, media_type=media)
|
||||
try:
|
||||
data = get_object_bytes(key)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
media = mimetypes.guess_type(file_path)[0] or "application/octet-stream"
|
||||
return Response(content=data, media_type=media)
|
||||
|
||||
|
||||
@router.post("/upload-image/")
|
||||
def upload_help_image(
|
||||
async def upload_help_image(
|
||||
file: UploadFile = File(...),
|
||||
current_user: Dict[str, Any] = Depends(has_role("admin"))
|
||||
):
|
||||
"""Sube una imagen para usar en los artículos."""
|
||||
try:
|
||||
file_ext = os.path.splitext(file.filename)[1]
|
||||
file_ext = os.path.splitext(file.filename or "")[1] or ".png"
|
||||
new_filename = f"{uuid.uuid4()}{file_ext}"
|
||||
file_location = f"uploads/help/{new_filename}"
|
||||
|
||||
# Ensure directory exists
|
||||
body = await file.read()
|
||||
if settings.use_s3_object_storage:
|
||||
key = help_asset_key("", new_filename)
|
||||
ct = mimetypes.guess_type(new_filename)[0] or "image/png"
|
||||
put_object_bytes(key, body, content_type=ct)
|
||||
rel = help_s3_key_to_public_relative_path(key)
|
||||
return {"url": help_public_api_path(rel)}
|
||||
os.makedirs("uploads/help", exist_ok=True)
|
||||
|
||||
with open(file_location, "wb+") as buffer:
|
||||
shutil.copyfileobj(file.file, buffer)
|
||||
|
||||
file_location = f"uploads/help/{new_filename}"
|
||||
with open(file_location, "wb") as f:
|
||||
f.write(body)
|
||||
return {"url": f"/api/uploads/help/{new_filename}"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.post("/upload-asset/")
|
||||
def upload_help_asset(
|
||||
async def upload_help_asset(
|
||||
file: UploadFile = File(...),
|
||||
current_user: Dict[str, Any] = Depends(has_role("admin"))
|
||||
):
|
||||
"""Sube cualquier tipo de archivo (PDF, Video, etc.) para la biblioteca."""
|
||||
try:
|
||||
file_ext = os.path.splitext(file.filename)[1].lower()
|
||||
file_ext = os.path.splitext(file.filename or "")[1].lower()
|
||||
new_filename = f"{uuid.uuid4()}{file_ext}"
|
||||
|
||||
# Guardar en una carpeta segun el tipo o general
|
||||
folder = "uploads/help/assets"
|
||||
if file_ext in ['.pdf']:
|
||||
folder = "uploads/help/pdfs"
|
||||
elif file_ext in ['.mp4', '.mov', '.avi']:
|
||||
folder = "uploads/help/videos"
|
||||
|
||||
file_location = f"{folder}/{new_filename}"
|
||||
|
||||
# Ensure directory exists
|
||||
subfolder = "assets"
|
||||
if file_ext == ".pdf":
|
||||
subfolder = "pdfs"
|
||||
elif file_ext in [".mp4", ".mov", ".avi"]:
|
||||
subfolder = "videos"
|
||||
|
||||
if settings.use_s3_object_storage:
|
||||
body = await file.read()
|
||||
key = help_asset_key(subfolder, new_filename)
|
||||
ct = file.content_type or mimetypes.guess_type(new_filename)[0] or "application/octet-stream"
|
||||
put_object_bytes(key, body, content_type=ct)
|
||||
rel = help_s3_key_to_public_relative_path(key)
|
||||
return {
|
||||
"url": help_public_api_path(rel),
|
||||
"filename": file.filename,
|
||||
"size": len(body),
|
||||
"mime_type": file.content_type,
|
||||
}
|
||||
|
||||
folder = f"uploads/help/{subfolder}"
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
|
||||
with open(file_location, "wb+") as buffer:
|
||||
shutil.copyfileobj(file.file, buffer)
|
||||
|
||||
# Get file size
|
||||
file_location = f"{folder}/{new_filename}"
|
||||
body = await file.read()
|
||||
with open(file_location, "wb") as f:
|
||||
f.write(body)
|
||||
file_size = os.path.getsize(file_location)
|
||||
|
||||
return {
|
||||
"url": f"/api/{file_location}",
|
||||
"filename": file.filename,
|
||||
"size": file_size,
|
||||
"mime_type": file.content_type
|
||||
"mime_type": file.content_type,
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -1,76 +1,111 @@
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import httpx
|
||||
import logging
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from core.config import settings
|
||||
from core.s3_keys import SYSTEM_HELP_PREFIX
|
||||
from core.storage_s3 import object_exists, put_object_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _asset_url_to_s3_key(asset_url: str) -> Optional[str]:
|
||||
"""Deriva la clave S3 bajo system/help/ a partir de una URL de artículo."""
|
||||
if "/help-center/files/" in asset_url:
|
||||
rel = asset_url.split("/help-center/files/", 1)[1].lstrip("/")
|
||||
if ".." in rel:
|
||||
return None
|
||||
return f"{SYSTEM_HELP_PREFIX}{rel}"
|
||||
u = asset_url.replace("/api/uploads/", "uploads/")
|
||||
if u.startswith("/"):
|
||||
u = u[1:]
|
||||
if u.startswith("uploads/help/"):
|
||||
return f"{SYSTEM_HELP_PREFIX}{u[len('uploads/help/') :]}"
|
||||
return None
|
||||
|
||||
|
||||
def download_file_from_hub(relative_path: str) -> bool:
|
||||
"""
|
||||
Downloads a file from the Hub to the local storage.
|
||||
relative_path: e.g., 'uploads/help/pdfs/myfile.pdf' or '/api/uploads/help/image.png'
|
||||
Descarga un asset del Hub y lo guarda en MinIO (system/help/...) o en disco si no hay almacenamiento S3 activo.
|
||||
relative_path: URL parcial, p. ej. '/api/uploads/help/x.png' o '/api/v1/core/help-center/files/pdfs/x.pdf'
|
||||
"""
|
||||
if not settings.CENTRAL_SERVER_URL or settings.CENTRAL_SERVER_URL == '""':
|
||||
return False
|
||||
|
||||
# Clean the path
|
||||
clean_path = relative_path.replace("/api/uploads/", "uploads/")
|
||||
if clean_path.startswith("/"):
|
||||
clean_path = clean_path[1:]
|
||||
|
||||
# Check if it starts with uploads
|
||||
if not clean_path.startswith("uploads/"):
|
||||
# If it doesn't start with uploads, it might just be the filename or a subpath
|
||||
# We assume it's relative to /app/
|
||||
pass
|
||||
key = _asset_url_to_s3_key(relative_path)
|
||||
if not key:
|
||||
logger.warning("download_file_from_hub: could not map URL to S3 key: %s", relative_path)
|
||||
return False
|
||||
|
||||
local_path = Path(clean_path)
|
||||
if local_path.exists():
|
||||
logger.info(f"File {clean_path} already exists, skipping download.")
|
||||
if settings.use_s3_object_storage and object_exists(key):
|
||||
logger.info("S3 object %s already exists, skipping download.", key)
|
||||
return True
|
||||
|
||||
# Ensure directories exist
|
||||
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Resolve Hub Base URL
|
||||
# CENTRAL_SERVER_URL is usually http://hub:8000/api/v1/core/help-center/sync/
|
||||
# We want http://hub:8000/api/
|
||||
base_url = settings.CENTRAL_SERVER_URL.split("/v1/")[0]
|
||||
# The file in backend is served usually under /api/uploads/...
|
||||
# But clean_path is just "uploads/...". So the Hub route is base_url + "/" + clean_path
|
||||
hub_file_url = f"{base_url}/{clean_path}"
|
||||
if "/help-center/files/" in relative_path:
|
||||
rel = relative_path.split("/help-center/files/", 1)[1].lstrip("/")
|
||||
hub_file_url = f"{base_url.rstrip('/')}/api/v1/core/help-center/files/{rel}"
|
||||
else:
|
||||
clean_path = relative_path.replace("/api/uploads/", "uploads/")
|
||||
if clean_path.startswith("/"):
|
||||
clean_path = clean_path[1:]
|
||||
hub_file_url = f"{base_url.rstrip('/')}/{clean_path}"
|
||||
|
||||
logger.info("Downloading asset from Hub: %s", hub_file_url)
|
||||
|
||||
logger.info(f"Downloading asset from Hub: {hub_file_url} -> {local_path}")
|
||||
|
||||
try:
|
||||
with httpx.Client() as client:
|
||||
response = client.get(hub_file_url, timeout=30.0)
|
||||
if response.status_code == 200:
|
||||
with open(local_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
logger.info(f"Successfully downloaded {clean_path}")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Failed to download {clean_path}: Status {response.status_code} URL: {hub_file_url}")
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"Failed to download %s: Status %s URL: %s",
|
||||
relative_path,
|
||||
response.status_code,
|
||||
hub_file_url,
|
||||
)
|
||||
return False
|
||||
body = response.content
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading {clean_path}: {str(e)}")
|
||||
logger.error("Error downloading %s: %s", relative_path, str(e))
|
||||
return False
|
||||
|
||||
if settings.use_s3_object_storage:
|
||||
rel = key[len(SYSTEM_HELP_PREFIX) :]
|
||||
ct = mimetypes.guess_type(rel)[0] or "application/octet-stream"
|
||||
try:
|
||||
put_object_bytes(key, body, content_type=ct)
|
||||
logger.info("Stored hub asset in S3: %s", key)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("S3 put failed for %s: %s", key, e)
|
||||
return False
|
||||
|
||||
rel = key[len(SYSTEM_HELP_PREFIX) :]
|
||||
local_path = Path("uploads/help") / rel
|
||||
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
local_path.write_bytes(body)
|
||||
logger.info("Stored hub asset locally: %s", local_path)
|
||||
return True
|
||||
|
||||
|
||||
def sync_assets_from_content(content: str):
|
||||
"""
|
||||
Parses markdown content for image URLs and downloads them if they are local references.
|
||||
Example: 
|
||||
"""
|
||||
"""Parsea markdown y descarga imágenes referenciadas (rutas legacy y nuevas)."""
|
||||
if not content:
|
||||
return
|
||||
|
||||
# Regex for markdown images: 
|
||||
image_pattern = r'!\[.*?\]\((/api/uploads/.*?)\)'
|
||||
matches = re.findall(image_pattern, content)
|
||||
|
||||
for asset_url in matches:
|
||||
download_file_from_hub(asset_url)
|
||||
patterns = [
|
||||
r'!\[.*?\]\((/api/uploads/.*?)\)',
|
||||
r'!\[.*?\]\((/api/v1/core/help-center/files/.*?)\)',
|
||||
]
|
||||
seen = set()
|
||||
for pattern in patterns:
|
||||
for asset_url in re.findall(pattern, content):
|
||||
if asset_url in seen:
|
||||
continue
|
||||
seen.add(asset_url)
|
||||
download_file_from_hub(asset_url)
|
||||
|
||||
@@ -2,14 +2,20 @@
|
||||
Rutas para gestión de usuarios de Keycloak
|
||||
"""
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
from typing import Optional
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from core.config import settings
|
||||
from core.database import get_core_db
|
||||
from core.s3_keys import public_user_avatar_api_path, user_avatar_key
|
||||
from core.storage_s3 import delete_object_if_exists, get_object_bytes, put_object_bytes
|
||||
from core.security import get_current_user, validate_access_to_resource
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..user_tenant.models import UserTenant
|
||||
@@ -25,6 +31,10 @@ from .service import UserService
|
||||
|
||||
router = APIRouter(prefix="/users", tags=["Users"])
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_AVATAR_EXT = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
|
||||
|
||||
|
||||
@router.get("/stats", response_model=UserStatsDTO)
|
||||
def get_user_statistics(
|
||||
@@ -67,6 +77,52 @@ def list_users(
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/avatar/{tenant_id}/{keycloak_user_id}")
|
||||
def get_user_avatar_image(
|
||||
tenant_id: int,
|
||||
keycloak_user_id: str,
|
||||
db: Session = Depends(get_core_db),
|
||||
):
|
||||
"""
|
||||
Sirve la imagen de avatar (público para poder usarla en <img src> sin Bearer).
|
||||
El almacenamiento interno puede ser clave S3 o ruta bajo uploads/.
|
||||
"""
|
||||
ut = (
|
||||
db.query(UserTenant)
|
||||
.filter(
|
||||
UserTenant.tenant_id == tenant_id,
|
||||
UserTenant.keycloak_user_id == keycloak_user_id,
|
||||
UserTenant.is_active == True,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not ut or not ut.avatar_url:
|
||||
raise HTTPException(status_code=404, detail="Avatar not found")
|
||||
|
||||
raw = ut.avatar_url
|
||||
if raw.startswith("tenants/"):
|
||||
try:
|
||||
data = get_object_bytes(raw)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=404, detail="Avatar not found")
|
||||
media = mimetypes.guess_type(raw)[0] or "image/jpeg"
|
||||
return Response(content=data, media_type=media)
|
||||
|
||||
rel = raw.lstrip("/")
|
||||
path = Path(rel)
|
||||
if not path.is_file():
|
||||
path = Path.cwd() / rel
|
||||
if not path.is_file():
|
||||
alt = Path("/app") / rel
|
||||
if alt.is_file():
|
||||
path = alt
|
||||
if not path.is_file():
|
||||
raise HTTPException(status_code=404, detail="Avatar file not found")
|
||||
data = path.read_bytes()
|
||||
media = mimetypes.guess_type(str(path))[0] or "image/jpeg"
|
||||
return Response(content=data, media_type=media)
|
||||
|
||||
|
||||
@router.get("/{user_id}", response_model=UserResponseDTO)
|
||||
def get_user(
|
||||
user_id: str,
|
||||
@@ -282,33 +338,74 @@ async def upload_avatar(
|
||||
db: Session = Depends(get_core_db),
|
||||
):
|
||||
"""
|
||||
Sube un avatar para el usuario actual
|
||||
Retorna la URL del avatar subido
|
||||
Sube un avatar para el usuario actual.
|
||||
Con MinIO guarda en tenants/{tid}/users/{sub}/avatar.{ext} y persiste la clave en UserTenant.
|
||||
Retorna URL pública para <img src> (GET /users/avatar/...).
|
||||
"""
|
||||
# Validar tipo de archivo
|
||||
if not file.content_type or not file.content_type.startswith("image/"):
|
||||
raise HTTPException(status_code=400, detail="El archivo debe ser una imagen")
|
||||
|
||||
# Validar tamaño (max 2MB)
|
||||
keycloak_user_id = current_user.get("sub")
|
||||
if not keycloak_user_id:
|
||||
raise HTTPException(status_code=400, detail="User ID not found in token")
|
||||
|
||||
user_tenant = (
|
||||
db.query(UserTenant)
|
||||
.filter(
|
||||
UserTenant.keycloak_user_id == keycloak_user_id,
|
||||
UserTenant.is_active == True,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not user_tenant:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="User does not belong to any tenant"
|
||||
)
|
||||
|
||||
ext = Path(file.filename or "image.jpg").suffix.lower() or ".jpg"
|
||||
if ext not in _AVATAR_EXT:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Extensión no permitida. Use: {', '.join(sorted(_AVATAR_EXT))}",
|
||||
)
|
||||
|
||||
contents = await file.read()
|
||||
if len(contents) > 2 * 1024 * 1024:
|
||||
raise HTTPException(status_code=400, detail="La imagen debe ser menor a 2MB")
|
||||
|
||||
# Crear directorio si no existe
|
||||
upload_dir = Path("/app/uploads/avatars")
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
tenant_id = user_tenant.tenant_id
|
||||
|
||||
# Generar nombre con keycloak_user_id (sobrescribe si existe)
|
||||
keycloak_user_id = current_user.get("sub")
|
||||
ext = Path(file.filename or "image.jpg").suffix
|
||||
filename = f"{keycloak_user_id}{ext}"
|
||||
file_path = upload_dir / filename
|
||||
try:
|
||||
if settings.use_s3_object_storage:
|
||||
if user_tenant.avatar_url and str(user_tenant.avatar_url).startswith(
|
||||
"tenants/"
|
||||
):
|
||||
delete_object_if_exists(str(user_tenant.avatar_url))
|
||||
key = user_avatar_key(tenant_id, keycloak_user_id, ext)
|
||||
ct = file.content_type or mimetypes.guess_type(f"x{ext}")[0] or "image/jpeg"
|
||||
put_object_bytes(key, contents, content_type=ct)
|
||||
user_tenant.avatar_url = key
|
||||
logger.info(
|
||||
"User avatar stored in S3 key=%s bytes=%s", key, len(contents)
|
||||
)
|
||||
else:
|
||||
upload_dir = Path("uploads/avatars")
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"{keycloak_user_id}{ext}"
|
||||
file_path = upload_dir / filename
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(contents)
|
||||
user_tenant.avatar_url = f"/uploads/avatars/{filename}"
|
||||
|
||||
# Guardar archivo
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(contents)
|
||||
db.add(user_tenant)
|
||||
db.commit()
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error al guardar el avatar: {str(e)}"
|
||||
) from e
|
||||
|
||||
# Retornar URL relativa
|
||||
avatar_url = f"/uploads/avatars/{filename}"
|
||||
|
||||
return {"avatar_url": avatar_url}
|
||||
public_url = public_user_avatar_api_path(tenant_id, keycloak_user_id)
|
||||
return {"avatar_url": public_url}
|
||||
|
||||
@@ -43,9 +43,16 @@ def _normalize_keycloak_user(
|
||||
|
||||
# Agregar campos de perfil si user_tenant está disponible
|
||||
if user_tenant:
|
||||
avatar_out = user_tenant.avatar_url
|
||||
if avatar_out and str(avatar_out).startswith("tenants/"):
|
||||
from core.s3_keys import public_user_avatar_api_path
|
||||
|
||||
avatar_out = public_user_avatar_api_path(
|
||||
user_tenant.tenant_id, user_tenant.keycloak_user_id
|
||||
)
|
||||
normalized.update(
|
||||
{
|
||||
"avatar_url": user_tenant.avatar_url,
|
||||
"avatar_url": avatar_out,
|
||||
"phone": user_tenant.phone,
|
||||
"bio": user_tenant.bio,
|
||||
"preferences": user_tenant.preferences or {},
|
||||
@@ -435,7 +442,9 @@ class UserService:
|
||||
# Actualizar campos en UserTenant
|
||||
if role is not None:
|
||||
user_tenant.role = role
|
||||
if avatar_url is not None:
|
||||
if avatar_url is not None and not str(avatar_url).startswith(
|
||||
"/api/v1/core/users/avatar/"
|
||||
):
|
||||
user_tenant.avatar_url = avatar_url
|
||||
if phone is not None:
|
||||
user_tenant.phone = phone
|
||||
@@ -641,7 +650,9 @@ class UserService:
|
||||
self.keycloak_admin.update_user(keycloak_user_id, update_data)
|
||||
|
||||
# Actualizar campos de perfil en UserTenant
|
||||
if avatar_url is not None:
|
||||
if avatar_url is not None and not str(avatar_url).startswith(
|
||||
"/api/v1/core/users/avatar/"
|
||||
):
|
||||
user_tenant.avatar_url = avatar_url
|
||||
if phone is not None:
|
||||
user_tenant.phone = phone
|
||||
|
||||
Reference in New Issue
Block a user