Mezlca de migraciones mas recientes

This commit is contained in:
2026-04-07 16:13:01 -05:00
120 changed files with 5446 additions and 1215 deletions

View File

@@ -1,19 +1,184 @@
"""
Audit Log Router
"""
from typing import List, Optional
from datetime import date
from fastapi import APIRouter, Depends, Query, HTTPException
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from sqlalchemy import or_, desc, distinct
from core.database import get_core_db
from core.security import get_current_user # Assuming this exists
from core.security import get_current_user, get_tenant_from_token
from core.storage_s3 import get_object_bytes, list_objects_tree, should_ensure_s3_bucket
from .models import AuditLog
from .schemas import AuditLogListResponse, AuditLogResponse, AuditLogDetailResponse
from .schemas import (
AuditFileBreadcrumb,
AuditFileBrowserResponse,
AuditFileFolderItem,
AuditFileObjectItem,
AuditLogDetailResponse,
AuditLogListResponse,
)
from api.v1.modules.a76.general_catalogs.company.models import Company
router = APIRouter()
_SEGMENT_LABELS = {
"tenants": "Espacio",
"companies": "Companias",
"users": "Usuarios",
"imports": "Importaciones",
"csv": "Archivos CSV",
"branding": "Logotipos",
"certificates": "Certificados",
"customs_brokers": "Agentes aduanales",
"keys": "Llaves",
"cove": "COVE",
"doda": "DODA",
"system": "Sistema",
"help": "Ayuda",
}
def _tenant_id_from_user(current_user: Dict[str, Any]) -> int:
tenant_id = get_tenant_from_token(current_user) or current_user.get("tenant_id")
if not tenant_id:
raise HTTPException(status_code=401, detail="User context is invalid")
return int(tenant_id)
def _normalize_relative_path(raw: Optional[str]) -> str:
if not raw:
return ""
val = raw.strip().strip("/")
if not val:
return ""
if ".." in val or "\\" in val:
raise HTTPException(status_code=400, detail="Invalid path")
parts = [p for p in val.split("/") if p]
for part in parts:
if part in (".", ".."):
raise HTTPException(status_code=400, detail="Invalid path segment")
return "/".join(parts)
def _tenant_prefix(tenant_id: int) -> str:
return f"tenants/{tenant_id}/"
def _relative_from_tenant_prefix(key: str, tenant_prefix: str) -> str:
if not key.startswith(tenant_prefix):
raise HTTPException(status_code=403, detail="Access denied to object key")
return key[len(tenant_prefix) :].strip("/")
def _companies_map(db: Session, tenant_id: int) -> Dict[str, str]:
rows = (
db.query(Company.id, Company.name)
.filter(Company.tenant_id == tenant_id, Company.deleted_at.is_(None))
.all()
)
out: Dict[str, str] = {}
for company_id, company_name in rows:
if company_id is None:
continue
safe_name = (company_name or "").strip()
out[str(company_id)] = safe_name or "Compania"
return out
def _display_segment(
part: str,
prev_part: Optional[str],
company_names: Dict[str, str],
current_user_id: Optional[str] = None,
current_user_label: Optional[str] = None,
) -> str:
if prev_part == "companies":
return company_names.get(part, "Compania")
if prev_part == "users":
# Para carpetas de usuarios, mostrar un nombre amigable:
# - Si es el propio usuario actual, usar preferred_username/email/nombre.
# - Para otros IDs (UUIDs) mostrar un label genérico.
if current_user_id and part == str(current_user_id):
return (current_user_label or "").strip() or "Usuario"
return "Usuario"
if part in _SEGMENT_LABELS:
return _SEGMENT_LABELS[part]
# Evita exponer IDs puros en UI.
if part.isdigit():
return "Elemento"
return part.replace("_", " ").strip().title() or "Elemento"
def _display_path(
rel_path: str,
company_names: Dict[str, str],
current_user_id: Optional[str] = None,
current_user_label: Optional[str] = None,
) -> str:
if not rel_path:
return "Raiz de archivos"
parts = [p for p in rel_path.split("/") if p]
labels: List[str] = []
prev: Optional[str] = None
for part in parts:
labels.append(
_display_segment(
part,
prev,
company_names,
current_user_id=current_user_id,
current_user_label=current_user_label,
)
)
prev = part
return " / ".join(labels)
def _display_file_name(filename: str) -> str:
stem, dot, ext = filename.rpartition(".")
if not dot:
stem = filename
ext = ""
if stem.isdigit():
return f"Archivo{f'.{ext}' if ext else ''}"
return filename
def _build_breadcrumbs(
rel_path: str,
company_names: Dict[str, str],
current_user_id: Optional[str] = None,
current_user_label: Optional[str] = None,
) -> List[AuditFileBreadcrumb]:
breadcrumbs: List[AuditFileBreadcrumb] = [
AuditFileBreadcrumb(path="", display_name="Raiz de archivos")
]
if not rel_path:
return breadcrumbs
parts = [p for p in rel_path.split("/") if p]
prev: Optional[str] = None
acc: List[str] = []
for part in parts:
acc.append(part)
breadcrumbs.append(
AuditFileBreadcrumb(
path="/".join(acc),
display_name=_display_segment(
part,
prev,
company_names,
current_user_id=current_user_id,
current_user_label=current_user_label,
),
)
)
prev = part
return breadcrumbs
@router.get("/bitacora", response_model=AuditLogListResponse)
async def get_bitacora(
page: int = Query(1, ge=1),
@@ -91,3 +256,127 @@ async def get_audit_detail(spec_id: int, db: Session = Depends(get_core_db)):
if not log:
raise HTTPException(status_code=404, detail="Log entry not found")
return log
@router.get("/files", response_model=AuditFileBrowserResponse)
async def list_tenant_files(
path: Optional[str] = Query(default="", description="Ruta relativa de navegación."),
continuation_token: Optional[str] = Query(default=None),
max_keys: int = Query(default=100, ge=1, le=500),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Explorador de archivos de solo lectura para Auditoría.
"""
if not should_ensure_s3_bucket():
raise HTTPException(status_code=400, detail="S3 storage is disabled")
tenant_id = _tenant_id_from_user(current_user)
tenant_prefix = _tenant_prefix(tenant_id)
rel_path = _normalize_relative_path(path)
list_prefix = f"{tenant_prefix}{rel_path}/" if rel_path else tenant_prefix
# Datos del usuario actual para etiquetas amigables bajo /users/{id}/...
current_user_id = str(current_user.get("sub") or "")
current_user_label = (
(current_user.get("preferred_username") or "").strip()
or (current_user.get("name") or "").strip()
or (current_user.get("email") or "").strip()
or "Usuario"
)
data = list_objects_tree(
prefix=list_prefix,
delimiter="/",
max_keys=max_keys,
continuation_token=continuation_token,
)
company_names = _companies_map(db, tenant_id)
folders: List[AuditFileFolderItem] = []
for prefix in data.get("prefixes", []):
rel = _relative_from_tenant_prefix(prefix, tenant_prefix)
folders.append(
AuditFileFolderItem(
path=rel,
# Para el gestor de archivos mostramos el nombre amigable del último segmento
# (empresa, usuario actual, etc.), no el ID bruto.
display_name=_display_path(
rel,
company_names,
current_user_id=current_user_id,
current_user_label=current_user_label,
).split(" / ")[-1],
)
)
files: List[AuditFileObjectItem] = []
for obj in data.get("objects", []):
key = obj.get("key")
if not key:
continue
rel = _relative_from_tenant_prefix(key, tenant_prefix)
name = rel.rsplit("/", 1)[-1]
files.append(
AuditFileObjectItem(
path=rel,
display_name=_display_file_name(name),
size=int(obj.get("size", 0) or 0),
last_modified=obj.get("last_modified"),
)
)
return AuditFileBrowserResponse(
current_path=rel_path,
display_path=_display_path(
rel_path,
company_names,
current_user_id=current_user_id,
current_user_label=current_user_label,
),
breadcrumbs=_build_breadcrumbs(
rel_path,
company_names,
current_user_id=current_user_id,
current_user_label=current_user_label,
),
folders=sorted(folders, key=lambda x: x.display_name.lower()),
files=sorted(files, key=lambda x: x.display_name.lower()),
next_token=data.get("next_continuation_token"),
)
@router.get("/files/download")
async def download_tenant_file(
path: str = Query(..., description="Ruta relativa del archivo a descargar."),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Descarga segura (backend streaming) de archivos autorizados.
"""
if not should_ensure_s3_bucket():
raise HTTPException(status_code=400, detail="S3 storage is disabled")
tenant_id = _tenant_id_from_user(current_user)
tenant_prefix = _tenant_prefix(tenant_id)
rel_path = _normalize_relative_path(path)
if not rel_path or rel_path.endswith("/"):
raise HTTPException(status_code=400, detail="A file path is required")
object_key = f"{tenant_prefix}{rel_path}"
if not object_key.startswith(tenant_prefix):
raise HTTPException(status_code=403, detail="Access denied to object key")
try:
body = get_object_bytes(object_key)
except Exception as e:
raise HTTPException(status_code=404, detail=f"File not found: {e}") from e
filename = rel_path.rsplit("/", 1)[-1]
headers = {"Content-Disposition": f'attachment; filename="{filename}"'}
return StreamingResponse(
iter([body]),
media_type="application/octet-stream",
headers=headers,
)

View File

@@ -50,3 +50,29 @@ class AuditLogListResponse(BaseModel):
total: int
page: int
page_size: int
class AuditFileBreadcrumb(BaseModel):
path: str = Field(default="")
display_name: str
class AuditFileFolderItem(BaseModel):
path: str = Field(description="Ruta relativa interna del archivo, para navegación.")
display_name: str
class AuditFileObjectItem(BaseModel):
path: str = Field(description="Ruta relativa interna del archivo, para descarga.")
display_name: str
size: int
last_modified: Optional[datetime] = None
class AuditFileBrowserResponse(BaseModel):
current_path: str = Field(default="")
display_path: str = Field(default="")
breadcrumbs: List[AuditFileBreadcrumb]
folders: List[AuditFileFolderItem]
files: List[AuditFileObjectItem]
next_token: Optional[str] = None

View File

@@ -1,16 +1,69 @@
from typing import Dict, Any
from fastapi import APIRouter, Depends, HTTPException, Query
import logging
import mimetypes
import os
from datetime import datetime
from typing import Any, Dict, Optional
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status
from sqlalchemy.orm import Session
from core.config import settings
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from core.s3_keys import (
customs_broker_vu_certificate_key,
customs_broker_vu_cove_key,
customs_broker_vu_doda_certificate_key,
customs_broker_vu_doda_cove_key,
customs_broker_vu_doda_private_key_key,
customs_broker_vu_private_key_key,
)
from core.security import get_current_user, get_tenant_from_token, validate_access_to_resource
from core.storage_s3 import delete_object_if_exists, put_object_bytes
from api.v1.common.tenant_crud_routes import TenantCRUDRoutes
from . import dto, services
from ..layouts_csv.customs_brokers.routes import router as imports_router
logger = logging.getLogger(__name__)
router = APIRouter()
MAX_VU_CER_KEY_BYTES = 5 * 1024 * 1024 # 5 MB
MAX_COVE_BYTES = 15 * 1024 * 1024 # 15 MB (xml/zip)
def _resolve_tenant_id_int(current_user: dict) -> int:
tid = get_tenant_from_token(current_user)
if tid is not None:
return int(tid)
raw = current_user.get("tenant_id")
if isinstance(raw, list) and raw:
raw = raw[0]
if raw is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Tenant ID not found in user data",
)
try:
return int(raw)
except (TypeError, ValueError):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid tenant ID in token",
)
def _remove_stored_vu_path(ref: Optional[str]) -> None:
if not ref:
return
if ref.startswith("tenants/"):
delete_object_if_exists(ref)
elif os.path.isfile(ref):
try:
os.remove(ref)
except OSError:
pass
# CSV import (mismo flujo que a76.imports: upload → scan → commit)
router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"])
@@ -112,4 +165,246 @@ def update_customs_broker_personnel(
raise HTTPException(
status_code=404, detail="Customs Broker Personnel not found"
)
return updated_personnel
return updated_personnel
@router.post(
"/customs-brokers/{broker_key}/vu/upload",
summary="Sube VU/DODA (CER, KEY, COVE) al bucket bajo tenants/.../customs_brokers/{id}/...",
)
async def upload_customs_broker_vu_file(
broker_key: str,
file_kind: str = Query(
...,
description=(
"certificate (.cer), key (.key), cove (xml/zip/txt/pdf/json), "
"doda_certificate (.cer), doda_key (.key), doda_cove (xml/zip/txt/pdf/json)"
),
),
company_id: int = Query(..., description="Company ID"),
file: UploadFile = File(...),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Persiste el archivo bajo la misma jerarquía que logos/avatares (tenant/company/...).
Guarda la clave S3 o ruta local en:
- VU: certificate_path, key_path, xml_files_path
- DODA: doda_certificate_path, doda_key_path, doda_xml_files_path
"""
validate_access_to_resource(db, company_id, current_user)
tenant_id = _resolve_tenant_id_int(current_user)
broker = services.CustomsBrokerService.get_by_id(db, broker_key, tenant_id, company_id)
if not broker:
raise HTTPException(status_code=404, detail="Customs Broker not found")
fk = file_kind.lower().strip()
if fk not in (
"certificate",
"key",
"cove",
"doda_certificate",
"doda_key",
"doda_cove",
):
raise HTTPException(
status_code=400,
detail=(
"file_kind must be certificate, key, cove, "
"doda_certificate, doda_key, or doda_cove"
),
)
content = await file.read()
max_bytes = MAX_COVE_BYTES if fk in ("cove", "doda_cove") else MAX_VU_CER_KEY_BYTES
if len(content) > max_bytes:
raise HTTPException(
status_code=400,
detail=f"File too large (max {max_bytes // (1024 * 1024)} MB)",
)
file_ext = os.path.splitext(file.filename or "")[1].lower()
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
vu = services.CustomsBrokerVUService.ensure_vu_for_broker(db, broker)
broker_id = broker.id
field_name: str
stored: str
try:
if settings.use_s3_object_storage:
if fk == "certificate":
if file_ext != ".cer":
raise HTTPException(status_code=400, detail="certificate must be .cer")
key = customs_broker_vu_certificate_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
ct = "application/x-x509-ca-cert"
field_name = "certificate_path"
elif fk == "key":
if file_ext != ".key":
raise HTTPException(status_code=400, detail="key must be .key")
key = customs_broker_vu_private_key_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
ct = "application/pkcs8"
field_name = "key_path"
elif fk == "cove":
key = customs_broker_vu_cove_key(
tenant_id,
company_id,
broker_id,
timestamp,
file.filename or "cove.xml",
)
ct = (
file.content_type
or mimetypes.guess_type(file.filename or "")[0]
or "application/octet-stream"
)
field_name = "xml_files_path"
elif fk == "doda_certificate":
if file_ext != ".cer":
raise HTTPException(status_code=400, detail="doda_certificate must be .cer")
key = customs_broker_vu_doda_certificate_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
ct = "application/x-x509-ca-cert"
field_name = "doda_certificate_path"
elif fk == "doda_key":
if file_ext != ".key":
raise HTTPException(status_code=400, detail="doda_key must be .key")
key = customs_broker_vu_doda_private_key_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
ct = "application/pkcs8"
field_name = "doda_key_path"
else:
key = customs_broker_vu_doda_cove_key(
tenant_id,
company_id,
broker_id,
timestamp,
file.filename or "doda.xml",
)
ct = (
file.content_type
or mimetypes.guess_type(file.filename or "")[0]
or "application/octet-stream"
)
field_name = "doda_xml_files_path"
old = getattr(vu, field_name)
_remove_stored_vu_path(old)
put_object_bytes(key, content, content_type=ct)
logger.info(
"Customs broker VU upload kind=%s key=%s bytes=%s",
fk,
key,
len(content),
)
stored = key
else:
base = os.path.join(
"uploads", "customs_brokers", str(company_id), str(broker_id)
)
if fk == "certificate":
if file_ext != ".cer":
raise HTTPException(status_code=400, detail="certificate must be .cer")
key = customs_broker_vu_certificate_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
field_name = "certificate_path"
subdir = "certificates"
elif fk == "key":
if file_ext != ".key":
raise HTTPException(status_code=400, detail="key must be .key")
key = customs_broker_vu_private_key_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
field_name = "key_path"
subdir = "keys"
elif fk == "cove":
try:
key = customs_broker_vu_cove_key(
tenant_id,
company_id,
broker_id,
timestamp,
file.filename or "cove.xml",
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
field_name = "xml_files_path"
subdir = "cove"
elif fk == "doda_certificate":
if file_ext != ".cer":
raise HTTPException(status_code=400, detail="doda_certificate must be .cer")
key = customs_broker_vu_doda_certificate_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
field_name = "doda_certificate_path"
subdir = "doda/certificates"
elif fk == "doda_key":
if file_ext != ".key":
raise HTTPException(status_code=400, detail="doda_key must be .key")
key = customs_broker_vu_doda_private_key_key(
tenant_id, company_id, broker_id, timestamp, file_ext
)
field_name = "doda_key_path"
subdir = "doda/keys"
else:
try:
key = customs_broker_vu_doda_cove_key(
tenant_id,
company_id,
broker_id,
timestamp,
file.filename or "doda.xml",
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
field_name = "doda_xml_files_path"
subdir = "doda/cove"
fname = key.rsplit("/", 1)[-1]
dest_dir = os.path.join(base, subdir)
os.makedirs(dest_dir, exist_ok=True)
path = os.path.join(dest_dir, fname)
old = getattr(vu, field_name)
_remove_stored_vu_path(old)
with open(path, "wb") as f:
f.write(content)
logger.info(
"Customs broker VU upload kind=%s path=%s bytes=%s",
fk,
path,
len(content),
)
stored = path
setattr(vu, field_name, stored)
db.add(vu)
db.commit()
db.refresh(vu)
except HTTPException:
db.rollback()
raise
except ValueError as e:
db.rollback()
raise HTTPException(status_code=400, detail=str(e)) from e
except Exception as e:
db.rollback()
raise HTTPException(
status_code=500, detail=f"Error saving file: {str(e)}"
) from e
return {
"message": "File uploaded successfully",
"file_kind": fk,
"field": field_name,
"path": stored,
"broker_key": broker_key,
"company_id": company_id,
}

View File

@@ -160,6 +160,26 @@ class CustomsBrokerVUService:
db.commit()
return vu
@staticmethod
def ensure_vu_for_broker(db: Session, broker: models.CustomsBroker) -> models.CustomsBrokerVU:
"""Crea fila VU vacía si no existe (p. ej. antes de subir CER/KEY/COVE al bucket)."""
vu = (
db.query(models.CustomsBrokerVU)
.filter(models.CustomsBrokerVU.customs_broker_id == broker.id)
.first()
)
if vu:
return vu
vu = models.CustomsBrokerVU(
customs_broker_id=broker.id,
tenant_id=broker.tenant_id,
company_id=broker.company_id,
)
db.add(vu)
db.commit()
db.refresh(vu)
return vu
class CustomsBrokerPersonnelService:
@staticmethod

View File

@@ -57,7 +57,7 @@ class CompanyCreateDTO(BaseModel):
)
# Configuration
logo: Optional[str] = Field(None, max_length=255, description="Company logo")
logo: Optional[str] = Field(None, max_length=512, description="Company logo (path local o clave S3)")
has_express_line: Optional[bool] = Field(None, description="Has express line")
order_format_type: Optional[str] = Field(
None, max_length=19, description="Order format type"

View File

@@ -60,7 +60,7 @@ class Company(Base, TimestampMixin):
position: Mapped[Optional[str]] = mapped_column(String(30))
# Configuración básica
logo: Mapped[Optional[str]] = mapped_column(String(255))
logo: Mapped[Optional[str]] = mapped_column(String(512))
has_express_line: Mapped[Optional[bool]] = mapped_column(Boolean, default=False, server_default="false")
order_format_type: Mapped[Optional[str]] = mapped_column(String(19))
is_service_company: Mapped[Optional[bool]] = mapped_column(Boolean, default=False, server_default="false")

View File

@@ -2,20 +2,23 @@
Rutas para gestión de empresa
"""
import logging
import mimetypes
import os
import shutil
from typing import List, Optional
import os
import shutil
from pathlib import Path
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, status, File, UploadFile
from fastapi.responses import FileResponse
from fastapi.responses import FileResponse, Response
from sqlalchemy.orm import Session
from core.config import settings
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from core.s3_keys import company_certificate_key, company_logo_key
from core.storage_s3 import delete_object_if_exists, get_object_bytes, put_object_bytes
from core.security import get_current_user, get_tenant_from_token, validate_access_to_resource
from .....common.tenant_crud_routes import TenantCRUDRoutes
from .dto import CompanyCreateDTO, CompanyResponseDTO, CompanyUpdateDTO
from .models import Company
@@ -26,6 +29,44 @@ UPLOAD_DIR = "uploads/companies"
ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
MAX_FILE_SIZE = 5 * 1024 * 1024 # 5MB
logger = logging.getLogger(__name__)
def _resolve_tenant_id_int(current_user: dict) -> int:
"""Misma lógica que validate_access_to_resource: entero estable para BD y claves S3."""
tid = get_tenant_from_token(current_user)
if tid is not None:
return int(tid)
raw = current_user.get("tenant_id")
if isinstance(raw, list) and raw:
raw = raw[0]
if raw is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Tenant ID not found in user data",
)
try:
return int(raw)
except (TypeError, ValueError):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid tenant ID in token",
)
def _is_s3_object_key(ref: Optional[str]) -> bool:
return bool(ref and ref.startswith("tenants/"))
def _remove_stored_file(ref: str) -> None:
if _is_s3_object_key(ref):
delete_object_if_exists(ref)
elif ref and os.path.isfile(ref):
try:
os.remove(ref)
except OSError:
pass
# Main router that includes base CRUD
router = APIRouter(prefix="/company")
@@ -219,14 +260,20 @@ async def get_company_logo_image(
if not company or not company.logo:
raise HTTPException(status_code=404, detail="Logo not found")
if _is_s3_object_key(company.logo):
try:
data = get_object_bytes(company.logo)
except Exception:
raise HTTPException(status_code=404, detail="Logo file not found on server")
media = mimetypes.guess_type(company.logo)[0] or "image/jpeg"
return Response(content=data, media_type=media)
file_path = Path(company.logo)
if not file_path.exists():
# Fallback for old paths or moved files
# Check if it exists in the 'standard' location even if DB thinks otherwise
standard_path = Path(f"app_data/logos/{company_id}") / file_path.name
if standard_path.exists():
return FileResponse(standard_path)
raise HTTPException(status_code=404, detail="Logo file not found on server")
return FileResponse(file_path)
@@ -272,12 +319,7 @@ async def upload_company_logo(
current_user: dict = Depends(get_current_user),
):
"""Upload a logo for a company"""
tenant_id = current_user.get("tenant_id")
if not tenant_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Tenant ID not found in user data",
)
tenant_id = _resolve_tenant_id_int(current_user)
# Validar que la empresa existe
company = CompanyService.get_by_id(db, company_id, tenant_id, 0)
@@ -303,35 +345,34 @@ async def upload_company_logo(
detail=f"File too large. Maximum size: {MAX_FILE_SIZE / 1024 / 1024}MB",
)
# Crear directorio si no existe
os.makedirs(UPLOAD_DIR, exist_ok=True)
# Eliminar logo anterior si existe
if company.logo:
old_logo_path = company.logo
if os.path.exists(old_logo_path):
try:
os.remove(old_logo_path)
except Exception:
pass # No es crítico si falla
_remove_stored_file(company.logo)
# Generar nombre único
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"company_{company_id}_{timestamp}{file_ext}"
file_path = os.path.join(UPLOAD_DIR, filename)
filename = f"logo_{company_id}_{timestamp}{file_ext}"
# Guardar archivo
try:
await file.seek(0)
with open(file_path, "wb") as buffer:
shutil.copyfileobj(file.file, buffer)
if settings.use_s3_object_storage:
ct = mimetypes.guess_type(filename)[0] or "image/jpeg"
key = company_logo_key(tenant_id, company_id, filename)
put_object_bytes(key, content, content_type=ct)
logger.info(
"Company logo stored in S3 key=%s bytes=%s",
key,
len(content),
)
file_path = key
else:
os.makedirs(UPLOAD_DIR, exist_ok=True)
file_path = os.path.join(UPLOAD_DIR, filename)
with open(file_path, "wb") as f:
f.write(content)
except Exception as e:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error saving file: {str(e)}",
)
# Actualizar la empresa con la ruta del logo
update_data = CompanyUpdateDTO(logo=file_path)
service = CompanyService(db)
updated_company = service.update(db, company_id, tenant_id, 0, update_data)
@@ -359,12 +400,7 @@ async def upload_company_certificate(
Upload a certificate for a company
certificate_type: fiel_cer, fiel_key, cfdi_cert_cer, cfdi_cert_key, cancel_cer, cancel_key
"""
tenant_id = current_user.get("tenant_id")
if not tenant_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Tenant ID not found in user data",
)
tenant_id = _resolve_tenant_id_int(current_user)
# Validar que la empresa existe
service = CompanyService(db)
@@ -410,27 +446,38 @@ async def upload_company_certificate(
detail=f"File too large. Maximum size: {MAX_FILE_SIZE / 1024 / 1024}MB",
)
# Crear directorio si no existe
certs_dir = os.path.join(UPLOAD_DIR, str(company_id), "certificates")
os.makedirs(certs_dir, exist_ok=True)
# Generar nombre único
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"{certificate_type}_{timestamp}{file_ext}"
file_path = os.path.join(certs_dir, filename)
# Guardar archivo
try:
await file.seek(0)
with open(file_path, "wb") as buffer:
shutil.copyfileobj(file.file, buffer)
if settings.use_s3_object_storage:
key = company_certificate_key(
tenant_id, company_id, certificate_type, timestamp, file_ext
)
ct = (
"application/x-x509-ca-cert"
if file_ext == ".cer"
else "application/pkcs8"
)
put_object_bytes(key, content, content_type=ct)
logger.info(
"Company certificate stored in S3 key=%s bytes=%s",
key,
len(content),
)
file_path = key
else:
certs_dir = os.path.join(UPLOAD_DIR, str(company_id), "certificates")
os.makedirs(certs_dir, exist_ok=True)
filename = f"{certificate_type}_{timestamp}{file_ext}"
file_path = os.path.join(certs_dir, filename)
with open(file_path, "wb") as f:
f.write(content)
except Exception as e:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error saving file: {str(e)}",
)
# Actualizar la base de datos
service.upload_certificate(company_id, certificate_type, file_path, tenant_id)
return {

View File

@@ -10,6 +10,10 @@ from fastapi import HTTPException
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from core.config import settings
from core.s3_keys import tenant_company_prefix
from core.storage_s3 import delete_objects_with_prefix
from .dto import CompanyCreateDTO, CompanyResponseDTO, CompanyUpdateDTO
from .models import Company
from ...audit_log.services.service import AuditService
@@ -594,9 +598,24 @@ class CompanyService:
# ----------------------
try:
# 1) Borrado lógico en base de datos
company.deleted_at = datetime.utcnow()
db.flush()
# 2) Limpieza de objetos S3/MinIO asociados a la compañía
if settings.use_s3_object_storage:
try:
prefix = tenant_company_prefix(tenant_id, company_id)
delete_objects_with_prefix(prefix)
except Exception as e:
# No bloquear la eliminación lógica si falla la limpieza de objetos
logger.error(
"Error deleting S3 objects for company %s (tenant %s): %s",
company_id,
tenant_id,
e,
)
db.commit()
# --- Audit Log ---
@@ -678,6 +697,11 @@ class CompanyService:
try:
if target_cert:
old_path = getattr(target_cert, field_to_update, None)
if old_path and str(old_path).startswith("tenants/"):
from core.storage_s3 import delete_object_if_exists
delete_object_if_exists(str(old_path))
# Si existe, actualizamos
setattr(target_cert, field_to_update, file_path)
else:

View File

@@ -13,12 +13,19 @@ from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRat
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
from api.v1.modules.a76.transportation.transporters.models import Transporter
from api.v1.modules.a76.transportation.vehicles.models import Vehicle
from api.v1.modules.a76.transportation.trailers.models import Trailer
from api.v1.modules.a76.transportation.drivers.models import Driver
from api.v1.modules.a76.manifests.manifest.models import Manifest
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
def _logistics_str_nonempty(value) -> bool:
return value is not None and str(value).strip() != ""
def _normalize_invoice_currency_value(value) -> str:
"""Lowercase currency code (foreign/local/manual) for comparisons."""
if value is None or value == "":
@@ -344,25 +351,40 @@ def validate_common(
code="INVALID_VALUE",
value=invoice.compliance_mx.remesa,
)
else:
# Misma remesa puede repetirse en distintos pedimentos; solo debe ser única
# por (pedimento_id, empresa, tenant). Creación: existing_invoice es None.
current_invoice_id = None
if existing_invoice is not None:
current_invoice_id = existing_invoice.id
else:
current_invoice_id = getattr(invoice, "id", None)
duplicated_remesa = (
db.query(InvoiceComplianceMx)
.filter(
InvoiceComplianceMx.remesa == invoice.compliance_mx.remesa,
InvoiceComplianceMx.tenant_id == tenant_id,
InvoiceComplianceMx.company_id == company_id,
)
.first()
)
if duplicated_remesa and hasattr(invoice, "id"):
if invoice.id != duplicated_remesa.invoice_id:
errors.add_error(
field="compliance_mx.remesa",
message="El valor de Remesa ya está asociado a otro Pedimento.",
solution=["Proporciona un valor único para Remesa"],
code="DUPLICATE_VALUE",
value=invoice.compliance_mx.remesa,
duplicated_remesa = (
db.query(InvoiceComplianceMx)
.filter(
InvoiceComplianceMx.remesa == invoice.compliance_mx.remesa,
InvoiceComplianceMx.pedimento_id
== invoice.compliance_mx.pedimento_id,
InvoiceComplianceMx.tenant_id == tenant_id,
InvoiceComplianceMx.company_id == company_id,
)
.first()
)
if duplicated_remesa and (
current_invoice_id is None
or duplicated_remesa.invoice_id != current_invoice_id
):
errors.add_error(
field="compliance_mx.remesa",
message=(
"El valor de Remesa ya está registrado para este mismo Pedimento "
"en otra factura."
),
solution=["Usa otra remesa o revisa la factura que ya la tiene capturada"],
code="DUPLICATE_VALUE",
value=invoice.compliance_mx.remesa,
)
# Financials checks (if provided)
if invoice.financials:
@@ -511,16 +533,31 @@ def validate_common(
)
if invoice.logistics:
carrier_exists = None
if invoice.logistics.carrier_id:
carrier_exists = (
db.query(Transporter)
.filter(
Transporter.id == invoice.logistics.carrier_id,
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
# `carrier_id` is the frontend/export key (string). Prefer validating by the internal int
# when present, otherwise validate by `transporter_key`.
if getattr(invoice.logistics, "carrier_int_id", None) is not None:
carrier_exists = (
db.query(Transporter)
.filter(
Transporter.transporter_id
== invoice.logistics.carrier_int_id,
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
)
.first()
)
else:
carrier_exists = (
db.query(Transporter)
.filter(
Transporter.transporter_key == invoice.logistics.carrier_id,
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
)
.first()
)
.first()
)
if not carrier_exists:
errors.add_error(
field="logistics.carrier_id",
@@ -528,8 +565,36 @@ def validate_common(
solution=["Verifica el ID del Transportista", "Revisa el catálogo"],
code="NOT_FOUND",
value=invoice.logistics.carrier_id,
)
)
if carrier_exists and _logistics_str_nonempty(
getattr(invoice.logistics, "driver_name", None)
):
driver_row = (
db.query(Driver)
.filter(
Driver.transporter_key == str(invoice.logistics.carrier_id).strip(),
Driver.driver_name
== str(invoice.logistics.driver_name).strip(),
Driver.tenant_id == tenant_id,
Driver.company_id == company_id,
)
.first()
)
if not driver_row:
errors.add_error(
field="logistics.driver_name",
message=(
"El conductor no existe en el catálogo de conductores "
"para el transportista indicado."
),
solution=[
"Registra el conductor en el catálogo o elige uno de la lista"
],
code="NOT_FOUND",
value=invoice.logistics.driver_name,
)
if invoice.logistics.transport_type not in [t.value for t in TransportType]:
errors.add_error(
field="logistics.transport_type",
@@ -541,31 +606,138 @@ def validate_common(
value=invoice.logistics.transport_type,
)
else:
if (
invoice.logistics.transport_type == "none"
and invoice.logistics.transport_num
):
errors.add_error(
field="logistics.transport_num",
message="El Número de Transporte no debe proporcionarse cuando el Tipo de Transporte es 'none'.",
solution=[
"Elimina el Número de Transporte o selecciona un Tipo de Transporte válido"
],
code="INVALID_VALUE",
value=invoice.logistics.transport_num,
)
else:
if (
not invoice.logistics.transport_num
and invoice.logistics.transport_type != "none"
):
errors.add_error(
field="logistics.transport_num",
message="El Número de Transporte es obligatorio cuando se proporciona un Tipo de Transporte distinto de 'none'.",
solution=["Proporciona un Número de Transporte válido"],
code="REQUIRED_FIELD",
value=invoice.logistics.transport_num,
# Vehículo: solo catálogo si viene informado (cualquier tipo de transporte).
tid = getattr(invoice.logistics, "transport_int_id", None)
tcode = invoice.logistics.transport_id
if tid is not None:
v_by_int = (
db.query(Vehicle)
.filter(
Vehicle.vehicle_id == tid,
Vehicle.tenant_id == tenant_id,
Vehicle.company_id == company_id,
)
.first()
)
if not v_by_int:
errors.add_error(
field="logistics.transport_int_id",
message="El Vehículo (transporte) no existe en el catálogo.",
solution=[
"Verifica el ID interno del vehículo o usa la clave de transporte"
],
code="NOT_FOUND",
value=tid,
)
elif _logistics_str_nonempty(tcode):
v_by_key = (
db.query(Vehicle)
.filter(
func.upper(Vehicle.vehicle_key)
== str(tcode).strip().upper(),
Vehicle.tenant_id == tenant_id,
Vehicle.company_id == company_id,
)
.first()
)
if not v_by_key:
errors.add_error(
field="logistics.transport_id",
message="El Vehículo (clave transporte) no existe en el catálogo.",
solution=["Verifica la clave en el catálogo de vehículos"],
code="NOT_FOUND",
value=tcode,
)
# Remolque: obligatorio si tipo != 'none'; con 'none' solo se valida catálogo si hay dato.
tr_int = getattr(invoice.logistics, "trailer_int_id", None)
tr_num = invoice.logistics.trailer_num
if invoice.logistics.transport_type != "none":
if tr_int is not None:
tr = (
db.query(Trailer)
.filter(
Trailer.trailer_id == tr_int,
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if not tr:
errors.add_error(
field="logistics.trailer_int_id",
message="El Remolque no existe en el catálogo.",
solution=["Verifica el ID interno del remolque"],
code="NOT_FOUND",
value=tr_int,
)
elif not _logistics_str_nonempty(tr_num):
errors.add_error(
field="logistics.trailer_num",
message=(
"El Remolque es obligatorio cuando el Tipo de Transporte es distinto de 'none'."
),
solution=["Selecciona un remolque del catálogo"],
code="REQUIRED_FIELD",
value=tr_num,
)
else:
tr = (
db.query(Trailer)
.filter(
func.upper(Trailer.trailer_number)
== str(tr_num).strip().upper(),
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if not tr:
errors.add_error(
field="logistics.trailer_num",
message="El Remolque no existe en el catálogo.",
solution=["Verifica el número de remolque"],
code="NOT_FOUND",
value=tr_num,
)
else:
if tr_int is not None:
tr = (
db.query(Trailer)
.filter(
Trailer.trailer_id == tr_int,
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if not tr:
errors.add_error(
field="logistics.trailer_int_id",
message="El Remolque no existe en el catálogo.",
solution=["Verifica el ID interno del remolque"],
code="NOT_FOUND",
value=tr_int,
)
elif _logistics_str_nonempty(tr_num):
tr = (
db.query(Trailer)
.filter(
func.upper(Trailer.trailer_number)
== str(tr_num).strip().upper(),
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if not tr:
errors.add_error(
field="logistics.trailer_num",
message="El Remolque no existe en el catálogo.",
solution=["Verifica el número de remolque"],
code="NOT_FOUND",
value=tr_num,
)
if invoice.financials:
submitted = invoice.financials.currency

View File

@@ -563,9 +563,15 @@ class InvoiceLogistics(Base, TenantScopedMixin, TimestampMixin):
carrier_id: Mapped[Optional[str]] = mapped_column(
String(10)
) # TRANSPORTISTA / Transportista
carrier_int_id: Mapped[Optional[int]] = mapped_column(
BigInteger, ForeignKey("a76.transporter.transporter_id"), nullable=True
) # Internal surrogate ID for Transporter (by transporter_key)
transport_id: Mapped[Optional[str]] = mapped_column(
String(10)
) # NUMTRAILER / Transportista
transport_int_id: Mapped[Optional[int]] = mapped_column(
BigInteger, ForeignKey("a76.vehicle.vehicle_id"), nullable=True
) # Internal surrogate ID for Vehicle (by vehicle_key)
transport_us_id: Mapped[Optional[str]] = mapped_column(
String(10)
) # TRANSPORTISTAAME / Transportista americano
@@ -601,6 +607,9 @@ class InvoiceLogistics(Base, TenantScopedMixin, TimestampMixin):
trailer_num: Mapped[Optional[str]] = mapped_column(
String(20)
) # NUMTRAILER / Número de trailer
trailer_int_id: Mapped[Optional[int]] = mapped_column(
BigInteger, ForeignKey("a76.trailer.trailer_id"), nullable=True
) # Internal surrogate ID for Trailer (by trailer_number)
seal_number: Mapped[Optional[str]] = mapped_column(
String(15)
) # PRECINTO / Precinto

View File

@@ -95,8 +95,11 @@ def list_invoices(
invoice_type: str = Query(None, description="Filter by invoice type"),
manifest_number: str = Query(None, description="Filter by manifest number"),
pedimento: str = Query(None, description="Filter by pedimento"),
invoice_number: str = Query(None, description="Filter by invoice number"),
project_number: str = Query(None, description="Filter by project number"),
year: str = Query(None, description="Filter by year"),
sort_by: Optional[str] = Query(None, description="Column to sort by"),
sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"),
sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
@@ -109,12 +112,14 @@ def list_invoices(
skip = (page - 1) * page_size
filters = {
"invoice_number": search,
"invoice_number": invoice_number or search,
"status": status,
"operation_type": operation_type,
"invoice_type": invoice_type,
"manifest_number": manifest_number,
"pedimento": pedimento,
"project_number": project_number,
"year": year,
}
# Remove None values

View File

@@ -299,9 +299,14 @@ class InvoiceFinancialsBase(BaseModel):
class InvoiceLogisticsBase(BaseModel):
"""Base fields for Logistics"""
carrier_id: Optional[str] = Field(None, max_length=10, description="Carrier ID")
carrier_id: Optional[str] = Field(None, max_length=10, description="Carrier code (front-end key)")
carrier_int_id: Optional[int] = Field(
None, description="Internal carrier integer ID (mapped from carrier_id key)"
)
transport_id: Optional[str] = Field(None, max_length=10, description="Transport ID")
transport_int_id: Optional[int] = Field(
None, description="Internal vehicle integer ID (mapped from transport_id key)"
)
transport_us_id: Optional[str] = Field(
None, max_length=10, description="US transport ID"
)
@@ -329,6 +334,9 @@ class InvoiceLogisticsBase(BaseModel):
trailer_num: Optional[str] = Field(
None, max_length=20, description="Trailer number"
)
trailer_int_id: Optional[int] = Field(
None, description="Internal trailer integer ID (mapped from trailer_num key)"
)
seal_number: Optional[str] = Field(None, max_length=15, description="Seal number")
guide_number: Optional[str] = Field(None, max_length=20, description="Guide number")
bill_number: Optional[str] = Field(None, max_length=15, description="Bill number")

View File

@@ -11,10 +11,142 @@ from .exports.validators.create import validate_create as validate_create_export
from .exports.validators.update import validate_update as validate_update_export
from .common.common_validators import invoice_exists
from api.v1.modules.a76.items.models import LineItem
from . import models, schemas
def _autofill_transport_int_ids(
db: Session,
logistics_target,
tenant_id: int,
company_id: int,
) -> None:
"""
Ensures invoice logistics internal int IDs are populated when catalog rows exist.
String keys (carrier_id, transport_id, trailer_num) are matched to existing catalog
rows for the tenant/company; no placeholder rows are created. Invalid references must
be rejected by validate_common before persist.
"""
# Local imports to avoid circular dependencies.
from api.v1.modules.a76.transportation.transporters.models import Transporter
from api.v1.modules.a76.transportation.vehicles.models import Vehicle
from api.v1.modules.a76.transportation.trailers.models import Trailer
def _get(field: str):
if isinstance(logistics_target, dict):
return logistics_target.get(field)
return getattr(logistics_target, field, None)
def _set(field: str, value):
if isinstance(logistics_target, dict):
logistics_target[field] = value
else:
setattr(logistics_target, field, value)
carrier_code = _get("carrier_id")
carrier_int_id = _get("carrier_int_id")
if carrier_code and carrier_int_id is None:
transporter_obj = (
db.query(Transporter)
.filter(
Transporter.transporter_key == carrier_code,
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
)
.first()
)
if transporter_obj:
_set("carrier_int_id", transporter_obj.transporter_id)
transport_code = _get("transport_id")
transport_int_id = _get("transport_int_id")
if transport_code and transport_int_id is None:
vehicle_obj = (
db.query(Vehicle)
.filter(
func.upper(Vehicle.vehicle_key) == str(transport_code).strip().upper(),
Vehicle.tenant_id == tenant_id,
Vehicle.company_id == company_id,
)
.first()
)
if vehicle_obj:
_set("transport_int_id", vehicle_obj.vehicle_id)
trailer_code = _get("trailer_num")
trailer_int_id = _get("trailer_int_id")
if trailer_code and trailer_int_id is None:
trailer_obj = (
db.query(Trailer)
.filter(
func.upper(Trailer.trailer_number)
== str(trailer_code).strip().upper(),
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if trailer_obj:
_set("trailer_int_id", trailer_obj.trailer_id)
# Placa: si no vino valor, tomar del vehículo y si no del remolque (catálogo)
lp = _get("license_plate")
if lp is None or str(lp).strip() == "":
veh = None
tv_id = _get("transport_int_id")
tv_code = _get("transport_id")
if tv_id is not None:
veh = (
db.query(Vehicle)
.filter(
Vehicle.vehicle_id == tv_id,
Vehicle.tenant_id == tenant_id,
Vehicle.company_id == company_id,
)
.first()
)
elif tv_code:
veh = (
db.query(Vehicle)
.filter(
func.upper(Vehicle.vehicle_key) == str(tv_code).strip().upper(),
Vehicle.tenant_id == tenant_id,
Vehicle.company_id == company_id,
)
.first()
)
if veh and getattr(veh, "plate_number", None):
_set("license_plate", veh.plate_number)
else:
tr = None
tr_id = _get("trailer_int_id")
tr_code = _get("trailer_num")
if tr_id is not None:
tr = (
db.query(Trailer)
.filter(
Trailer.trailer_id == tr_id,
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
elif tr_code:
tr = (
db.query(Trailer)
.filter(
func.upper(Trailer.trailer_number)
== str(tr_code).strip().upper(),
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if tr and getattr(tr, "plate_number", None):
_set("license_plate", tr.plate_number)
def _get_current_username() -> str:
"""Helper to get current username from context or fallback to System"""
try:
@@ -126,46 +258,70 @@ class InvoiceService:
# Apply filters if provided
if filters:
# Join compliance_mx if needed for filters
needs_compliance_join = any(k in filters for k in ["pedimento", "manifest_number"])
if needs_compliance_join:
query = query.join(models.InvoiceComplianceMx)
if filters.get("status") is not None:
status = models.InvoiceStatus.PROCESSED if filters["status"] == True else models.InvoiceStatus.PENDING
status = (
models.InvoiceStatus.PROCESSED
if filters["status"] == True
else models.InvoiceStatus.PENDING
)
query = query.filter(models.InvoiceHeader.status == status)
if filters.get("operation_type"):
ot = filters["operation_type"]
ot_val = ot.value if hasattr(ot, "value") else ot
query = query.filter(
models.InvoiceHeader.operation_type == ot_val
)
query = query.filter(models.InvoiceHeader.operation_type == ot_val)
if filters.get("invoice_type"):
query = query.filter(
models.InvoiceHeader.invoice_type == filters["invoice_type"]
)
if filters.get("invoice_number"):
query = query.filter(
models.InvoiceHeader.invoice_number.ilike(
f"%{filters['invoice_number']}%"
)
)
if filters.get("pedimento"):
query = query.join(models.InvoiceComplianceMx).filter(
models.InvoiceComplianceMx.pedimento.ilike(
f"%{filters['pedimento']}%"
if filters.get("project_number"):
query = query.filter(
models.InvoiceHeader.project_number.ilike(
f"%{filters['project_number']}%"
)
)
if filters.get("year"):
try:
year_val = int(filters["year"])
query = query.filter(
func.extract("year", models.InvoiceHeader.invoice_date) == year_val
)
except (ValueError, TypeError):
pass
if filters.get("pedimento"):
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
query = query.join(models.InvoiceComplianceMx.pedimento).filter(
Pedimentos.pedimento_number.ilike(f"%{filters['pedimento']}%")
)
if filters.get("manifest_number"):
query = query.filter(
models.InvoiceComplianceMx.manifest_number.ilike(
f"%{filters['manifest_number']}%"
)
)
# Special case for exports: exclude REPAR if no invoice_type specified
ot_exp = filters.get("operation_type")
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
if not filters.get("invoice_type") and ot_exp_val == "exp":
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
if filters.get("manifest_number"):
# Avoid duplicate joins if pedimento filter was also applied (though rare in this context)
# For safety, we can just use the relationship attribute directly if mapped,
# but explicit join is clearer given the previous pattern.
# Assuming SQLAlchemy handles the join overlap or we just accept it for now.
# To be safe and consistent with previous 'pedimento' block:
query = query.join(models.InvoiceComplianceMx).filter(
models.InvoiceComplianceMx.manifest_number.ilike(f"%{filters['manifest_number']}%")
)
query = query.filter(models.InvoiceHeader.invoice_type != "REPAR")
# Apply sorting
if sort_by:
@@ -298,6 +454,12 @@ class InvoiceService:
logistics_dict["invoice_id"] = new_invoice.id
logistics_dict["tenant_id"] = tenant_id
logistics_dict["company_id"] = company_id
_autofill_transport_int_ids(
db,
logistics_dict,
tenant_id=tenant_id,
company_id=company_id,
)
new_logistics = models.InvoiceLogistics(**logistics_dict)
db.add(new_logistics)
@@ -460,11 +622,23 @@ class InvoiceService:
if value == "":
value = None
setattr(invoice.logistics, key, value)
_autofill_transport_int_ids(
db,
invoice.logistics,
tenant_id=tenant_id,
company_id=company_id,
)
else:
logistics_dict = invoice_data.logistics.model_dump()
logistics_dict["invoice_id"] = invoice.id
logistics_dict["tenant_id"] = tenant_id
logistics_dict["company_id"] = company_id
_autofill_transport_int_ids(
db,
logistics_dict,
tenant_id=tenant_id,
company_id=company_id,
)
new_logistics = models.InvoiceLogistics(**logistics_dict)
db.add(new_logistics)

View File

@@ -84,7 +84,7 @@ async def list_items(
search: Optional[str] = Query(
None, description="Search term for invoice number, reference, order, or guide"),
sort_by: Optional[str] = Query(None, description="Column to sort by"),
sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"),
sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
@@ -178,7 +178,7 @@ async def list_items_by_invoice(
skip: int = Query(0, ge=0, description="Number of records to skip"),
limit: int = Query(100, ge=1, le=1000, description="Maximum records to return"),
sort_by: Optional[str] = Query(None, description="Column to sort by"),
sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"),
sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para BOMs.
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,7 +13,6 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -22,12 +20,14 @@ from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
BOM_IMPORT_FILE_PREFIX,
JOB_TYPE,
BOM_IMPORT_META_PREFIX,
BOM_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -66,31 +66,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{BOM_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=BOM_IMPORT_REDIS_TTL,
)
r.set(
f"{BOM_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=BOM_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=BOM_IMPORT_REDIS_TTL,
log_label="BOMs import",
)
except common_storage.ImportStoreError as e:
logger.error(f"BOMs import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"BOMs import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"BOMs import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -127,12 +119,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -62,8 +62,7 @@ def scan_file(self, job_id: str, config: str = None):
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv.iter_csv_rows(file_path):
if i % 500 == 0:
self.update_state(
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Cambio de régimen y Regularización (encabezado y partidas).
Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún.
"""
import base64
import json
import logging
import os
@@ -14,7 +13,6 @@ from typing import Literal, Optional, Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -22,6 +20,7 @@ from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from .schemas import ImportJobResponse, CommitRequest
from .tasks import scan_file, insert_valid_rows, JOB_TYPE, CRREG_IMPORT_REDIS_TTL
from ..common import storage as common_storage
from ..common.responses import normalize_commit_status_payload
from ..common.error_csv import download_scan_errors_csv_stream
router = APIRouter()
@@ -64,7 +63,6 @@ async def upload_import_file(
job_id = str(uuid4())
contents = await file.read()
file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id)
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
@@ -75,25 +73,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(file_key, base64.b64encode(contents), ex=CRREG_IMPORT_REDIS_TTL)
r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=CRREG_IMPORT_REDIS_TTL)
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=CRREG_IMPORT_REDIS_TTL,
log_label="Cambio régimen/Regularización import",
)
except common_storage.ImportStoreError as e:
logger.error("Cambio régimen/Regularización import: store error: %s", e)
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error("Cambio régimen/Regularización import: Redis store error: %s", e)
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
with open(csv_path, "wb") as f:
f.write(contents)
meta_path = csv_path.replace(".csv", ".meta.json")
with open(meta_path, "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning("Cambio régimen/Regularización import: local file save failed: %s", e)
track_and_dispatch(
db=db,
task=scan_file,
@@ -133,11 +129,11 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"):
return task_result.result
return normalize_commit_status_payload(task_result.result)
logger.warning("Cambio régimen/Regularización import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -75,8 +75,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
try:
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None):
if i % 500 == 0:
on_progress(i, total_rows)
on_progress(i, total_rows)
_norm_row(row, template_id)
processed_rows += 1
except Exception as e:

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Clases de Materiales.
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,19 +13,20 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
CLS_IMPORT_FILE_PREFIX,
JOB_TYPE,
CLS_IMPORT_META_PREFIX,
CLS_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
router = APIRouter()
@@ -70,31 +70,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=CLS_IMPORT_REDIS_TTL,
)
r.set(
f"{CLS_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=CLS_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=CLS_IMPORT_REDIS_TTL,
log_label="Classes import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Classes import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"Classes import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Classes import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -133,12 +125,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
# Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica
if isinstance(result, str):
@@ -150,7 +142,7 @@ async def get_import_status(job_id: str):
):
return parsed
if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"):
return parsed
return normalize_commit_status_payload(parsed)
except (json.JSONDecodeError, TypeError):
pass

View File

@@ -84,8 +84,7 @@ def scan_file(self, job_id: str, config: str = None):
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
if i % 500 == 0:
self.update_state(
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Clientes y Proveedores.
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,19 +13,20 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
CP_IMPORT_FILE_PREFIX,
JOB_TYPE,
CP_IMPORT_META_PREFIX,
CP_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
router = APIRouter()
@@ -69,31 +69,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{CP_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=CP_IMPORT_REDIS_TTL,
)
r.set(
f"{CP_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=CP_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=CP_IMPORT_REDIS_TTL,
log_label="CP import",
)
except common_storage.ImportStoreError as e:
logger.error(f"CP import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"CP import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"CP import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -135,13 +127,13 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("CP import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -77,7 +77,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv_reader.iter_csv_rows(file_path):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, common_normalize.normalize_header)

View File

@@ -72,3 +72,36 @@ def commit_result(
if error:
out["error"] = error
return out
def normalize_commit_status_payload(d: Dict[str, Any]) -> Dict[str, Any]:
"""
Homogeneiza respuestas de commit CSV (finished / warning) al contrato usado por pedimentos:
inserted, updated, skipped_*, skipped_details, skipped_summary, critical_reference_gaps,
reference_state_ready. No modifica waiting_confirmation ni otros status.
"""
if not isinstance(d, dict):
return d
st = d.get("status")
if st not in ("finished", "warning"):
return d
out = dict(d)
out.setdefault("inserted", 0)
out.setdefault("updated", 0)
out.setdefault("skipped_invalid", 0)
out.setdefault("skipped_missing_fk", 0)
out.setdefault("skipped_duplicate", 0)
out.setdefault("skipped_missing_invoice", 0)
details = out.get("skipped_details")
if not isinstance(details, list):
out["skipped_details"] = []
else:
out["skipped_details"] = details
out.setdefault("critical_reference_gaps", 0)
if "reference_state_ready" not in out:
out["reference_state_ready"] = out.get("critical_reference_gaps", 0) == 0
if "skipped_summary" not in out:
out["skipped_summary"] = (
_summarize_reasons(out["skipped_details"], "reason") if out["skipped_details"] else []
)
return out

View File

@@ -15,6 +15,12 @@ logger = logging.getLogger(__name__)
IMPORT_REDIS_TTL = 3600
class ImportStoreError(Exception):
"""Fallo al guardar CSV en Redis/MinIO."""
pass
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
@@ -60,23 +66,103 @@ def error_path_for_job(job_type: str, job_id: str) -> str:
return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl")
def store_import_file(
job_type: str,
job_id: str,
raw_bytes: bytes,
meta_dict: dict,
tenant_id,
company_id: int,
ttl: int = IMPORT_REDIS_TTL,
log_label: str = "",
) -> None:
"""
Guarda CSV y meta en Redis. Si CSV_IMPORT_STORAGE=minio, sube el CSV a S3 y en Redis
guarda JSON {"v":2,"s3_key":...}; si no, base64 en Redis (comportamiento anterior).
En modo redis opcionalmente escribe copia local bajo layouts/imports/temp (debug).
"""
from core.config import settings
file_key, meta_key, _ = storage_keys(job_type, job_id)
r = _get_redis()
meta_bytes = json.dumps(meta_dict).encode("utf-8")
if settings.CSV_IMPORT_STORAGE == "minio":
from core.storage_s3 import put_csv_object, s3_key_for_csv_import
key = s3_key_for_csv_import(tenant_id, company_id, job_type, job_id)
try:
put_csv_object(key, raw_bytes)
except Exception as e:
logger.exception("%s MinIO put failed: %s", log_label or job_type, e)
raise ImportStoreError(str(e)) from e
payload = json.dumps({"v": 2, "s3_key": key}).encode("utf-8")
r.set(file_key, payload, ex=ttl)
else:
r.set(file_key, base64.b64encode(raw_bytes), ex=ttl)
r.set(meta_key, meta_bytes, ex=ttl)
if settings.CSV_IMPORT_STORAGE != "minio":
try:
path = file_path_for_job(job_type, job_id)
os.makedirs(upload_dir(), exist_ok=True)
with open(path, "wb") as f:
f.write(raw_bytes)
meta_path = path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta_dict, f)
except Exception as e:
logger.warning("%s local file save failed: %s", log_label or job_type, e)
def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]:
"""
Descarga contenido del CSV desde Redis y lo escribe en disco.
Obtiene el CSV desde Redis (referencia MinIO o base64 legacy) y lo escribe en disco.
Devuelve la ruta del archivo o None si no hay datos o falla.
"""
from core.config import settings
file_key, _, _ = storage_keys(job_type, job_id)
r = _get_redis()
data = r.get(file_key)
if not data:
return None
path = file_path_for_job(job_type, job_id)
os.makedirs(upload_dir(), exist_ok=True)
if data.startswith(b"{"):
try:
obj = json.loads(data.decode("utf-8"))
if isinstance(obj, dict) and obj.get("v") == 2 and obj.get("s3_key"):
if settings.CSV_IMPORT_STORAGE != "minio":
logger.warning(
"%s Redis has MinIO ref but CSV_IMPORT_STORAGE=%s",
log_prefix or job_type,
settings.CSV_IMPORT_STORAGE,
)
from core.storage_s3 import get_object_bytes
try:
raw = get_object_bytes(obj["s3_key"])
except Exception as e:
logger.warning("%s MinIO get failed: %s", log_prefix or job_type, e)
return None
with open(path, "wb") as f:
f.write(raw)
return path
except json.JSONDecodeError:
pass
except Exception as e:
logger.warning("%s failed to parse MinIO ref from Redis: %s", log_prefix or job_type, e)
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e)
return None
path = file_path_for_job(job_type, job_id)
os.makedirs(upload_dir(), exist_ok=True)
with open(path, "wb") as f:
f.write(raw)
return path
@@ -152,7 +238,31 @@ def cleanup_import_job(
error_path: Optional[str] = None,
meta_path: Optional[str] = None,
) -> None:
"""Elimina archivos locales y claves Redis del job."""
"""Elimina archivos locales, objeto MinIO si aplica, y claves Redis del job."""
from core.config import settings
if settings.CSV_IMPORT_STORAGE == "minio":
from core.storage_s3 import delete_object_if_exists
from core.s3_keys import legacy_csv_import_key
file_key, _, _ = storage_keys(job_type, job_id)
try:
r = _get_redis()
data = r.get(file_key)
deleted = False
if data and data.startswith(b"{"):
try:
obj = json.loads(data.decode("utf-8"))
if isinstance(obj, dict) and obj.get("v") == 2 and obj.get("s3_key"):
delete_object_if_exists(obj["s3_key"])
deleted = True
except Exception as e:
logger.warning("Cleanup: parse Redis file ref: %s", e)
if not deleted:
delete_object_if_exists(legacy_csv_import_key(job_type, job_id))
except Exception as e:
logger.warning("Cleanup: MinIO delete failed: %s", e)
if file_path and os.path.exists(file_path):
try:
os.remove(file_path)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Agentes Aduanales.
Mismo flujo que a76.imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,19 +13,20 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
CB_IMPORT_FILE_PREFIX,
JOB_TYPE,
CB_IMPORT_META_PREFIX,
CB_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
router = APIRouter()
@@ -69,31 +69,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{CB_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=CB_IMPORT_REDIS_TTL,
)
r.set(
f"{CB_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=CB_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=CB_IMPORT_REDIS_TTL,
log_label="CB import",
)
except common_storage.ImportStoreError as e:
logger.error(f"CB import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"CB import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"CB import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -135,13 +127,13 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
# A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("CB import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -88,7 +88,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER,
headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL,
):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, common_normalize.normalize_header)

View File

@@ -2,7 +2,6 @@
Rutas de importacion CSV para Conductores.
Flujo: upload -> scan -> status (polling) -> commit.
"""
import base64
import json
import logging
import os
@@ -15,19 +14,19 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from .schemas import ImportJobResponse
from .tasks import (
run_scan_sync,
run_commit_sync,
DRV_IMPORT_FILE_PREFIX,
DRV_IMPORT_META_PREFIX,
JOB_TYPE,
DRV_IMPORT_STATUS_PREFIX,
DRV_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -66,31 +65,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{DRV_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=DRV_IMPORT_REDIS_TTL,
)
r.set(
f"{DRV_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=DRV_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=DRV_IMPORT_REDIS_TTL,
log_label="Drivers import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Drivers import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"Drivers import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Drivers import: local file save failed: {e}")
def run_scan_background():
try:
run_scan_sync(job_id)
@@ -113,6 +104,8 @@ async def get_import_status(job_id: str):
raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
return normalize_commit_status_payload(data)
return data
except Exception as e:
logger.debug(f"Drivers import: could not read status from Redis: {e}")
@@ -131,12 +124,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -135,7 +135,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
for i, row in enumerate(dict_reader, start=1):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, common_normalize.normalize_header)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Tipos de Cambio.
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,19 +13,20 @@ from typing import Dict, Any, Optional
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
ER_IMPORT_FILE_PREFIX,
JOB_TYPE,
ER_IMPORT_META_PREFIX,
ER_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
router = APIRouter()
@@ -80,31 +80,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{ER_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=ER_IMPORT_REDIS_TTL,
)
r.set(
f"{ER_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=ER_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=ER_IMPORT_REDIS_TTL,
log_label="ER import",
)
except common_storage.ImportStoreError as e:
logger.error(f"ER import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"ER import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"ER import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -146,12 +138,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("ER import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -66,7 +66,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv_reader.iter_csv_rows(file_path):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = _norm_row(row)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Exportación (encabezado y partidas).
Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún.
"""
import base64
import json
import logging
import os
@@ -17,7 +16,6 @@ from typing import Literal, Optional, Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -25,6 +23,7 @@ from .schemas import ImportJobResponse, CommitRequest
from .tasks import scan_file, insert_valid_rows, JOB_TYPE, EXP_IMPORT_REDIS_TTL
from ..common import storage as common_storage
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -60,7 +59,6 @@ async def upload_import_file(
job_id = str(uuid4())
contents = await file.read()
file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id)
default_template = (
"exp_def_header" if model_target == "invoice_header"
else "exp_def_series" if model_target == "invoice_series"
@@ -82,25 +80,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(file_key, base64.b64encode(contents), ex=EXP_IMPORT_REDIS_TTL)
r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=EXP_IMPORT_REDIS_TTL)
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=EXP_IMPORT_REDIS_TTL,
log_label="Exportación import",
)
except common_storage.ImportStoreError as e:
logger.error("Exportación import: store error: %s", e)
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error("Exportación import: Redis store error: %s", e)
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
with open(csv_path, "wb") as f:
f.write(contents)
meta_path = csv_path.replace(".csv", ".meta.json")
with open(meta_path, "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning("Exportación import: local file save failed: %s", e)
track_and_dispatch(
db=db,
task=scan_file,
@@ -138,11 +134,11 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"):
return task_result.result
return normalize_commit_status_payload(task_result.result)
logger.warning("Exportación import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -74,8 +74,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None):
processed_rows = 0
try:
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None):
if i % 500 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows})
_norm_row(row, template_id)
processed_rows += 1
except Exception as e:

View File

@@ -1,6 +1,5 @@
from datetime import datetime
from uuid import uuid4
import base64
import os
import json
import logging
@@ -14,19 +13,19 @@ from typing import Optional, Literal, Dict, Any
from core.celery_app import celery_app
from core.config import settings
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from .tasks import (
scan_file,
insert_valid_rows,
IMPORT_FILE_KEY_PREFIX,
JOB_TYPE,
IMPORT_META_KEY_PREFIX,
IMPORT_REDIS_TTL,
)
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
from ..common import storage as common_storage
from ..common.responses import normalize_commit_status_payload
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
router = APIRouter()
@@ -82,37 +81,25 @@ async def upload_import_file(
"template_id": template_id,
}
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
try:
redis_client = _get_redis()
redis_client.set(
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
base64.b64encode(contents),
ex=IMPORT_REDIS_TTL,
)
redis_client.set(
f"{IMPORT_META_KEY_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=IMPORT_REDIS_TTL,
log_label="Facturas import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Import store error: {e}")
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
except Exception as e:
logger.error(f"Redis store error: {e}")
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"{job_id}.csv")
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
with open(file_path, "wb") as f:
f.write(contents)
with open(meta_path, "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Local file save failed (worker will use Redis): {e}")
# Trigger Celery Task (Async). Worker loads file from Redis.
# Trigger Celery Task (Async). Worker loads file from Redis / MinIO.
track_and_dispatch(
db=db,
task=scan_file,
@@ -150,7 +137,7 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)

View File

@@ -27,10 +27,20 @@ from ..common import storage as common_storage
from ..common import meta as common_meta
from ..common import responses as common_responses
from .template_config import row_from_template
from .validators.encabezados_impo_temp import csv_tipo_moneda_es_me_mn_mc
# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process
logger = logging.getLogger(__name__)
def _merge_unique_invoice_scan_error_lines(
precheck_lines: Set[int],
row_validation_lines: List[int],
) -> List[int]:
"""Unión deduplicada: líneas marcadas en precheck + líneas con error en validación por fila (imp_temp_header)."""
return sorted(set(precheck_lines) | set(row_validation_lines))
# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py)
JOB_TYPE = ""
@@ -618,8 +628,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
errors_detail = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "imp_def_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
row_errors = validate_row_series_impo_def(
@@ -812,8 +821,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
errors_detail = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "exp_def_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
row_errors = validate_row_series_expo(
@@ -994,8 +1002,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
errors_detail = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "cmex_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
row_errors = validate_row_series_impo_def(
@@ -1149,8 +1156,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
errors_detail: List[Dict[str, Any]] = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "imp_temp_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
row_errors = validate_row_series_impo_temp(
@@ -1485,8 +1491,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_temp_details", normalize_header)
row_errors = validate_row_partidas_impo_temp(
row_norm,
@@ -1796,8 +1801,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_def_details", normalize_header)
row_errors = validate_row_partidas_impo_def(
row_norm,
@@ -2363,8 +2367,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "cmex_details", normalize_header)
row_errors = validate_row_partidas_impo_def(
row_norm,
@@ -2453,6 +2456,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
parse_pedimento_col_a,
_pedimento_key_from_parsed,
)
from .validators.transport_catalog import logistics_scan_row_errors
def _ped_key_from_row(ped_str: str) -> Optional[str]:
parsed = parse_pedimento_col_a(ped_str)
@@ -2781,7 +2785,31 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
}
)
# No return temprano: el commit siempre valida por fila (logística, etc.). Si solo se
# devolvía el precheck, valid_rows del scan quedaba inflado vs inserted/skipped del commit.
scan_precheck_message: Optional[str] = None
if precheck_errors:
scan_precheck_message = (
"Precheck de referencias falló. Corrige catálogos/pedimentos según los errores indicados. "
"Se aplicó también validación completa por fila (logística, etc.) para paridad con el commit."
)
processed_rows = 0
error_lines_list: List[int] = []
errors_detail: List[Dict[str, Any]] = []
for e in precheck_errors:
if len(errors_detail) < 5000:
errors_detail.append(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
"warning": False,
}
)
with CoreSessionLocal() as session:
with open(error_path, "w", encoding="utf-8") as f_err:
for e in precheck_errors:
f_err.write(
@@ -2795,96 +2823,95 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
)
+ "\n"
)
unique_precheck_lines = sorted(precheck_lines)
common_storage.store_error_lines(effective_job_type, job_id, unique_precheck_lines)
return common_responses.scan_result(
job_id,
len(rows_list),
len(unique_precheck_lines),
precheck_errors,
total_rows_in_file=total_rows,
message="Precheck de referencias falló. Corrige catálogos/pedimentos antes de confirmar importación.",
)
error_count = 0
processed_rows = 0
error_lines_list: List[int] = []
errors_detail: List[Dict[str, Any]] = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_temp_header", normalize_header)
warnings_row: List[Dict[str, Any]] = []
row_errors = validate_row_encabezados_impo_temp(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
pedimento_rows=pedimento_rows,
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
valid_broker_ids=valid_broker_ids,
valid_broker_claves=valid_broker_claves,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_aduana_codes=valid_aduana_codes,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
autonumerar_remesas=autonumerar_remesas,
control_remesa=control_remesa,
remesa_inicio=remesa_inicio,
remesa_fin=remesa_fin,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
blocking = [e for e in row_errors if not e.get("warning")]
if blocking:
error_count += 1
error_lines_list.append(i)
for e in blocking:
f_err.write(
json.dumps(
for i, row in enumerate(rows_list, start=1):
self.update_state(
state="PROGRESS",
meta={
"current": i,
"total": total_rows,
"errors": len(
_merge_unique_invoice_scan_error_lines(
precheck_lines, error_lines_list
)
),
},
)
row_norm = row_from_template(row, "imp_temp_header", normalize_header)
warnings_row: List[Dict[str, Any]] = []
row_errors = validate_row_encabezados_impo_temp(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
pedimento_rows=pedimento_rows,
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
valid_broker_ids=valid_broker_ids,
valid_broker_claves=valid_broker_claves,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_aduana_codes=valid_aduana_codes,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
autonumerar_remesas=autonumerar_remesas,
control_remesa=control_remesa,
remesa_inicio=remesa_inicio,
remesa_fin=remesa_fin,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
row_errors.extend(
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
)
blocking = [e for e in row_errors if not e.get("warning")]
if blocking:
error_lines_list.append(i)
for e in blocking:
f_err.write(
json.dumps(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
}
)
+ "\n"
)
for e in row_errors + warnings_row:
if len(errors_detail) < 5000:
errors_detail.append(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
"warning": bool(e.get("warning", False)),
}
)
+ "\n"
)
for e in row_errors + warnings_row:
if len(errors_detail) < 5000:
errors_detail.append(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
"warning": bool(e.get("warning", False)),
}
)
processed_rows += 1
processed_rows += 1
# El commit usa líneas únicas para omitir filas; el resumen preliminar
# debe usar la misma base para evitar discrepancias de válidos/errores.
unique_error_lines = sorted(set(error_lines_list))
# El commit usa líneas únicas para omitir filas; unir precheck + validación por fila.
unique_error_lines = _merge_unique_invoice_scan_error_lines(precheck_lines, error_lines_list)
error_count = len(unique_error_lines)
common_storage.store_error_lines(effective_job_type, job_id, unique_error_lines)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
job_id,
processed_rows,
error_count,
errors_detail,
total_rows_in_file=total_rows,
message=scan_precheck_message,
)
except Exception as e:
logger.exception("Encabezados importación temporal scan failed: %s", e)
@@ -2909,6 +2936,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
parse_pedimento_col_a_impo_def,
)
from .validators.encabezados_impo_temp import _pedimento_key_from_parsed
from .validators.transport_catalog import logistics_scan_row_errors
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF")
@@ -3169,8 +3197,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_def_header", normalize_header)
warnings_row = []
row_errors = validate_row_encabezados_impo_def(
@@ -3205,6 +3232,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
parse_date_fn=parse_date,
warnings=warnings_row,
)
row_errors.extend(
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
)
blocking = [e for e in row_errors if not e.get("warning")]
if blocking:
error_count += 1
@@ -3260,6 +3290,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
from .validators.encabezados_expo import validate_row_encabezados_expo
from .validators.encabezados_impo_temp import _pedimento_key_from_parsed
from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def
from .validators.transport_catalog import logistics_scan_row_errors
_fc = parse_footer_config(meta.get("footer_config"))
actualizar = meta.get("actualizar", False)
@@ -3542,8 +3573,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "exp_def_header", normalize_header)
warnings_row = []
row_errors = validate_row_encabezados_expo(
@@ -3582,6 +3612,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
parse_date_fn=parse_date,
warnings=warnings_row,
)
row_errors.extend(
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
)
blocking = [e for e in row_errors if not e.get("warning")]
if blocking:
error_count += 1
@@ -3630,6 +3663,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
from api.v1.modules.a76.transportation.transporters.models import Transporter
from .validators.encabezados_cmex import validate_row_encabezados_cmex
from .validators.transport_catalog import logistics_scan_row_errors
_fc = parse_footer_config(meta.get("footer_config"))
actualizar = meta.get("actualizar", False)
@@ -3751,62 +3785,68 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "cmex_header", normalize_header)
warnings_row = []
row_errors = validate_row_encabezados_cmex(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
blocking = [e for e in row_errors if not e.get("warning")]
if blocking:
error_count += 1
error_lines_list.append(i)
for e in blocking:
f_err.write(
json.dumps(
with CoreSessionLocal() as session:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, "cmex_header", normalize_header)
warnings_row = []
row_errors = validate_row_encabezados_cmex(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
row_errors.extend(
logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i)
)
blocking = [e for e in row_errors if not e.get("warning")]
if blocking:
error_count += 1
error_lines_list.append(i)
for e in blocking:
f_err.write(
json.dumps(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
}
)
+ "\n"
)
for e in row_errors + warnings_row:
if len(errors_detail) < 5000:
errors_detail.append(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
"warning": bool(e.get("warning", False)),
}
)
+ "\n"
)
for e in row_errors + warnings_row:
if len(errors_detail) < 5000:
errors_detail.append(
{
"line": e["line"],
"col": e.get("col", ""),
"msg": e.get("msg", ""),
"solution": e.get("solution", ""),
"warning": bool(e.get("warning", False)),
}
)
processed_rows += 1
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
@@ -3898,12 +3938,11 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] =
for i, row in enumerate(reader, start=1):
# Check for Progress Update
if i % 1000 == 0:
self.update_state(state='PROGRESS', meta={
'current': i,
'total': total_rows,
'errors': error_count
})
self.update_state(state='PROGRESS', meta={
'current': i,
'total': total_rows,
'errors': error_count
})
# Solo columnas de la plantilla (respetar plantilla tal cual)
row_norm = row_from_template(row, template_id, normalize_header)
@@ -4016,6 +4055,8 @@ def validate_row_phase_1(
def check_currency(col_name):
val = row.get(col_name)
if val and str(val).strip():
if csv_tipo_moneda_es_me_mn_mc(str(val)):
return None
parsed_currency = parse_currency(val, None)
val_norm = normalize_header(val)
# parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid
@@ -4223,6 +4264,18 @@ def validate_row_strict(
if err:
return err
from .validators.transport_catalog import logistics_scan_row_errors
log_errs = logistics_scan_row_errors(
row,
validator.session,
validator.tenant_id,
validator.company_id,
line_num,
)
if log_errs:
return log_errs[0]
elif target == "invoice_details":
invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip()
if not invoice_number:
@@ -4398,6 +4451,13 @@ def parse_currency(value: Optional[str], currency_type: Optional[str]):
return Currency.FOREIGN
if "MANUAL" in normalized:
return Currency.MANUAL
# MC (moneda por clave): paridad con encabezados_impo_temp / InvoiceFinancials + CLAVE MONEDA
if normalized.replace(" ", "") == "MC":
if currency_type and str(currency_type).strip().upper() == "MXN":
return Currency.LOCAL
if currency_type:
return Currency.FOREIGN
return Currency.FOREIGN
if currency_type and str(currency_type).strip().upper() == "MXN":
return Currency.LOCAL
if currency_type:
@@ -5370,6 +5430,12 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt
row_to_transport_type_clarion,
_patente_from_agente_aduanal,
)
from .validators.transport_catalog import (
normalize_transport_cell,
resolve_logistics_transport_fields_for_commit,
resolve_transport_csv_fields,
validate_csv_invoice_logistics_transport,
)
from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def
from .validators.pedimento_resolution import resolve_pedimento_candidates
@@ -5832,14 +5898,15 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt
currency_val = row_norm.get('TIPO MONEDA')
if currency_val and str(currency_val).strip():
parsed_currency = parse_currency(currency_val, None)
val_norm = normalize_header(currency_val)
if parsed_currency.value == "manual" and "MANUAL" not in val_norm:
skipped_invalid += 1
reason = "TIPO MONEDA: Moneda invalida"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
if not csv_tipo_moneda_es_me_mn_mc(str(currency_val)):
parsed_currency = parse_currency(currency_val, None)
val_norm = normalize_header(currency_val)
if parsed_currency.value == "manual" and "MANUAL" not in val_norm:
skipped_invalid += 1
reason = "TIPO MONEDA: Moneda invalida"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
# 2. Client/Provider and broker checks are handled above
@@ -6064,18 +6131,84 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt
weight_type = parse_weight_unit(row_norm.get('TIPO PESO'))
logistics = None
if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'):
transport_key_csv, trailer_num_csv, transport_num_csv = resolve_transport_csv_fields(row_norm)
has_logistics_data = bool(
weight_type
or row_norm.get('TIPO TRANSPORTE')
or row_norm.get('NUMERO TRANSPORTE')
or transport_key_csv
or trailer_num_csv
or row_norm.get('CLAVE TRANSPORTISTA')
or row_norm.get('NOMBRE CONDUCTOR')
)
if has_logistics_data:
raw_transport = row_norm.get('TIPO TRANSPORTE')
transport_str = (row_to_transport_type_clarion(raw_transport) or str(raw_transport or "").strip().lower() or "none")
try:
transport_type = TransportType(transport_str)
except ValueError:
transport_type = TransportType.NONE
err_transport = validate_csv_invoice_logistics_transport(
row_norm=row_norm,
session=session,
tenant_id=tenant_id,
company_id=company_id,
transport_type=transport_type,
)
if err_transport:
skipped_invalid += 1
reason = err_transport
skipped_fk_details.append(
{"line": i, "invoice": invoice_number, "reason": reason}
)
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
transport_num_effective = (transport_num_csv or None)
if transport_type == TransportType.NONE:
transport_num_effective = None
carrier_code = normalize_transport_cell(row_norm.get('CLAVE TRANSPORTISTA')) or None
carrier_int_id = None
if carrier_code:
from api.v1.modules.a76.transportation.transporters.models import (
Transporter,
)
carrier_obj = (
session.query(Transporter)
.filter(
Transporter.transporter_key == carrier_code,
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
)
.first()
)
if carrier_obj:
carrier_int_id = carrier_obj.transporter_id
(
transport_id_value,
trailer_num_value,
transport_int_id,
trailer_int_id,
) = resolve_logistics_transport_fields_for_commit(
session=session,
tenant_id=tenant_id,
company_id=company_id,
clave_transporte_raw=transport_key_csv,
numero_caja_raw=trailer_num_csv,
)
logistics = InvoiceLogistics(
carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None),
carrier_id=carrier_code,
carrier_int_id=carrier_int_id,
driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None),
transport_type=transport_type,
transport_num=(row_norm.get('NUMERO TRANSPORTE') or None),
transport_num=transport_num_effective,
transport_id=transport_id_value,
trailer_num=trailer_num_value,
transport_int_id=transport_int_id,
trailer_int_id=trailer_int_id,
weight_type=weight_type or WeightUnit.KGS,
seal_number=(row_norm.get('PRECINTO') or None),
incoterm=(row_norm.get('CLAVE INCOTERM') or None),

View File

@@ -29,6 +29,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
{"canonical": "CLAVE TRANSPORTISTA"},
{"canonical": "NOMBRE CONDUCTOR"},
{"canonical": "TIPO TRANSPORTE"},
{"canonical": "CLAVE TRANSPORTE"},
{"canonical": "NUMERO CAJA"},
{"canonical": "NUMERO TRANSPORTE"},
{"canonical": "TIPO MONEDA"},
{"canonical": "CLAVE MONEDA"},
@@ -66,6 +68,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
{"canonical": "CLAVE TRANSPORTISTA"},
{"canonical": "NOMBRE CONDUCTOR"},
{"canonical": "TIPO TRANSPORTE"},
{"canonical": "CLAVE TRANSPORTE"},
{"canonical": "NUMERO CAJA"},
{"canonical": "NUMERO TRANSPORTE"},
{"canonical": "TIPO MONEDA"},
{"canonical": "CLAVE MONEDA"},
@@ -102,6 +106,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
{"canonical": "CLAVE TRANSPORTISTA"},
{"canonical": "NOMBRE CONDUCTOR"},
{"canonical": "TIPO TRANSPORTE"},
{"canonical": "CLAVE TRANSPORTE"},
{"canonical": "NUMERO CAJA"},
{"canonical": "NUMERO TRANSPORTE"},
{"canonical": "TIPO MONEDA"},
{"canonical": "CLAVE MONEDA"},
@@ -135,6 +141,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
{"canonical": "CLAVE TRANSPORTISTA"},
{"canonical": "NOMBRE CONDUCTOR"},
{"canonical": "TIPO TRANSPORTE"},
{"canonical": "CLAVE TRANSPORTE"},
{"canonical": "NUMERO CAJA"},
{"canonical": "NUMERO TRANSPORTE"},
{"canonical": "TIPO MONEDA"},
{"canonical": "CLAVE MONEDA"},

View File

@@ -51,18 +51,6 @@ def _validaciones_transporte_cmex(row: Dict[str, Any], line_num: int) -> Optiona
"NUMERO TRANSPORTE",
"Error: (Celda N{}) El Tipo de Transporte está vacío y está capturado un número de transporte.".format(line_num),
)
if m_raw and (m == "NINGUNO" or m_with_space == "NINGUNO") and n:
return _err(
line_num,
"NUMERO TRANSPORTE",
"Error: (Celda N{}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.".format(line_num),
)
if m_raw and m != "NINGUNO" and m_with_space != "NINGUNO" and not n:
return _err(
line_num,
"NUMERO TRANSPORTE",
"Error: (Celda N{}) El Tipo de Transporte es {} y no está capturado el número de transporte.".format(line_num, m_raw),
)
return None

View File

@@ -19,6 +19,16 @@ REGIMENES_VALIDOS = frozenset({"ITE", "ITR"})
TIPOS_MONEDA_VALIDOS = frozenset({"ME", "MN", "MC"})
TIPO_PESO_VALIDOS = frozenset({"KILOS", "LIBRAS"})
def csv_tipo_moneda_es_me_mn_mc(value: Any) -> bool:
"""
ME / MN / MC con la misma regla que _validaciones_moneda (strip + upper).
Compartido entre scan (encabezados) y commit / validate_row_strict en tasks.
"""
if value is None or not str(value).strip():
return False
return str(value).strip().upper() in TIPOS_MONEDA_VALIDOS
# Clarion Col M → valor normalizado (minúscula para TransportType enum)
TIPO_TRANSPORTE_CLARION_TO_NORM = {
"NINGUNO": "none",
@@ -344,18 +354,6 @@ def _validaciones_transporte(row: Dict[str, Any], line_num: int) -> Optional[Dic
"NUMERO TRANSPORTE",
f"Error: (Celda N{line_num}) El Tipo de Transporte está vacío y está capturado un número de transporte.",
)
if m == "NINGUNO" and n:
return _err(
line_num,
"NUMERO TRANSPORTE",
f"Error: (Celda N{line_num}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.",
)
if m and m != "NINGUNO" and not n:
return _err(
line_num,
"NUMERO TRANSPORTE",
f"Error: (Celda N{line_num}) El Tipo de Transporte es {m} y no está capturado el número de transporte.",
)
return None

View File

@@ -0,0 +1,239 @@
"""Mapeo y validación de transporte para CSV (paridad con facturas manuales / validate_common)."""
from __future__ import annotations
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy import func
from sqlalchemy.orm import Session
from api.v1.modules.a76.invoices.models import TransportType
def normalize_transport_cell(raw: Optional[str]) -> str:
if raw is None:
return ""
return str(raw).replace("\u00a0", " ").strip()
def resolve_transport_csv_fields(row: dict) -> Tuple[str, str, str]:
"""
Prioridad de columnas:
1) CLAVE TRANSPORTE / NUMERO CAJA
2) NUMERO TRANSPORTE como fallback legacy.
"""
legacy = normalize_transport_cell(row.get("NUMERO TRANSPORTE"))
clave_transporte = normalize_transport_cell(row.get("CLAVE TRANSPORTE")) or legacy
numero_caja = normalize_transport_cell(row.get("NUMERO CAJA")) or legacy
numero_transporte = legacy
return clave_transporte, numero_caja, numero_transporte
def resolve_logistics_transport_fields_for_commit(
session: Session,
tenant_id: int,
company_id: int,
clave_transporte_raw: str,
numero_caja_raw: str,
) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[int]]:
"""
Resuelve claves y IDs internos para logística:
- transport_id / transport_int_id a partir de Vehicle.vehicle_key
- trailer_num / trailer_int_id a partir de Trailer.trailer_number
"""
from api.v1.modules.a76.transportation.vehicles.models import Vehicle
from api.v1.modules.a76.transportation.trailers.models import Trailer
clave_transporte = normalize_transport_cell(clave_transporte_raw)
numero_caja = normalize_transport_cell(numero_caja_raw)
transport_id: Optional[str] = clave_transporte or None
trailer_num: Optional[str] = numero_caja or None
transport_int_id: Optional[int] = None
trailer_int_id: Optional[int] = None
if transport_id:
vehicle = (
session.query(Vehicle)
.filter(
func.upper(Vehicle.vehicle_key) == transport_id.upper(),
Vehicle.tenant_id == tenant_id,
Vehicle.company_id == company_id,
)
.first()
)
if vehicle is not None:
transport_int_id = vehicle.vehicle_id
if trailer_num:
trailer = (
session.query(Trailer)
.filter(
func.upper(Trailer.trailer_number) == trailer_num.upper(),
Trailer.tenant_id == tenant_id,
Trailer.company_id == company_id,
)
.first()
)
if trailer is not None:
trailer_int_id = trailer.trailer_id
return transport_id, trailer_num, transport_int_id, trailer_int_id
def validate_csv_invoice_logistics_transport(
row_norm: dict,
session: Session,
tenant_id: int,
company_id: int,
transport_type: TransportType,
) -> Optional[str]:
"""
Paridad con validate_common (factura manual):
- Transportista y conductor: solo catálogo si vienen informados.
- Vehículo: solo catálogo si viene CLAVE TRANSPORTE explícita y no resuelve.
- Remolque: obligatorio (y catálogo) si tipo ≠ none; con none, solo catálogo si NUMERO CAJA explícito.
"""
from api.v1.modules.a76.transportation.transporters.models import Transporter
from api.v1.modules.a76.transportation.drivers.models import Driver
carrier_code = normalize_transport_cell(row_norm.get("CLAVE TRANSPORTISTA"))
carrier_ok = False
if carrier_code:
tr_obj = (
session.query(Transporter)
.filter(
Transporter.transporter_key == carrier_code,
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
)
.first()
)
if not tr_obj:
return "CLAVE TRANSPORTISTA no existe en el catálogo"
carrier_ok = True
driver_raw = normalize_transport_cell(row_norm.get("NOMBRE CONDUCTOR"))
if carrier_ok and driver_raw:
dr = (
session.query(Driver)
.filter(
Driver.transporter_key == carrier_code,
Driver.driver_name == driver_raw,
Driver.tenant_id == tenant_id,
Driver.company_id == company_id,
)
.first()
)
if not dr:
return (
"NOMBRE CONDUCTOR: no existe en el catálogo de conductores "
"para el transportista indicado"
)
transport_key_csv, trailer_num_csv, _ = resolve_transport_csv_fields(row_norm)
(
_transport_id_value,
trailer_num_resolved,
transport_int_id,
trailer_int_id,
) = resolve_logistics_transport_fields_for_commit(
session=session,
tenant_id=tenant_id,
company_id=company_id,
clave_transporte_raw=transport_key_csv,
numero_caja_raw=trailer_num_csv,
)
explicit_clave = normalize_transport_cell(row_norm.get("CLAVE TRANSPORTE"))
if explicit_clave and transport_int_id is None:
return "CLAVE TRANSPORTE: vehículo no existe en el catálogo"
explicit_caja = normalize_transport_cell(row_norm.get("NUMERO CAJA"))
if transport_type == TransportType.NONE:
if explicit_caja and trailer_int_id is None:
return "NUMERO CAJA: remolque no existe en el catálogo"
return None
if trailer_int_id is None:
if not (trailer_num_resolved and str(trailer_num_resolved).strip()):
return (
"NUMERO CAJA (remolque) obligatorio cuando TIPO TRANSPORTE es distinto de NINGUNO"
)
return "NUMERO CAJA: remolque no existe en el catálogo"
return None
def _csv_row_has_resolved_weight_unit(row_norm: dict) -> bool:
"""
Paridad con insert_valid_rows: has_logistics_data incluye parse_weight_unit(TIPO PESO).
Import diferido para evitar ciclo al cargar tasks.
"""
from api.v1.modules.a76.layouts_csv.facturas.tasks import parse_weight_unit
return parse_weight_unit(row_norm.get("TIPO PESO")) is not None
def logistics_scan_row_errors(
row_norm: dict,
session: Session,
tenant_id: int,
company_id: int,
line_num: int,
) -> List[Dict[str, Any]]:
"""
Errores de scan (misma lógica que commit: validate_csv_invoice_logistics_transport).
Solo corre si la fila trae algún dato de logística de transporte.
"""
from .encabezados_impo_temp import row_to_transport_type_clarion
transport_key_csv, trailer_num_csv, _ = resolve_transport_csv_fields(row_norm)
def _cell(key: str) -> str:
return normalize_transport_cell(row_norm.get(key))
has_data = bool(
_cell("TIPO TRANSPORTE")
or _cell("NUMERO TRANSPORTE")
or _cell("CLAVE TRANSPORTE")
or transport_key_csv
or trailer_num_csv
or _cell("CLAVE TRANSPORTISTA")
or _cell("NOMBRE CONDUCTOR")
or _csv_row_has_resolved_weight_unit(row_norm)
)
if not has_data:
return []
raw_transport = row_norm.get("TIPO TRANSPORTE")
transport_str = (
row_to_transport_type_clarion(raw_transport)
or str(raw_transport or "").strip().lower()
or "none"
)
try:
tt = TransportType(transport_str)
except ValueError:
tt = TransportType.NONE
msg = validate_csv_invoice_logistics_transport(
row_norm=row_norm,
session=session,
tenant_id=tenant_id,
company_id=company_id,
transport_type=tt,
)
if not msg:
return []
return [
{
"line": line_num,
"col": "LOGISTICA TRANSPORTE",
"msg": msg,
"solution": (
"Verifica CLAVE TRANSPORTISTA, NOMBRE CONDUCTOR, CLAVE TRANSPORTE, "
"NUMERO CAJA y TIPO TRANSPORTE (remolque obligatorio si no es NINGUNO)."
),
}
]

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Números de Parte.
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,7 +13,6 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -22,12 +20,14 @@ from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
PART_IMPORT_FILE_PREFIX,
JOB_TYPE,
PART_IMPORT_META_PREFIX,
PART_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -70,31 +70,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{PART_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=PART_IMPORT_REDIS_TTL,
)
r.set(
f"{PART_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=PART_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=PART_IMPORT_REDIS_TTL,
log_label="Parts import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Parts import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"Parts import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Parts import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -131,12 +123,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
# Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica
if isinstance(result, str):
@@ -148,7 +140,7 @@ async def get_import_status(job_id: str):
):
return parsed
if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"):
return parsed
return normalize_commit_status_payload(parsed)
except (json.JSONDecodeError, TypeError):
pass

View File

@@ -94,8 +94,7 @@ def scan_file(self, job_id: str, config: str = None):
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
if i % 500 == 0:
self.update_state(
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Pedimentos.
Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,11 +13,11 @@ from typing import Dict, Any, Optional
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
@@ -63,7 +62,6 @@ async def upload_import_file(
job_id = str(uuid4())
contents = await file.read()
file_key, meta_key, _ = common_storage.storage_keys(PED_JOB_TYPE, job_id)
meta_data: Dict[str, Any] = {
"tenant_id": tenant_id,
"company_id": company_id,
@@ -75,33 +73,23 @@ async def upload_import_file(
meta_data["dateFormat"] = dateFormat
try:
r = _get_redis()
r.set(
file_key,
base64.b64encode(contents),
ex=PED_IMPORT_REDIS_TTL,
)
r.set(
meta_key,
json.dumps(meta_data).encode("utf-8"),
ex=PED_IMPORT_REDIS_TTL,
common_storage.store_import_file(
PED_JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=PED_IMPORT_REDIS_TTL,
log_label="Pedimentos import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Pedimentos import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"Pedimentos import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
csv_path = common_storage.file_path_for_job(PED_JOB_TYPE, job_id)
with open(csv_path, "wb") as f:
f.write(contents)
meta_path = csv_path.replace(".csv", ".meta.json")
with open(meta_path, "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Pedimentos import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -143,12 +131,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -99,7 +99,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = _norm_row(row)

View File

@@ -25,8 +25,8 @@ def load_trailers_fk_sets(
Devuelve:
- valid_trailer_type_keys: códigos de trailer_type (GTipoTrailer)
- valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas
- state_descriptions_upper: descripciones de estados en mayúsculas (GEstados)
- state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país"
- state_descriptions_upper: descripciones y mex_key en mayúsculas (GEstados)
- state_country_set: (ame_key_pais, token_upper) para description o mex_key
- state_ame_to_description: dict clave_ame_upper -> description_upper (opcional; vacío si State no tiene ame_key)
"""
valid_trailer_type_keys: Set[str] = set()
@@ -50,21 +50,35 @@ def load_trailers_fk_sets(
valid_country_ame.add((row[0] or "").strip().upper())
for state in session.query(State).all():
country = (
session.query(Country)
.filter(Country.m3_key == state.m3_key)
.first()
)
ame_country = None
if country and (country.ame_key or "").strip():
ame_country = (country.ame_key or "").strip().upper()
desc = (state.description or "").strip()
if desc:
state_descriptions_upper.add(desc.upper())
country = (
session.query(Country)
.filter(Country.m3_key == state.m3_key)
.first()
if ame_country:
state_country_set.add((ame_country, desc.upper()))
mex_key = (state.mex_key or "").strip()
if mex_key:
mk = mex_key.upper()
state_descriptions_upper.add(mk)
if ame_country:
state_country_set.add((ame_country, mk))
if desc:
state_ame_to_description[mk] = desc.upper()
ame_state = getattr(state, "ame_key", None)
if ame_state and (ame_state or "").strip():
state_ame_to_description[(ame_state or "").strip().upper()] = (
desc.upper() if desc else (ame_state or "").strip().upper()
)
if country and (country.ame_key or "").strip():
state_country_set.add(
((country.ame_key or "").strip().upper(), desc.upper())
)
ame = getattr(state, "ame_key", None)
if ame and (ame or "").strip():
state_ame_to_description[(ame or "").strip().upper()] = desc.upper() if desc else (ame or "").strip().upper()
except Exception as e:
logger.warning("Trailers import: could not load FK sets: %s", e)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Trailers y Cajas.
Flujo: upload -> scan -> status (polling) -> commit.
"""
import base64
import json
import logging
import os
@@ -15,7 +14,6 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -24,12 +22,13 @@ from .tasks import (
scan_file,
run_scan_sync,
run_commit_sync,
TRL_IMPORT_FILE_PREFIX,
TRL_IMPORT_META_PREFIX,
JOB_TYPE,
TRL_IMPORT_STATUS_PREFIX,
TRL_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -70,31 +69,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{TRL_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=TRL_IMPORT_REDIS_TTL,
)
r.set(
f"{TRL_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=TRL_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=TRL_IMPORT_REDIS_TTL,
log_label="Trailers import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Trailers import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"Trailers import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Trailers import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -132,6 +123,8 @@ async def get_import_status(job_id: str):
raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
return normalize_commit_status_payload(data)
return data
except Exception as e:
logger.debug(f"Trailers import: could not read status from Redis: {e}")
@@ -150,12 +143,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -121,7 +121,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect)
for i, row in enumerate(dict_reader, start=1):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, common_normalize.normalize_header)

View File

@@ -24,8 +24,8 @@ def load_transportistas_fk_sets(
Devuelve:
- existing_transporter_keys: claves de transportistas existentes (tenant/company) en mayúsculas
- valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas
- state_descriptions_upper: descripciones de estados en mayúsculas (GEstados)
- state_country_set: set de (ame_key_pais, description_estado_upper) para validar estado pertenece a país
- state_descriptions_upper: descripciones y claves cortas (mex_key) de estados en mayúsculas (GEstados)
- state_country_set: set de (ame_key_pais, token_upper) para description o mex_key
"""
existing_transporter_keys: Set[str] = set()
valid_country_ame: Set[str] = set()
@@ -55,18 +55,27 @@ def load_transportistas_fk_sets(
valid_country_ame.add((row[0] or "").strip().upper())
for state in session.query(State).all():
country = (
session.query(Country)
.filter(Country.m3_key == state.m3_key)
.first()
)
ame = None
if country and (country.ame_key or "").strip():
ame = (country.ame_key or "").strip().upper()
desc = (state.description or "").strip()
if desc:
state_descriptions_upper.add(desc.upper())
country = (
session.query(Country)
.filter(Country.m3_key == state.m3_key)
.first()
)
if country and (country.ame_key or "").strip():
state_country_set.add(
((country.ame_key or "").strip().upper(), desc.upper())
)
if ame:
state_country_set.add((ame, desc.upper()))
mex_key = (state.mex_key or "").strip()
if mex_key:
mk = mex_key.upper()
state_descriptions_upper.add(mk)
if ame:
state_country_set.add((ame, mk))
except Exception as e:
logger.warning("Transportistas import: could not load FK sets: %s", e)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Transportistas.
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -15,7 +14,6 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -24,12 +22,13 @@ from .tasks import (
scan_file,
run_scan_sync,
run_commit_sync,
TRP_IMPORT_FILE_PREFIX,
TRP_IMPORT_META_PREFIX,
JOB_TYPE,
TRP_IMPORT_STATUS_PREFIX,
TRP_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -70,31 +69,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{TRP_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=TRP_IMPORT_REDIS_TTL,
)
r.set(
f"{TRP_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=TRP_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=TRP_IMPORT_REDIS_TTL,
log_label="Transportistas import",
)
except common_storage.ImportStoreError as e:
logger.error("Transportistas import: store error: %s", e)
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error("Transportistas import: Redis store error: %s", e)
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"trp_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"trp_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning("Transportistas import: local file save failed: %s", e)
track_and_dispatch(
db=db,
task=scan_file,
@@ -132,6 +123,8 @@ async def get_import_status(job_id: str):
raw = r.get(f"{TRP_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
return normalize_commit_status_payload(data)
return data
except Exception as e:
logger.debug("Transportistas import: could not read status from Redis: %s", e)
@@ -150,12 +143,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("Transportistas import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -79,7 +79,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv_reader.iter_csv_rows(file_path):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, common_normalize.normalize_header)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Fracción Americana (US Tariff Fractions).
Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -14,19 +13,20 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit
from ..common.responses import normalize_commit_status_payload
from .schemas import ImportJobResponse
from .tasks import (
scan_file,
insert_valid_rows,
FA_IMPORT_FILE_PREFIX,
JOB_TYPE,
FA_IMPORT_META_PREFIX,
FA_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
router = APIRouter()
@@ -72,31 +72,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{FA_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=FA_IMPORT_REDIS_TTL,
)
r.set(
f"{FA_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=FA_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=FA_IMPORT_REDIS_TTL,
log_label="FA import",
)
except common_storage.ImportStoreError as e:
logger.error(f"FA import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"FA import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"FA import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -138,12 +130,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("FA import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -68,7 +68,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv_reader.iter_csv_rows(file_path):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
warn = validate_row_desfase_fa(row, i)

View File

@@ -24,8 +24,8 @@ def load_vehicles_fk_sets(
Devuelve:
- valid_transport_codes: códigos de transport_types (GTipoTransportes)
- valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas
- state_descriptions_upper: descripciones de estados en mayúsculas (GEstados), para "estado existe"
- state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país"
- state_descriptions_upper: descripciones y mex_key en mayúsculas (GEstados)
- state_country_set: (ame_key_pais, token_upper) para description o mex_key
"""
valid_transport_codes: Set[str] = set()
valid_country_ame: Set[str] = set()
@@ -46,21 +46,28 @@ def load_vehicles_fk_sets(
if row[0]:
valid_country_ame.add((row[0] or "").strip().upper())
# States: description (GEstados.Descripcion); State.m3_key = Country.m3_key
for state in session.query(State).all():
country = (
session.query(Country)
.filter(Country.m3_key == state.m3_key)
.first()
)
ame = None
if country and (country.ame_key or "").strip():
ame = (country.ame_key or "").strip().upper()
desc = (state.description or "").strip()
if desc:
state_descriptions_upper.add(desc.upper())
# País para este estado vía m3_key
country = (
session.query(Country)
.filter(Country.m3_key == state.m3_key)
.first()
)
if country and (country.ame_key or "").strip():
state_country_set.add(
((country.ame_key or "").strip().upper(), desc.upper())
)
if ame:
state_country_set.add((ame, desc.upper()))
mex_key = (state.mex_key or "").strip()
if mex_key:
mk = mex_key.upper()
state_descriptions_upper.add(mk)
if ame:
state_country_set.add((ame, mk))
except Exception as e:
logger.warning("Vehicles import: could not load FK sets: %s", e)

View File

@@ -2,7 +2,6 @@
Rutas de importación CSV para Vehículos (Transportes).
Flujo: upload → scan → status (polling) → commit.
"""
import base64
import json
import logging
import os
@@ -15,7 +14,6 @@ from typing import Dict, Any
from core.celery_app import celery_app
from core.database import get_core_db
from core.paths import layout_path
from core.security import get_current_user, validate_access_to_resource
from api.v1.modules.core.tasks_tracking import track_and_dispatch
@@ -24,12 +22,13 @@ from .tasks import (
scan_file,
run_scan_sync,
run_commit_sync,
VEHL_IMPORT_FILE_PREFIX,
VEHL_IMPORT_META_PREFIX,
JOB_TYPE,
VEHL_IMPORT_STATUS_PREFIX,
VEHL_IMPORT_REDIS_TTL,
)
from ..common import storage as common_storage
from ..common.error_csv import download_scan_errors_csv_stream
from ..common.responses import normalize_commit_status_payload
router = APIRouter()
logger = logging.getLogger(__name__)
@@ -70,31 +69,23 @@ async def upload_import_file(
}
try:
r = _get_redis()
r.set(
f"{VEHL_IMPORT_FILE_PREFIX}{job_id}",
base64.b64encode(contents),
ex=VEHL_IMPORT_REDIS_TTL,
)
r.set(
f"{VEHL_IMPORT_META_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=VEHL_IMPORT_REDIS_TTL,
common_storage.store_import_file(
JOB_TYPE,
job_id,
contents,
meta_data,
tenant_id=int(tenant_id),
company_id=company_id,
ttl=VEHL_IMPORT_REDIS_TTL,
log_label="Vehicles import",
)
except common_storage.ImportStoreError as e:
logger.error(f"Vehicles import: store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
except Exception as e:
logger.error(f"Vehicles import: Redis store error: {e}")
raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.")
try:
upload_dir = layout_path("imports", "temp")
os.makedirs(upload_dir, exist_ok=True)
with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f:
f.write(contents)
with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.warning(f"Vehicles import: local file save failed: {e}")
track_and_dispatch(
db=db,
task=scan_file,
@@ -131,6 +122,8 @@ async def get_import_status(job_id: str):
raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}")
if raw:
data = json.loads(raw.decode("utf-8"))
if isinstance(data, dict) and data.get("status") in ("finished", "warning"):
return normalize_commit_status_payload(data)
return data
except Exception as e:
logger.debug(f"Vehicles import: could not read status from Redis: {e}")
@@ -149,12 +142,12 @@ async def get_import_status(job_id: str):
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return normalize_commit_status_payload(result)
return {"status": "finished", "result": result}
result = getattr(task_result, "result", None)
if isinstance(result, dict) and result.get("status") in ("finished", "warning"):
return result
return normalize_commit_status_payload(result)
logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state)
err_msg = None

View File

@@ -92,7 +92,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str,
try:
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv_reader.iter_csv_rows(file_path):
if progress_callback and i % 500 == 0:
if progress_callback:
progress_callback(i, total_rows, error_count)
row_norm = row_from_template(row, common_normalize.normalize_header)

View File

@@ -298,7 +298,13 @@ class ConsolidadoImportacionMexService:
if logistics:
# 1. Transporter (CAAT / SCAC)
if logistics.carrier_id:
if logistics.carrier_int_id is not None:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_id == logistics.carrier_int_id)
.first()
)
elif logistics.carrier_id:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_key == logistics.carrier_id)
@@ -342,11 +348,18 @@ class ConsolidadoImportacionMexService:
# 2. Vehicle (Placas Tracto) - Try transport_id first
if logistics.transport_id:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if logistics.transport_int_id is not None:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
.first()
)
else:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if veh_obj:
placas_val = veh_obj.plate_number or placas_val
elif (
@@ -362,11 +375,18 @@ class ConsolidadoImportacionMexService:
# 3. Trailer (Placas Remolque)
if logistics.trailer_num:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if logistics.trailer_int_id is not None:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_id == logistics.trailer_int_id)
.first()
)
else:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if trl_obj:
placas_remolque_val = trl_obj.plate_number or ""

View File

@@ -175,8 +175,16 @@ class ConsolidadoImportacionMexService:
if logistics:
# 1. Transporter (CAAT / SCAC)
if logistics.carrier_id:
transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first()
if logistics.carrier_int_id is not None:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_id == logistics.carrier_int_id)
.first()
)
elif logistics.carrier_id:
transporter_obj = db.query(Transporter).filter(
Transporter.transporter_key == logistics.carrier_id
).first()
if transporter_obj:
caat_val = transporter_obj.caat_code or ""
scac_val = transporter_obj.transport_code or "" # Mapping transport_code to SCAC
@@ -213,7 +221,16 @@ class ConsolidadoImportacionMexService:
# 2. Vehicle (Placas Tracto) - Try transport_id first
if logistics.transport_id:
veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first()
if logistics.transport_int_id is not None:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
.first()
)
else:
veh_obj = db.query(Vehicle).filter(
Vehicle.vehicle_key == logistics.transport_id
).first()
if veh_obj:
placas_val = veh_obj.plate_number or placas_val
elif logistics.vehicle_num: # Fallback to vehicle_num if populated and transport_id failed/empty
@@ -223,7 +240,16 @@ class ConsolidadoImportacionMexService:
# 3. Trailer (Placas Remolque)
if logistics.trailer_num:
trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first()
if logistics.trailer_int_id is not None:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_id == logistics.trailer_int_id)
.first()
)
else:
trl_obj = db.query(Trailer).filter(
Trailer.trailer_number == logistics.trailer_num
).first()
if trl_obj:
placas_remolque_val = trl_obj.plate_number or ""

View File

@@ -317,7 +317,17 @@ class FacturaImportacionMexService:
if logistics:
# 1. Transporter (CAAT / SCAC)
if logistics.carrier_id:
if logistics.carrier_int_id is not None:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_id == logistics.carrier_int_id)
.first()
)
if transporter_obj:
caat_val = transporter_obj.caat_code or ""
scac_val = transporter_obj.transport_code or ""
transportista_val = transporter_obj.name or logistics.carrier_id
elif logistics.carrier_id:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_key == logistics.carrier_id)
@@ -332,11 +342,18 @@ class FacturaImportacionMexService:
# 2. Vehicle (Placas Tracto) - Try transport_id first
if logistics.transport_id:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if logistics.transport_int_id is not None:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
.first()
)
else:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if veh_obj:
placas_val = veh_obj.plate_number or placas_val
elif (
@@ -352,11 +369,18 @@ class FacturaImportacionMexService:
# 3. Trailer (Placas Remolque)
if logistics.trailer_num:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if logistics.trailer_int_id is not None:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_id == logistics.trailer_int_id)
.first()
)
else:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if trl_obj:
placas_remolque_val = trl_obj.plate_number or ""

View File

@@ -166,8 +166,16 @@ class FacturaImportacionMexService:
if logistics:
# 1. Transporter (CAAT / SCAC)
if logistics.carrier_id:
transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first()
if logistics.carrier_int_id is not None:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_id == logistics.carrier_int_id)
.first()
)
elif logistics.carrier_id:
transporter_obj = db.query(Transporter).filter(
Transporter.transporter_key == logistics.carrier_id
).first()
if transporter_obj:
caat_val = transporter_obj.caat_code or ""
scac_val = transporter_obj.transport_code or "" # Mapping transport_code to SCAC
@@ -175,7 +183,16 @@ class FacturaImportacionMexService:
# 2. Vehicle (Placas Tracto) - Try transport_id first
if logistics.transport_id:
veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first()
if logistics.transport_int_id is not None:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
.first()
)
else:
veh_obj = db.query(Vehicle).filter(
Vehicle.vehicle_key == logistics.transport_id
).first()
if veh_obj:
placas_val = veh_obj.plate_number or placas_val
elif logistics.vehicle_num: # Fallback to vehicle_num if populated and transport_id failed/empty
@@ -185,7 +202,16 @@ class FacturaImportacionMexService:
# 3. Trailer (Placas Remolque)
if logistics.trailer_num:
trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first()
if logistics.trailer_int_id is not None:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_id == logistics.trailer_int_id)
.first()
)
else:
trl_obj = db.query(Trailer).filter(
Trailer.trailer_number == logistics.trailer_num
).first()
if trl_obj:
placas_remolque_val = trl_obj.plate_number or ""

View File

@@ -317,7 +317,17 @@ class FacturaImportacionUsaService:
if logistics:
# 1. Transporter (CAAT / SCAC)
if logistics.carrier_id:
if logistics.carrier_int_id is not None:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_id == logistics.carrier_int_id)
.first()
)
if transporter_obj:
caat_val = transporter_obj.caat_code or ""
scac_val = transporter_obj.transport_code or ""
transportista_val = transporter_obj.name or logistics.carrier_id
elif logistics.carrier_id:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_key == logistics.carrier_id)
@@ -325,18 +335,23 @@ class FacturaImportacionUsaService:
)
if transporter_obj:
caat_val = transporter_obj.caat_code or ""
scac_val = (
transporter_obj.transport_code or ""
) # Mapping transport_code to SCAC
scac_val = transporter_obj.transport_code or ""
transportista_val = transporter_obj.name or logistics.carrier_id
# 2. Vehicle (Plates)
if logistics.transport_id:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if logistics.transport_int_id is not None:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
.first()
)
else:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if veh_obj:
placas_val = veh_obj.plate_number or placas_val
elif logistics.vehicle_num:
@@ -350,11 +365,18 @@ class FacturaImportacionUsaService:
# 3. Trailer
if logistics.trailer_num:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if logistics.trailer_int_id is not None:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_id == logistics.trailer_int_id)
.first()
)
else:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if trl_obj:
placas_remolque_val = trl_obj.plate_number or ""

View File

@@ -186,8 +186,18 @@ class PackingListService:
if logistics:
# 1. Transporter (CAAT / SCAC)
if logistics.carrier_id:
transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first()
if logistics.carrier_int_id is not None:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_id == logistics.carrier_int_id)
.first()
)
elif logistics.carrier_id:
transporter_obj = (
db.query(Transporter)
.filter(Transporter.transporter_key == logistics.carrier_id)
.first()
)
if transporter_obj:
caat_val = transporter_obj.caat_code or ""
scac_val = transporter_obj.transport_code or ""
@@ -195,7 +205,18 @@ class PackingListService:
# 2. Vehicle (Placas Tracto)
if logistics.transport_id:
veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first()
if logistics.transport_int_id is not None:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_id == logistics.transport_int_id)
.first()
)
else:
veh_obj = (
db.query(Vehicle)
.filter(Vehicle.vehicle_key == logistics.transport_id)
.first()
)
if veh_obj:
placas_val = veh_obj.plate_number or placas_val
elif logistics.vehicle_num:
@@ -205,7 +226,18 @@ class PackingListService:
# 3. Trailer (Placas Remolque)
if logistics.trailer_num:
trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first()
if logistics.trailer_int_id is not None:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_id == logistics.trailer_int_id)
.first()
)
else:
trl_obj = (
db.query(Trailer)
.filter(Trailer.trailer_number == logistics.trailer_num)
.first()
)
if trl_obj:
placas_remolque_val = trl_obj.plate_number or ""

View File

@@ -5,6 +5,7 @@ from pydantic import BaseModel
class DriverBaseDTO(BaseModel):
transporter_key: str
driver_id: Optional[int] = None
line: int
driver_name: Optional[str] = None
license_number: Optional[str] = None

View File

@@ -1,6 +1,6 @@
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
from core.database import Base
from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
from sqlalchemy import BigInteger, Column, ForeignKey, ForeignKeyConstraint, Integer, String
class Driver(Base, TenantScopedMixin, TimestampMixin):
@@ -16,6 +16,8 @@ class Driver(Base, TenantScopedMixin, TimestampMixin):
nullable=False,
)
line = Column(Integer, primary_key=True, nullable=False)
# Internal integer surrogate ID. The frontend continues to use transporter_key/line.
driver_id = Column(BigInteger, nullable=False, unique=True, index=True)
driver_name = Column(String(80), nullable=True)
license_number = Column(String(29), nullable=True)
express_line_id = Column(String(17), nullable=True)

View File

@@ -1,9 +1,17 @@
from typing import List, Optional
from sqlalchemy import text
from sqlalchemy.orm import Session
from . import dto, models
DRIVER_ID_SEQ = "a76.driver_driver_id_seq"
def allocate_driver_id(db: Session) -> int:
"""Next surrogate driver_id (sequence from migration ca7d3c4e8b2a)."""
return db.execute(text(f"SELECT nextval('{DRIVER_ID_SEQ}')")).scalar()
class DriverService:
@staticmethod
@@ -34,7 +42,10 @@ class DriverService:
@staticmethod
def create_driver(db: Session, driver_data: dto.DriverCreateDTO):
new_driver = models.Driver(**driver_data.dict())
data = driver_data.model_dump()
if data.get("driver_id") is None:
data["driver_id"] = allocate_driver_id(db)
new_driver = models.Driver(**data)
db.add(new_driver)
db.commit()
db.refresh(new_driver)

View File

@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
class TrailerBaseDTO(BaseModel):
trailer_number: str = Field(..., description="Trailer number (primary identifier)")
trailer_id: Optional[int] = Field(None, description="Internal trailer integer ID")
ace_trailer_number: Optional[str] = None
trailer_type_key: Optional[str] = None
seal: Optional[str] = None

View File

@@ -1,6 +1,6 @@
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
from core.database import Base
from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, String
from sqlalchemy import BigInteger, Column, ForeignKey, ForeignKeyConstraint, String
class Trailer(Base, TenantScopedMixin, TimestampMixin):
@@ -10,6 +10,8 @@ class Trailer(Base, TenantScopedMixin, TimestampMixin):
)
trailer_number = Column(String(20), primary_key=True, nullable=False)
# Internal integer identifier (surrogate). The frontend continues to use trailer_number.
trailer_id = Column(BigInteger, nullable=False, unique=True, index=True)
ace_trailer_number = Column(String(10), nullable=True)
trailer_type_key = Column(
String(2), ForeignKey("public.trailer_type.trailer_type_key"), nullable=True

View File

@@ -1,9 +1,17 @@
from typing import Optional, Tuple, List, Dict, Any
from sqlalchemy.orm import Session
from sqlalchemy import text
from . import dto, models
TRAILER_ID_SEQ = "a76.trailer_trailer_id_seq"
def allocate_trailer_id(db: Session) -> int:
"""Next surrogate trailer_id (sequence from migration ca7d3c4e8b2a)."""
return db.execute(text(f"SELECT nextval('{TRAILER_ID_SEQ}')")).scalar()
class TrailerService:
"""Service for Trailer CRUD operations with tenant support"""
@@ -66,8 +74,11 @@ class TrailerService:
company_id: int,
) -> models.Trailer:
"""Create a new trailer"""
data = trailer_data.model_dump()
if data.get("trailer_id") is None:
data["trailer_id"] = allocate_trailer_id(db)
new_trailer = models.Trailer(
**trailer_data.model_dump(), tenant_id=tenant_id, company_id=company_id
**data, tenant_id=tenant_id, company_id=company_id
)
db.add(new_trailer)
db.commit()

View File

@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
class TransporterBaseDTO(BaseModel):
transporter_key: str = Field(..., description="Transporter key (primary identifier)")
transporter_id: Optional[int] = Field(None, description="Internal transporter integer ID")
name: Optional[str] = None
short_name: Optional[str] = None
responsible: Optional[str] = None

View File

@@ -1,6 +1,6 @@
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
from core.database import Base
from sqlalchemy import Column, ForeignKeyConstraint, String
from sqlalchemy import BigInteger, Column, ForeignKeyConstraint, String
class Transporter(Base, TenantScopedMixin, TimestampMixin):
@@ -10,6 +10,8 @@ class Transporter(Base, TenantScopedMixin, TimestampMixin):
)
transporter_key = Column(String(23), primary_key=True, nullable=False)
# Internal integer identifier (surrogate). The frontend continues to use transporter_key.
transporter_id = Column(BigInteger, nullable=False, unique=True, index=True)
name = Column(String(256), nullable=True)
short_name = Column(String(10), nullable=True)
responsible = Column(String(100), nullable=True)

View File

@@ -2,12 +2,19 @@ from typing import Optional, Tuple, List, Dict, Any
import logging
from sqlalchemy.orm import Session
from sqlalchemy import func
from sqlalchemy import func, text
from . import dto, models
logger = logging.getLogger(__name__)
TRANSPORTER_ID_SEQ = "a76.transporter_transporter_id_seq"
def allocate_transporter_id(db: Session) -> int:
"""Next surrogate transporter_id (sequence from migration ca7d3c4e8b2a)."""
return db.execute(text(f"SELECT nextval('{TRANSPORTER_ID_SEQ}')")).scalar()
class TransporterService:
"""Service for Transporter CRUD operations with tenant support"""
@@ -91,8 +98,11 @@ class TransporterService:
company_id: int,
) -> models.Transporter:
"""Create a new transporter"""
data = transporter_data.model_dump()
if data.get("transporter_id") is None:
data["transporter_id"] = allocate_transporter_id(db)
new_transporter = models.Transporter(
**transporter_data.model_dump(), tenant_id=tenant_id, company_id=company_id
**data, tenant_id=tenant_id, company_id=company_id
)
db.add(new_transporter)
db.commit()

View File

@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
class VehicleBaseDTO(BaseModel):
vehicle_key: str = Field(..., description="Vehicle key (primary identifier)")
vehicle_id: Optional[int] = Field(None, description="Internal vehicle integer ID")
ace_vehicle_key: Optional[str] = None
transporter_key: Optional[str] = None
transport_identifier: Optional[str] = None

View File

@@ -1,6 +1,6 @@
from api.v1.common.base_models import TenantScopedMixin, TimestampMixin
from core.database import Base
from sqlalchemy import DECIMAL, Column, Integer, String
from sqlalchemy import DECIMAL, Column, Integer, BigInteger, String
class Vehicle(Base, TenantScopedMixin, TimestampMixin):
@@ -10,6 +10,8 @@ class Vehicle(Base, TenantScopedMixin, TimestampMixin):
)
vehicle_key = Column(String(14), primary_key=True, nullable=False)
# Internal integer identifier (surrogate). The frontend continues to use vehicle_key.
vehicle_id = Column(BigInteger, nullable=False, unique=True, index=True)
ace_vehicle_key = Column(String(10), nullable=True)
transporter_key = Column(String(23), nullable=True)
transport_identifier = Column(String(30), nullable=True)

View File

@@ -1,9 +1,17 @@
from typing import Optional, Tuple, List, Dict, Any
from sqlalchemy.orm import Session
from sqlalchemy import text
from . import dto, models
VEHICLE_ID_SEQ = "a76.vehicle_vehicle_id_seq"
def allocate_vehicle_id(db: Session) -> int:
"""Next surrogate vehicle_id (sequence from migration ca7d3c4e8b2a)."""
return db.execute(text(f"SELECT nextval('{VEHICLE_ID_SEQ}')")).scalar()
class VehicleService:
"""Service for Vehicle CRUD operations with tenant support"""
@@ -66,8 +74,11 @@ class VehicleService:
company_id: int,
) -> models.Vehicle:
"""Create a new vehicle"""
data = vehicle_data.model_dump()
if data.get("vehicle_id") is None:
data["vehicle_id"] = allocate_vehicle_id(db)
new_vehicle = models.Vehicle(
**vehicle_data.model_dump(), tenant_id=tenant_id, company_id=company_id
**data, tenant_id=tenant_id, company_id=company_id
)
db.add(new_vehicle)
db.commit()

View File

@@ -1,13 +1,24 @@
import shutil
import mimetypes
import os
import shutil
import uuid
from datetime import datetime
from typing import List, Optional, Dict, Any
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Header, status, UploadFile, File
from fastapi.responses import Response
from sqlalchemy.orm import Session
from core.database import get_core_db
from core.config import settings
from core.database import get_core_db
from core.s3_keys import (
help_asset_key,
help_public_api_path,
help_s3_key_to_public_relative_path,
system_help_object_key,
)
from core.storage_s3 import get_object_bytes, put_object_bytes
from core.security import get_current_user, has_role
from .schemas import HelpArticleInDB, HelpArticleCreate, HelpArticleUpdate, HelpSyncRequest, HelpSyncResponse
from .services import HelpCenterService
@@ -80,60 +91,95 @@ def sync_help_article(sync_data: HelpSyncRequest, db: Session = Depends(get_core
return result
@router.get("/files/{file_path:path}")
def serve_help_file(file_path: str):
"""Sirve un objeto bajo system/help/ (público vía middleware)."""
if ".." in file_path or file_path.startswith("/"):
raise HTTPException(status_code=404, detail="Not found")
try:
key = system_help_object_key(file_path)
except ValueError:
raise HTTPException(status_code=404, detail="Not found")
if not settings.use_s3_object_storage:
legacy = os.path.join("uploads", "help", file_path)
if not os.path.isfile(legacy):
raise HTTPException(status_code=404, detail="Not found")
with open(legacy, "rb") as f:
data = f.read()
media = mimetypes.guess_type(file_path)[0] or "application/octet-stream"
return Response(content=data, media_type=media)
try:
data = get_object_bytes(key)
except Exception:
raise HTTPException(status_code=404, detail="Not found")
media = mimetypes.guess_type(file_path)[0] or "application/octet-stream"
return Response(content=data, media_type=media)
@router.post("/upload-image/")
def upload_help_image(
async def upload_help_image(
file: UploadFile = File(...),
current_user: Dict[str, Any] = Depends(has_role("admin"))
):
"""Sube una imagen para usar en los artículos."""
try:
file_ext = os.path.splitext(file.filename)[1]
file_ext = os.path.splitext(file.filename or "")[1] or ".png"
new_filename = f"{uuid.uuid4()}{file_ext}"
file_location = f"uploads/help/{new_filename}"
# Ensure directory exists
body = await file.read()
if settings.use_s3_object_storage:
key = help_asset_key("", new_filename)
ct = mimetypes.guess_type(new_filename)[0] or "image/png"
put_object_bytes(key, body, content_type=ct)
rel = help_s3_key_to_public_relative_path(key)
return {"url": help_public_api_path(rel)}
os.makedirs("uploads/help", exist_ok=True)
with open(file_location, "wb+") as buffer:
shutil.copyfileobj(file.file, buffer)
file_location = f"uploads/help/{new_filename}"
with open(file_location, "wb") as f:
f.write(body)
return {"url": f"/api/uploads/help/{new_filename}"}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/upload-asset/")
def upload_help_asset(
async def upload_help_asset(
file: UploadFile = File(...),
current_user: Dict[str, Any] = Depends(has_role("admin"))
):
"""Sube cualquier tipo de archivo (PDF, Video, etc.) para la biblioteca."""
try:
file_ext = os.path.splitext(file.filename)[1].lower()
file_ext = os.path.splitext(file.filename or "")[1].lower()
new_filename = f"{uuid.uuid4()}{file_ext}"
# Guardar en una carpeta segun el tipo o general
folder = "uploads/help/assets"
if file_ext in ['.pdf']:
folder = "uploads/help/pdfs"
elif file_ext in ['.mp4', '.mov', '.avi']:
folder = "uploads/help/videos"
file_location = f"{folder}/{new_filename}"
# Ensure directory exists
subfolder = "assets"
if file_ext == ".pdf":
subfolder = "pdfs"
elif file_ext in [".mp4", ".mov", ".avi"]:
subfolder = "videos"
if settings.use_s3_object_storage:
body = await file.read()
key = help_asset_key(subfolder, new_filename)
ct = file.content_type or mimetypes.guess_type(new_filename)[0] or "application/octet-stream"
put_object_bytes(key, body, content_type=ct)
rel = help_s3_key_to_public_relative_path(key)
return {
"url": help_public_api_path(rel),
"filename": file.filename,
"size": len(body),
"mime_type": file.content_type,
}
folder = f"uploads/help/{subfolder}"
os.makedirs(folder, exist_ok=True)
with open(file_location, "wb+") as buffer:
shutil.copyfileobj(file.file, buffer)
# Get file size
file_location = f"{folder}/{new_filename}"
body = await file.read()
with open(file_location, "wb") as f:
f.write(body)
file_size = os.path.getsize(file_location)
return {
"url": f"/api/{file_location}",
"filename": file.filename,
"size": file_size,
"mime_type": file.content_type
"mime_type": file.content_type,
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))

View File

@@ -1,76 +1,111 @@
import logging
import mimetypes
import os
import re
import httpx
import logging
import uuid
from pathlib import Path
from typing import Optional
import httpx
from core.config import settings
from core.s3_keys import SYSTEM_HELP_PREFIX
from core.storage_s3 import object_exists, put_object_bytes
logger = logging.getLogger(__name__)
def _asset_url_to_s3_key(asset_url: str) -> Optional[str]:
"""Deriva la clave S3 bajo system/help/ a partir de una URL de artículo."""
if "/help-center/files/" in asset_url:
rel = asset_url.split("/help-center/files/", 1)[1].lstrip("/")
if ".." in rel:
return None
return f"{SYSTEM_HELP_PREFIX}{rel}"
u = asset_url.replace("/api/uploads/", "uploads/")
if u.startswith("/"):
u = u[1:]
if u.startswith("uploads/help/"):
return f"{SYSTEM_HELP_PREFIX}{u[len('uploads/help/') :]}"
return None
def download_file_from_hub(relative_path: str) -> bool:
"""
Downloads a file from the Hub to the local storage.
relative_path: e.g., 'uploads/help/pdfs/myfile.pdf' or '/api/uploads/help/image.png'
Descarga un asset del Hub y lo guarda en MinIO (system/help/...) o en disco si no hay almacenamiento S3 activo.
relative_path: URL parcial, p. ej. '/api/uploads/help/x.png' o '/api/v1/core/help-center/files/pdfs/x.pdf'
"""
if not settings.CENTRAL_SERVER_URL or settings.CENTRAL_SERVER_URL == '""':
return False
# Clean the path
clean_path = relative_path.replace("/api/uploads/", "uploads/")
if clean_path.startswith("/"):
clean_path = clean_path[1:]
# Check if it starts with uploads
if not clean_path.startswith("uploads/"):
# If it doesn't start with uploads, it might just be the filename or a subpath
# We assume it's relative to /app/
pass
key = _asset_url_to_s3_key(relative_path)
if not key:
logger.warning("download_file_from_hub: could not map URL to S3 key: %s", relative_path)
return False
local_path = Path(clean_path)
if local_path.exists():
logger.info(f"File {clean_path} already exists, skipping download.")
if settings.use_s3_object_storage and object_exists(key):
logger.info("S3 object %s already exists, skipping download.", key)
return True
# Ensure directories exist
local_path.parent.mkdir(parents=True, exist_ok=True)
# Resolve Hub Base URL
# CENTRAL_SERVER_URL is usually http://hub:8000/api/v1/core/help-center/sync/
# We want http://hub:8000/api/
base_url = settings.CENTRAL_SERVER_URL.split("/v1/")[0]
# The file in backend is served usually under /api/uploads/...
# But clean_path is just "uploads/...". So the Hub route is base_url + "/" + clean_path
hub_file_url = f"{base_url}/{clean_path}"
if "/help-center/files/" in relative_path:
rel = relative_path.split("/help-center/files/", 1)[1].lstrip("/")
hub_file_url = f"{base_url.rstrip('/')}/api/v1/core/help-center/files/{rel}"
else:
clean_path = relative_path.replace("/api/uploads/", "uploads/")
if clean_path.startswith("/"):
clean_path = clean_path[1:]
hub_file_url = f"{base_url.rstrip('/')}/{clean_path}"
logger.info("Downloading asset from Hub: %s", hub_file_url)
logger.info(f"Downloading asset from Hub: {hub_file_url} -> {local_path}")
try:
with httpx.Client() as client:
response = client.get(hub_file_url, timeout=30.0)
if response.status_code == 200:
with open(local_path, "wb") as f:
f.write(response.content)
logger.info(f"Successfully downloaded {clean_path}")
return True
else:
logger.warning(f"Failed to download {clean_path}: Status {response.status_code} URL: {hub_file_url}")
if response.status_code != 200:
logger.warning(
"Failed to download %s: Status %s URL: %s",
relative_path,
response.status_code,
hub_file_url,
)
return False
body = response.content
except Exception as e:
logger.error(f"Error downloading {clean_path}: {str(e)}")
logger.error("Error downloading %s: %s", relative_path, str(e))
return False
if settings.use_s3_object_storage:
rel = key[len(SYSTEM_HELP_PREFIX) :]
ct = mimetypes.guess_type(rel)[0] or "application/octet-stream"
try:
put_object_bytes(key, body, content_type=ct)
logger.info("Stored hub asset in S3: %s", key)
return True
except Exception as e:
logger.error("S3 put failed for %s: %s", key, e)
return False
rel = key[len(SYSTEM_HELP_PREFIX) :]
local_path = Path("uploads/help") / rel
local_path.parent.mkdir(parents=True, exist_ok=True)
local_path.write_bytes(body)
logger.info("Stored hub asset locally: %s", local_path)
return True
def sync_assets_from_content(content: str):
"""
Parses markdown content for image URLs and downloads them if they are local references.
Example: ![alt text](/api/uploads/help/uuid.png)
"""
"""Parsea markdown y descarga imágenes referenciadas (rutas legacy y nuevas)."""
if not content:
return
# Regex for markdown images: ![...](/api/uploads/...)
image_pattern = r'!\[.*?\]\((/api/uploads/.*?)\)'
matches = re.findall(image_pattern, content)
for asset_url in matches:
download_file_from_hub(asset_url)
patterns = [
r'!\[.*?\]\((/api/uploads/.*?)\)',
r'!\[.*?\]\((/api/v1/core/help-center/files/.*?)\)',
]
seen = set()
for pattern in patterns:
for asset_url in re.findall(pattern, content):
if asset_url in seen:
continue
seen.add(asset_url)
download_file_from_hub(asset_url)

View File

@@ -2,14 +2,20 @@
Rutas para gestión de usuarios de Keycloak
"""
import logging
import mimetypes
from typing import Optional
import os
import uuid
from pathlib import Path
from core.config import settings
from core.database import get_core_db
from core.s3_keys import public_user_avatar_api_path, user_avatar_key
from core.storage_s3 import delete_object_if_exists, get_object_bytes, put_object_bytes
from core.security import get_current_user, validate_access_to_resource
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
from fastapi.responses import Response
from sqlalchemy.orm import Session
from ..user_tenant.models import UserTenant
@@ -25,6 +31,10 @@ from .service import UserService
router = APIRouter(prefix="/users", tags=["Users"])
logger = logging.getLogger(__name__)
_AVATAR_EXT = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
@router.get("/stats", response_model=UserStatsDTO)
def get_user_statistics(
@@ -67,6 +77,52 @@ def list_users(
return result
@router.get("/avatar/{tenant_id}/{keycloak_user_id}")
def get_user_avatar_image(
tenant_id: int,
keycloak_user_id: str,
db: Session = Depends(get_core_db),
):
"""
Sirve la imagen de avatar (público para poder usarla en <img src> sin Bearer).
El almacenamiento interno puede ser clave S3 o ruta bajo uploads/.
"""
ut = (
db.query(UserTenant)
.filter(
UserTenant.tenant_id == tenant_id,
UserTenant.keycloak_user_id == keycloak_user_id,
UserTenant.is_active == True,
)
.first()
)
if not ut or not ut.avatar_url:
raise HTTPException(status_code=404, detail="Avatar not found")
raw = ut.avatar_url
if raw.startswith("tenants/"):
try:
data = get_object_bytes(raw)
except Exception:
raise HTTPException(status_code=404, detail="Avatar not found")
media = mimetypes.guess_type(raw)[0] or "image/jpeg"
return Response(content=data, media_type=media)
rel = raw.lstrip("/")
path = Path(rel)
if not path.is_file():
path = Path.cwd() / rel
if not path.is_file():
alt = Path("/app") / rel
if alt.is_file():
path = alt
if not path.is_file():
raise HTTPException(status_code=404, detail="Avatar file not found")
data = path.read_bytes()
media = mimetypes.guess_type(str(path))[0] or "image/jpeg"
return Response(content=data, media_type=media)
@router.get("/{user_id}", response_model=UserResponseDTO)
def get_user(
user_id: str,
@@ -282,33 +338,74 @@ async def upload_avatar(
db: Session = Depends(get_core_db),
):
"""
Sube un avatar para el usuario actual
Retorna la URL del avatar subido
Sube un avatar para el usuario actual.
Con MinIO guarda en tenants/{tid}/users/{sub}/avatar.{ext} y persiste la clave en UserTenant.
Retorna URL pública para <img src> (GET /users/avatar/...).
"""
# Validar tipo de archivo
if not file.content_type or not file.content_type.startswith("image/"):
raise HTTPException(status_code=400, detail="El archivo debe ser una imagen")
# Validar tamaño (max 2MB)
keycloak_user_id = current_user.get("sub")
if not keycloak_user_id:
raise HTTPException(status_code=400, detail="User ID not found in token")
user_tenant = (
db.query(UserTenant)
.filter(
UserTenant.keycloak_user_id == keycloak_user_id,
UserTenant.is_active == True,
)
.first()
)
if not user_tenant:
raise HTTPException(
status_code=400, detail="User does not belong to any tenant"
)
ext = Path(file.filename or "image.jpg").suffix.lower() or ".jpg"
if ext not in _AVATAR_EXT:
raise HTTPException(
status_code=400,
detail=f"Extensión no permitida. Use: {', '.join(sorted(_AVATAR_EXT))}",
)
contents = await file.read()
if len(contents) > 2 * 1024 * 1024:
raise HTTPException(status_code=400, detail="La imagen debe ser menor a 2MB")
# Crear directorio si no existe
upload_dir = Path("/app/uploads/avatars")
upload_dir.mkdir(parents=True, exist_ok=True)
tenant_id = user_tenant.tenant_id
# Generar nombre con keycloak_user_id (sobrescribe si existe)
keycloak_user_id = current_user.get("sub")
ext = Path(file.filename or "image.jpg").suffix
filename = f"{keycloak_user_id}{ext}"
file_path = upload_dir / filename
try:
if settings.use_s3_object_storage:
if user_tenant.avatar_url and str(user_tenant.avatar_url).startswith(
"tenants/"
):
delete_object_if_exists(str(user_tenant.avatar_url))
key = user_avatar_key(tenant_id, keycloak_user_id, ext)
ct = file.content_type or mimetypes.guess_type(f"x{ext}")[0] or "image/jpeg"
put_object_bytes(key, contents, content_type=ct)
user_tenant.avatar_url = key
logger.info(
"User avatar stored in S3 key=%s bytes=%s", key, len(contents)
)
else:
upload_dir = Path("uploads/avatars")
upload_dir.mkdir(parents=True, exist_ok=True)
filename = f"{keycloak_user_id}{ext}"
file_path = upload_dir / filename
with open(file_path, "wb") as f:
f.write(contents)
user_tenant.avatar_url = f"/uploads/avatars/{filename}"
# Guardar archivo
with open(file_path, "wb") as f:
f.write(contents)
db.add(user_tenant)
db.commit()
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
except Exception as e:
db.rollback()
raise HTTPException(
status_code=500, detail=f"Error al guardar el avatar: {str(e)}"
) from e
# Retornar URL relativa
avatar_url = f"/uploads/avatars/{filename}"
return {"avatar_url": avatar_url}
public_url = public_user_avatar_api_path(tenant_id, keycloak_user_id)
return {"avatar_url": public_url}

View File

@@ -43,9 +43,16 @@ def _normalize_keycloak_user(
# Agregar campos de perfil si user_tenant está disponible
if user_tenant:
avatar_out = user_tenant.avatar_url
if avatar_out and str(avatar_out).startswith("tenants/"):
from core.s3_keys import public_user_avatar_api_path
avatar_out = public_user_avatar_api_path(
user_tenant.tenant_id, user_tenant.keycloak_user_id
)
normalized.update(
{
"avatar_url": user_tenant.avatar_url,
"avatar_url": avatar_out,
"phone": user_tenant.phone,
"bio": user_tenant.bio,
"preferences": user_tenant.preferences or {},
@@ -435,7 +442,9 @@ class UserService:
# Actualizar campos en UserTenant
if role is not None:
user_tenant.role = role
if avatar_url is not None:
if avatar_url is not None and not str(avatar_url).startswith(
"/api/v1/core/users/avatar/"
):
user_tenant.avatar_url = avatar_url
if phone is not None:
user_tenant.phone = phone
@@ -641,7 +650,9 @@ class UserService:
self.keycloak_admin.update_user(keycloak_user_id, update_data)
# Actualizar campos de perfil en UserTenant
if avatar_url is not None:
if avatar_url is not None and not str(avatar_url).startswith(
"/api/v1/core/users/avatar/"
):
user_tenant.avatar_url = avatar_url
if phone is not None:
user_tenant.phone = phone