ADUANASOFT/plantillas-proyectos
13
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature/csv-for-envoices

Browse Source
This commit is contained in:
hreyes
2026-02-23 13:05:02 -06:00
parent bbe74906ec
commit 0683aaa801
21 changed files with 1232 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
backend/api/v1/modules/a76/__init__.py Normal file
View File

141
backend/api/v1/modules/a76/imports/routes.py
View File

@@ -1,5 +1,6 @@
from datetime import datetime
from uuid import uuid4
import base64
import os
import json
import logging
@@ -12,24 +13,39 @@ from core.config import settings
from core.database import get_core_db
from core.security import get_current_user, validate_access_to_resource
from .tasks import scan_file, insert_valid_rows
from .tasks import (
scan_file,
insert_valid_rows,
IMPORT_FILE_KEY_PREFIX,
IMPORT_META_KEY_PREFIX,
IMPORT_REDIS_TTL,
)
from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest
router = APIRouter()
logger = logging.getLogger(__name__)
def _get_redis():
"""Redis client (same broker as Celery so worker can read)."""
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
@router.post("/upload/{model_target}", response_model=ImportJobResponse)
async def upload_import_file(
model_target: Literal["invoice_header", "invoice_details"],
file: UploadFile = File(...),
footer_config: Optional[str] = Form(None), # JSON string with settings
company_id: int = Query(..., description="Company ID"), # Required for context
template_id: Optional[str] = Form(None), # id de la plantilla (ej. imp_temp_header) para respetar columnas
company_id: int = Query(..., description="Company ID"), # Required for context
operation_type: Optional[str] = Query("imp"),
db: Session = Depends(get_core_db),
current_user: Dict[str, Any] = Depends(get_current_user),
):
"""
Step 1: Upload CSV, save to temp, trigger scan task.
Si se envía template_id, solo se leen las columnas de esa plantilla.
"""
# 1. Validate Access & Get Tenant
try:
@@ -40,40 +56,51 @@ async def upload_import_file(
if not file.filename.endswith(".csv"):
raise HTTPException(status_code=400, detail="Only .csv files allowed")
job_id = str(uuid4())
# Ensure directory exists (Safety check)
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"{job_id}.csv")
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
contents = await file.read()
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"footer_config": footer_config,
"operation_type": operation_type,
"template_id": template_id,
}
# Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed)
try:
# Save CSV
contents = await file.read()
redis_client = _get_redis()
redis_client.set(
f"{IMPORT_FILE_KEY_PREFIX}{job_id}",
base64.b64encode(contents),
ex=IMPORT_REDIS_TTL,
)
redis_client.set(
f"{IMPORT_META_KEY_PREFIX}{job_id}",
json.dumps(meta_data).encode("utf-8"),
ex=IMPORT_REDIS_TTL,
)
except Exception as e:
logger.error(f"Redis store error: {e}")
raise HTTPException(status_code=500, detail="Failed to queue file for processing.")
# Optional: also write to local disk (e.g. for same-machine worker or debugging)
try:
upload_dir = os.path.join(os.getcwd(), "uploads", "temp")
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"{job_id}.csv")
meta_path = os.path.join(upload_dir, f"{job_id}.meta.json")
with open(file_path, "wb") as f:
f.write(contents)
# Save Metadata (Context)
meta_data = {
"tenant_id": tenant_id,
"company_id": company_id,
"user_id": current_user.get("id"),
"footer_config": footer_config,
"operation_type": operation_type,
}
with open(meta_path, "w") as f:
json.dump(meta_data, f)
except Exception as e:
logger.error(f"File save error: {e}")
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
logger.warning(f"Local file save failed (worker will use Redis): {e}")
# Trigger Celery Task (Async)
# Use our job_id as the Celery task_id for easier tracking
scan_file.apply_async(args=[job_id, file_path, model_target, footer_config], task_id=job_id)
# Trigger Celery Task (Async). Worker loads file from Redis.
scan_file.apply_async(args=[job_id, model_target, footer_config], task_id=job_id)
return ImportJobResponse(
job_id=job_id,
@@ -84,24 +111,56 @@ async def upload_import_file(
@router.get("/{job_id}/status")
async def get_import_status(job_id: str):
"""
Poll this endpoint to get % progress or final report.
Poll to get progress or final report. Always returns an object with "status".
"""
# In a real app, query Redis or DB.
# For MVP, we might mock or use Celery AsyncResult if backend shares Redis.
task_result = celery_app.AsyncResult(job_id)
if task_result.state == 'PENDING':
if task_result.state == "PENDING":
return {"status": "processing", "progress": 0}
elif task_result.state == 'PROGRESS':
if task_result.state == "PROGRESS":
return {
"status": "processing",
"progress": task_result.info.get('current', 0),
"total": task_result.info.get('total', 0)
"status": "processing",
"progress": (task_result.info or {}).get("current", 0),
"total": (task_result.info or {}).get("total", 0),
}
elif task_result.state == 'SUCCESS':
return task_result.result # Should return the report
else:
return {"status": task_result.state, "error": str(task_result.info)}
if task_result.state == "SUCCESS":
result = task_result.result
if isinstance(result, dict) and "status" in result:
return result
return {"status": "finished", "result": result}
# FAILURE: obtener mensaje real (traceback, result o get(propagate=False))
logger.warning("Import task %s failed: state=%s", job_id, task_result.state)
err_msg = None
tb = getattr(task_result, "traceback", None)
if tb:
logger.debug("Task traceback: %s", tb[:500] if isinstance(tb, str) else tb)
if tb and isinstance(tb, str):
lines = [l.strip() for l in tb.strip().split("\n") if l.strip()]
if lines:
err_msg = lines[-1]
if not err_msg and len(lines) > 1:
err_msg = lines[-2] + " " + (lines[-1] or "")
if not err_msg:
try:
exc = task_result.get(propagate=False)
if exc is not None:
err_msg = str(exc)
except Exception:
pass
if not err_msg:
result = getattr(task_result, "result", None)
info = getattr(task_result, "info", None)
if result is not None and not isinstance(result, dict):
err_msg = str(result)
elif isinstance(result, dict) and (result.get("error") or result.get("message")):
err_msg = result.get("error") or result.get("message")
if not err_msg and isinstance(info, str):
err_msg = info
elif not err_msg and isinstance(info, dict) and "error" in info:
err_msg = str(info["error"])
if not err_msg:
err_msg = "Task failed"
return {"status": "failed", "error": err_msg}
@router.post("/{job_id}/commit")

810
backend/api/v1/modules/a76/imports/tasks.py
View File

File diff suppressed because it is too large Load Diff

118
backend/api/v1/modules/a76/imports/template_config.py Normal file
View File

@@ -0,0 +1,118 @@
"""
Configuración de plantillas CSV: columnas que trae cada plantilla y cómo se mapean.
La plantilla se respeta tal cual: solo se leen columnas definidas aquí; el resto se ignora.
Solo se escribe en BD lo que los modelos de facturas aceptan (respetando models).
"""
from typing import Dict, List, Any, Optional
# Cada plantilla define sus columnas canónicas y alias (otros nombres que aceptamos en el CSV).
# canonical = nombre estándar con el que trabajamos internamente; debe coincidir con lo que
# espera la lógica de validación e insert (tasks.py).
# aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA).
TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = {
# --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) ---
"imp_temp_header": [
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]},
{"canonical": "FECHA FACTURA", "aliases": ["FECHA"]},
{"canonical": "FECHA EMISION"},
{"canonical": "CLAVE PROVEEDOR"},
{"canonical": "CLAVE VENDIDO A"},
{"canonical": "CLAVE ENVIADO A"},
{"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]},
{"canonical": "ADUANA DE CRUCE"},
{"canonical": "CLAVE MONEDA"},
{"canonical": "CLAVE INCOTERM"},
{"canonical": "TIPO MONEDA"},
{"canonical": "TIPO DE CAMBIO"},
{"canonical": "TIPO PESO"},
{"canonical": "TIPO TRANSPORTE"},
{"canonical": "REMESA"},
{"canonical": "AGENTE ADUANAL"},
{"canonical": "FLETES"},
{"canonical": "VALOR SEGUROS"},
{"canonical": "SEGUROS"},
{"canonical": "EMBALAJES"},
{"canonical": "OTROS INCREMENTABLES"},
{"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]},
{"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]},
{"canonical": "FACTURA ALTERNA"},
{"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]},
{"canonical": "OBSERVACIONES E"},
{"canonical": "OBSERVACIONES I"},
{"canonical": "E DOCUMENT"},
{"canonical": "NUM OPERACION"},
{"canonical": "CLAVE TRANSPORTISTA"},
{"canonical": "NOMBRE CONDUCTOR"},
{"canonical": "NUMERO TRANSPORTE"},
{"canonical": "PRECINTO"},
],
# --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura ---
"imp_def_header": None, # se resuelve igual que imp_temp_header
# --- Encabezado factura: Expo (EstructuraEncFacExpoCamReg.xls) - misma estructura ---
"exp_def_header": None,
# --- Partidas factura: Impo Temp (EstructuraParFacImpoTempAF.xls) ---
"imp_temp_details": [
{"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA"]},
{"canonical": "LINEA", "aliases": ["RENGLON", "PARTIDA"]},
{"canonical": "NUMPARTE", "aliases": ["NUMERO PARTE"]},
{"canonical": "PRECIO UNITARIO", "aliases": ["PRECIOUNITARIO"]},
{"canonical": "VALOR COMERCIAL", "aliases": ["VALORCOMERCIAL"]},
{"canonical": "CANTIDAD"},
{"canonical": "CANTIDAD BULTOS", "aliases": ["CANTIDADBULTOS"]},
{"canonical": "DESCRIPCION"},
{"canonical": "PAIS ORIGEN", "aliases": ["PAISORIGEN"]},
{"canonical": "FRACCION"},
{"canonical": "ORDEN DE COMPRA", "aliases": ["ORDENCOMPRA"]},
],
# --- Partidas: Impo Def y Expo - misma estructura ---
"imp_def_details": None,
"exp_def_details": None,
}
def _resolve_template_columns(template_id: str) -> Optional[List[Dict[str, Any]]]:
cols = TEMPLATE_COLUMNS.get(template_id)
if cols is not None:
return cols
if template_id in ("imp_def_header", "exp_def_header"):
return TEMPLATE_COLUMNS.get("imp_temp_header")
if template_id in ("imp_def_details", "exp_def_details"):
return TEMPLATE_COLUMNS.get("imp_temp_details")
return None
def build_normalized_lookup(template_id: str, normalize_header_fn) -> Dict[str, str]:
"""
Construye un diccionario: normalized_header -> canonical_name.
normalize_header_fn(str) -> str debe ser la función que normaliza cabeceras (ej. mayúsculas, sin acentos).
"""
cols = _resolve_template_columns(template_id)
if not cols:
return {}
lookup: Dict[str, str] = {}
for item in cols:
canonical = item["canonical"]
lookup[normalize_header_fn(canonical)] = canonical
for alias in item.get("aliases") or []:
lookup[normalize_header_fn(alias)] = canonical
return lookup
def row_from_template(row: Dict[str, Any], template_id: str, normalize_header_fn) -> Dict[str, Any]:
"""
A partir de una fila CSV (dict header->value) y un template_id, devuelve un dict
solo con las columnas de la plantilla, usando nombres canónicos.
Así la plantilla se respeta: solo entran columnas definidas en la plantilla.
"""
lookup = build_normalized_lookup(template_id, normalize_header_fn)
if not lookup:
# Sin template definido: comportamiento legacy (normalizar todo)
return {normalize_header_fn(k): v for k, v in row.items()}
out: Dict[str, Any] = {}
for csv_header, value in row.items():
key_norm = normalize_header_fn(csv_header)
if key_norm in lookup:
out[lookup[key_norm]] = value
return out

1
backend/api/v1/modules/a76/invoices/catalog_service.py
View File

@@ -7,6 +7,7 @@ from api.v1.modules.public.reference_data.invoice_types.models import InvoiceTyp
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.public.reference_data.transport_types.models import TransportType
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.transport_modes.models import TransportMode

12
backend/api/v1/modules/a76/invoices/schemas.py
View File

@@ -19,6 +19,14 @@ class InvoiceHeaderBase(BaseModel):
operation_type: Optional[OperationType] = Field(
..., description="Operation type: imp/exp/sm/ctm"
)
@field_validator("operation_type", mode="before")
@classmethod
def normalize_operation_type(cls, v):
"""Accept DB string (e.g. 'IMP') and coerce to enum value ('imp')."""
if isinstance(v, str):
return v.lower() if v else v
return v
invoice_type: Optional[str] = Field(
None, max_length=5, description="Invoice type key"
)
@@ -198,9 +206,9 @@ class InvoiceComplianceMxBase(BaseModel):
class InvoiceFinancialsBase(BaseModel):
"""Base fields for Financials"""
currency: Currency = Field(None, max_length=7, description="Currency code")
currency: Optional[Currency] = Field(None, max_length=7, description="Currency code")
currency_type: Optional[str] = Field("USD", description="Currency type")
exchange_rate: Decimal = Field(0.00, description="Exchange rate")
exchange_rate: Optional[Decimal] = Field(0.00, description="Exchange rate")
exchange_rate_mm: Optional[Decimal] = Field(
None, description="Exchange rate currency to currency"
)

15
backend/api/v1/modules/a76/invoices/services.py
View File

@@ -45,11 +45,13 @@ class InvoiceService:
# Apply filters if provided
if filters:
if filters.get("status"):
query = query.filter(models.InvoiceHeader.status == filters["status"])
if filters.get("status") is not None:
query = query.filter(models.InvoiceHeader.is_updated == filters["status"])
if filters.get("operation_type"):
ot = filters["operation_type"]
ot_val = ot.value if hasattr(ot, "value") else ot
query = query.filter(
models.InvoiceHeader.operation_type == filters["operation_type"]
models.InvoiceHeader.operation_type == ot_val
)
if filters.get("invoice_type"):
query = query.filter(
@@ -67,10 +69,9 @@ class InvoiceService:
f"%{filters['pedimento']}%"
)
)
if (
not filters.get("invoice_type")
and filters.get("operation_type") == "exp"
):
ot_exp = filters.get("operation_type")
ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp
if not filters.get("invoice_type") and ot_exp_val == "exp":
query = query.filter(models.InvoiceHeader.operation_type != "REPAR")
if filters.get("manifest_number"):

2
backend/api/v1/modules/a76/router.py
View File

@@ -12,7 +12,7 @@ from .general_catalogs.router import router as general_catalogs_router
from .invoices.routes import router as invoices_router
from .items.routes import router as items_router
from .classes import router as classes_router
from .classes import router as classes_router
from .clients_and_providers import router as client_and_provider_router
from .imports.routes import router as imports_router
from .invoice_settings.routes import router as invoice_settings_router