ADUANASOFT/plantillas-proyectos
13
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
37920aa303bb2d7f515dfb6b0927f090968a64e6
plantillas-proyectos/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py
hreyes 3f16378ed0 feature/fix-invoice-status
2026-03-18 15:49:47 -06:00

5530 lines
284 KiB
Python
Raw Blame History

"""
Tareas Celery para importación CSV de facturas (encabezados, partidas, series).
Flujo: scan_file (validación) → insert_valid_rows (commit).
Objetivo en BD (paridad con flujo normal): al terminar el commit, los datos deben quedar
igual que por UI/API: encabezados con capture_user/who_processed; partidas con costos,
pesos y descripciones calculados/heredados según items/imports/validators; series con
campos no presentes en CSV en null. No se modifican plantillas CSV; no se inventan
datos sin fuente (p. ej. LineReference solo si hay fuente explícita).
"""
import os
from datetime import datetime
from decimal import Decimal
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List, Set, Tuple
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from core.paths import layout_path
from sqlalchemy import func
from ..common import storage as common_storage
from ..common import meta as common_meta
from ..common import responses as common_responses
from .template_config import row_from_template
# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process
logger = logging.getLogger(__name__)
# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py)
JOB_TYPE = ""
# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="")
IMPORT_FILE_KEY_PREFIX = "import_file:"
IMPORT_META_KEY_PREFIX = "import_meta:"
IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:"
IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
"""Usa common storage con job_type vacío (prefijo import_)."""
return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import")
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import")
def _delete_import_from_redis(job_id: str) -> None:
common_storage.delete_import_from_redis(JOB_TYPE, job_id)
class ForeignKeyValidator:
def __init__(self, session, tenant_id, company_id):
self.session = session
self.tenant_id = tenant_id
self.company_id = company_id
self.cache = {} # {(model_name, value): bool}
def check_exists(self, model, value, field_name="id", is_public=False):
if value is None:
return True # Assume optional if None, or let DB handle not-null
key = (model.__name__, value)
if key in self.cache:
return self.cache[key]
col = getattr(model, field_name)
if field_name == "short_name" and hasattr(model, "short_name"):
query = self.session.query(col).filter(func.upper(col) == (value.upper() if isinstance(value, str) else value))
else:
query = self.session.query(col).filter(col == value)
if not is_public:
query = query.filter(model.tenant_id == self.tenant_id, model.company_id == self.company_id)
exists = query.first() is not None
self.cache[key] = exists
return exists
TRANSPORT_TYPE_VALUES = {
"none",
"transport",
"box",
"licence plates",
"truck",
"vessel",
"rail_barge",
"container",
"airplane",
"gondola",
"flatbed",
}
def normalize_public_code(value: Optional[str]) -> Optional[str]:
if value is None:
return None
text = str(value).strip().upper()
return text or None
def validate_public_code(
validator: ForeignKeyValidator,
model,
value: Optional[str],
line_num: int,
col_name: str,
field_name: str = "code",
required: bool = False,
) -> Optional[Dict[str, Any]]:
code = normalize_public_code(value)
if not code:
if required:
return {"line": line_num, "col": col_name, "msg": "Requerido"}
return None
if not validator.check_exists(model, code, field_name=field_name, is_public=True):
return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"}
return None
def validate_tenant_fk_id(
validator: ForeignKeyValidator,
model,
value: Optional[int],
line_num: int,
col_name: str,
required: bool = False,
) -> Optional[Dict[str, Any]]:
if value is None:
if required:
return {"line": line_num, "col": col_name, "msg": "Requerido"}
return None
if not validator.check_exists(model, value):
return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"}
return None
def _validate_client_provider_ref(
validator: ForeignKeyValidator,
model,
raw_value: Any,
line_num: int,
col_name: str,
required: bool,
) -> Optional[Dict[str, Any]]:
"""Valida CLAVE PROVEEDOR / VENDIDO A / ENVIADO A: acepta ID (entero) o short_name (texto)."""
if raw_value is None or not str(raw_value).strip():
if required:
return {"line": line_num, "col": col_name, "msg": "Requerido"}
return None
pid = parse_int(raw_value)
if pid is not None:
return validate_tenant_fk_id(validator, model, pid, line_num, col_name, required=False)
short_norm = str(raw_value).strip().upper()
if not validator.check_exists(model, short_norm, field_name="short_name"):
return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"}
return None
def _validate_customs_broker_ref(
validator: ForeignKeyValidator,
model,
raw_value: Any,
line_num: int,
col_name: str,
required: bool = False,
) -> Optional[Dict[str, Any]]:
"""Valida AGENTE ADUANAL: acepta ID (entero) o clave broker_key (texto)."""
if raw_value is None or not str(raw_value).strip():
if required:
return {"line": line_num, "col": col_name, "msg": "Requerido"}
return None
pid = parse_int(raw_value)
if pid is not None:
return validate_tenant_fk_id(validator, model, pid, line_num, col_name, required=False)
clave = str(raw_value).strip()
if not validator.check_exists(model, clave, field_name="broker_key"):
return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"}
return None
@celery_app.task(bind=True)
def scan_file(self, job_id: str, model_target: str, config: str = None, job_type_override: Optional[str] = None):
"""Pass 1: Read CSV, Validate types, Write Errors to JSONL. Delegates to _do_scan_file."""
return _do_scan_file(self, job_id, model_target, config, job_type_override)
def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = None, job_type_override: Optional[str] = None) -> Dict[str, Any]:
"""Pass 1 body: load file/meta from storage, run validations, store error lines. Uses effective_job_type for storage."""
effective_job_type = job_type_override if job_type_override is not None else JOB_TYPE
log_prefix = "Exportación import" if effective_job_type else "Invoices import"
logger.info(f"Starting scan for job {job_id} target {model_target}")
# 1. Get file from Redis and write to worker local disk
file_path = common_storage.ensure_file_from_redis(effective_job_type, job_id, log_prefix)
if not file_path:
return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."}
common_storage.ensure_meta_from_redis(effective_job_type, job_id, file_path, log_prefix)
error_path = common_storage.error_path_for_job(effective_job_type, job_id)
total_rows = 0
error_count = 0
processed_rows = 0
# 3. Count Total (Quick Pass) or just estimate
try:
with open(file_path, 'r', encoding='utf-8-sig') as f:
total_rows = sum(1 for _ in f) - 1 # Minus header
except Exception as e:
return {"status": "failed", "error": f"Cannot read file: {e}"}
footer_config = parse_footer_config(config)
date_format = footer_config.get("dateFormat")
# Validate and set default date_format if not provided
if not date_format:
date_format = "yyyy-mm-dd" # Default to ISO format
logger.info(f"No date_format specified in config, using default: {date_format}")
try:
tenant_id, company_id = common_meta.require_tenant_context(file_path)
except ValueError as e:
return {"status": "failed", "error": str(e)}
meta = common_meta.load_meta(file_path)
template_id = meta.get("template_id") or (
"imp_temp_header" if model_target == "invoice_header" else
"imp_temp_details" if model_target == "invoice_details" else "imp_temp_series"
)
# Cuando el scan viene de Exportación (job_type_override "exp"), forzar exp_def_header o exp_def_partidas
if job_type_override == "exp" and model_target == "invoice_header":
template_id = "exp_def_header"
if job_type_override == "exp" and model_target == "invoice_details":
template_id = "exp_def_partidas"
if job_type_override == "exp" and model_target == "invoice_series":
template_id = "exp_def_series"
inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM")
if not inv_type_value:
inv_type_value = "TEM"
if model_target == "invoice_series" and inv_type_value in ("DEF", "MATDE", "EXDEF"):
template_id = "imp_def_series"
if meta.get("operation_type") == "exp" and model_target == "invoice_details":
template_id = "exp_def_partidas"
logger.info(
"Scan job %s template_id=%s model_target=%s job_type_override=%s",
job_id, template_id, model_target, job_type_override,
)
# --- Series de Importación Definitiva: flujo específico (Clarion VALIDA_TODA_SERIES_IMPO_DEF / VALIDA_PARCIAL) ---
DEF_SERIES_TEMPLATE_OR_TYPE = (
model_target == "invoice_series"
and (
template_id == "imp_def_series"
or (inv_type_value in ("DEF", "MATDE", "EXDEF"))
)
)
if DEF_SERIES_TEMPLATE_OR_TYPE:
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
from .validators.series_impo_def import (
validate_row_series_impo_def,
)
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF")
actualizar = meta.get("actualizar", False)
autonumerar = meta.get("autonumerar", True)
validar_series_exception = meta.get("validar_series", False)
_fc = parse_footer_config(meta.get("footer_config"))
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
elif _fc.get("mode") == "update":
actualizar = True
elif _fc.get("mode") == "replace":
actualizar = False
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
else:
as_val = _fc.get("autonumber_series", "true")
autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes")
if "validar_series" in _fc:
validar_series_exception = bool(_fc["validar_series"])
with CoreSessionLocal() as session:
q = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
rows_inv = q.all()
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in rows_inv:
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
partida_max_series: Dict[Tuple[str, str], int] = {}
q_qty = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
LineQuantity.quantity,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
for num, ln, qty in q_qty.all():
if num is not None and ln is not None:
key = (str(num).strip(), str(ln).strip())
if qty is not None:
partida_max_series[key] = int(qty) if qty else 0
else:
partida_max_series[key] = 0
existing_series_keys: Set[Tuple[str, str, str]] = set()
existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
if actualizar and not autonumerar:
q_ser = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
Serie.row,
Serie.serial_numbers,
Serie.model,
Serie.sub_model,
Serie.number_id,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(Serie, Serie.line_item_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
for num, ln, rw, sn, md, sm, nid in q_ser.all():
if num is not None:
k = (str(num).strip(), str(ln).strip(), str(rw).strip())
existing_series_keys.add(k)
existing_series_data.setdefault(k, {
"serial_numbers": sn or "",
"model": md or "",
"sub_model": sm or "",
"number_id": nid or "",
})
csv_series_count_so_far: Dict[Tuple[str, str], int] = {}
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
errors_detail = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "imp_def_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
err = validate_row_series_impo_def(
row_norm,
i,
actualizar=actualizar,
autonumerar=autonumerar,
validar_series_exception=validar_series_exception,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
partida_max_series=partida_max_series,
csv_series_count_so_far=csv_series_count_so_far,
existing_series_keys=existing_series_keys,
existing_series_data=existing_series_data,
warnings=warnings_list,
)
if err and not err.get("warning"):
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
else:
inv_num = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip()
line_fac = (row_norm.get("LINEA FACTURA") or row_norm.get("LINEA") or row_norm.get("PARTIDA") or "").strip()
if inv_num and line_fac:
key_csv = (inv_num, line_fac)
csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1
for w in warnings_list:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Series importación definitiva scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Series de Exportación Definitiva: flujo exp_def_series (Clarion VALIDA_TODA_SERIES_EXPO / VALIDA_PARCIAL) ---
if model_target == "invoice_series" and template_id == "exp_def_series":
logger.info("Series expo scan: running validation for job %s", job_id)
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
from api.v1.modules.a76.general_catalogs.company.models import Company
from .validators.series_expo import validate_row_series_expo
_fc = parse_footer_config(meta.get("footer_config"))
autonumerar = meta.get("autonumerar", True)
actualizar = meta.get("actualizar", False)
validar_series_exception = meta.get("validar_series", False)
if _fc:
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
elif _fc.get("autonumber_series", "true") is not None:
autonumerar = str(_fc.get("autonumber_series", "true")).lower() in ("true", "1", "si", "sí", "yes")
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "validar_series" in _fc:
validar_series_exception = bool(_fc["validar_series"])
RFC_EXCEPTION_EGM = {"EGM0303257J1"}
with CoreSessionLocal() as session:
company = session.query(Company).filter(Company.id == company_id).first()
company_rfc = (company.rfc or "").strip().upper() if company else ""
q_inv_expo = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in q_inv_expo.all():
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
partida_max_series: Dict[Tuple[str, str], int] = {}
q_qty = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
LineQuantity.quantity,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
for num, ln, qty in q_qty.all():
if num is not None and ln is not None:
key = (str(num).strip(), str(ln).strip())
if qty is not None:
partida_max_series[key] = int(qty) if qty else 0
else:
partida_max_series[key] = 0
existing_series_keys: Set[Tuple[str, str, str]] = set()
existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
if actualizar and not autonumerar:
q_ser = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
Serie.row,
Serie.serial_numbers,
Serie.model,
Serie.sub_model,
Serie.number_id,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(Serie, Serie.line_item_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
for num, ln, rw, sn, md, sm, nid in q_ser.all():
if num is not None:
k = (str(num).strip(), str(ln).strip(), str(rw).strip())
existing_series_keys.add(k)
existing_series_data.setdefault(k, {
"serial_numbers": sn or "",
"model": md or "",
"sub_model": sm or "",
"number_id": nid or "",
})
csv_series_count_so_far: Dict[Tuple[str, str], int] = {}
invoice_numbers_from_csv: Set[str] = set()
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for row in reader:
row_norm = row_from_template(row, "exp_def_series", normalize_header)
inv = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA EXPO") or "").strip()
if inv:
invoice_numbers_from_csv.add(inv)
if company_rfc in RFC_EXCEPTION_EGM:
rfc_exception_updated: Set[str] = invoice_numbers_from_csv
else:
rfc_exception_updated = set()
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err:
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(f_in.read(2048), delimiters=",;\t")
except Exception:
dialect = "excel"
f_in.seek(0)
reader = csv.DictReader(f_in, dialect=dialect)
errors_detail = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "exp_def_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
err = validate_row_series_expo(
row_norm,
i,
actualizar=actualizar,
autonumerar=autonumerar,
validar_series_exception=validar_series_exception,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
rfc_exception_updated=rfc_exception_updated,
partida_max_series=partida_max_series,
csv_series_count_so_far=csv_series_count_so_far,
existing_series_keys=existing_series_keys,
existing_series_data=existing_series_data,
warnings=warnings_list,
)
if err and not err.get("warning"):
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
else:
inv_num = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA EXPO") or "").strip()
line_fac = (row_norm.get("LINEA FACTURA") or row_norm.get("LINEA") or row_norm.get("PARTIDA") or "").strip()
if inv_num and line_fac:
key_csv = (inv_num, line_fac)
csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1
for w in warnings_list:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Series exportación definitiva scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Series Compras Mexicanas: misma lógica que Impo Def, facturas MEX ---
if model_target == "invoice_series" and template_id == "cmex_series":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
from .validators.series_impo_def import validate_row_series_impo_def
actualizar = meta.get("actualizar", False)
autonumerar = meta.get("autonumerar", True)
validar_series_exception = meta.get("validar_series", False)
_fc = parse_footer_config(meta.get("footer_config"))
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
elif _fc.get("mode") == "update":
actualizar = True
elif _fc.get("mode") == "replace":
actualizar = False
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
else:
as_val = _fc.get("autonumber_series", "true")
autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes")
if "validar_series" in _fc:
validar_series_exception = bool(_fc["validar_series"])
with CoreSessionLocal() as session:
q = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
rows_inv = q.all()
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in rows_inv:
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
partida_max_series: Dict[Tuple[str, str], int] = {}
q_qty = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
LineQuantity.quantity,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
for num, ln, qty in q_qty.all():
if num is not None and ln is not None:
key = (str(num).strip(), str(ln).strip())
if qty is not None:
partida_max_series[key] = int(qty) if qty else 0
else:
partida_max_series[key] = 0
existing_series_keys: Set[Tuple[str, str, str]] = set()
existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
if actualizar and not autonumerar:
q_ser = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
Serie.row,
Serie.serial_numbers,
Serie.model,
Serie.sub_model,
Serie.number_id,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(Serie, Serie.line_item_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
for num, ln, rw, sn, md, sm, nid in q_ser.all():
if num is not None:
k = (str(num).strip(), str(ln).strip(), str(rw).strip())
existing_series_keys.add(k)
existing_series_data.setdefault(k, {
"serial_numbers": sn or "",
"model": md or "",
"sub_model": sm or "",
"number_id": nid or "",
})
csv_series_count_so_far: Dict[Tuple[str, str], int] = {}
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
errors_detail = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "cmex_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
err = validate_row_series_impo_def(
row_norm,
i,
actualizar=actualizar,
autonumerar=autonumerar,
validar_series_exception=validar_series_exception,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
partida_max_series=partida_max_series,
csv_series_count_so_far=csv_series_count_so_far,
existing_series_keys=existing_series_keys,
existing_series_data=existing_series_data,
warnings=warnings_list,
catalog_label="Compras Mexicanas",
)
if err and not err.get("warning"):
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
else:
inv_num = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip()
line_fac = (row_norm.get("LINEA FACTURA") or row_norm.get("LINEA") or row_norm.get("PARTIDA") or "").strip()
if inv_num and line_fac:
key_csv = (inv_num, line_fac)
csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1
for w in warnings_list:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Series Compras Mexicanas scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Series de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) ---
if model_target == "invoice_series":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from .validators.series_impo_temp import (
validate_row_series_impo_temp,
)
actualizar = meta.get("actualizar", False)
autonumerar = meta.get("autonumerar", True)
validar_series_exception = meta.get("validar_series", False)
_fc = parse_footer_config(meta.get("footer_config"))
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
elif _fc.get("mode") == "update":
actualizar = True
elif _fc.get("mode") == "replace":
actualizar = False
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
else:
as_val = _fc.get("autonumber_series", "true")
autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes")
if "validar_series" in _fc:
validar_series_exception = bool(_fc["validar_series"])
with CoreSessionLocal() as session:
# Invoice lookup: imp + TEM
q = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
rows_inv = q.all()
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in rows_inv:
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
# Existing series keys: (invoice_number, linea_factura, linea_serie)
existing_series_keys: Set[Tuple[str, str, str]] = set()
existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
if actualizar and not autonumerar:
q_ser = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
Serie.row,
Serie.serial_numbers,
Serie.model,
Serie.sub_model,
Serie.number_id,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(Serie, Serie.line_item_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
for num, ln, rw, sn, md, sm, nid in q_ser.all():
if num is not None:
k = (str(num).strip(), str(ln).strip(), str(rw).strip())
existing_series_keys.add(k)
existing_series_data.setdefault(k, {
"serial_numbers": sn or "",
"model": md or "",
"sub_model": sm or "",
"number_id": nid or "",
})
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
errors_detail: List[Dict[str, Any]] = []
error_lines_list: List[int] = []
for i, row in enumerate(reader, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)})
row_norm = row_from_template(row, "imp_temp_series", normalize_header)
warnings_list: List[Dict[str, Any]] = []
err = validate_row_series_impo_temp(
row_norm,
i,
actualizar=actualizar,
autonumerar=autonumerar,
validar_series_exception=validar_series_exception,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
existing_series_keys=existing_series_keys,
existing_series_data=existing_series_data,
warnings=warnings_list,
)
if err and not err.get("warning"):
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
for w in warnings_list:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Series import scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Partidas de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) ---
if model_target == "invoice_details" and template_id == "imp_temp_details":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.classes.models import Class
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
from api.v1.modules.a76.general_catalogs.packages.models import Package
from api.v1.modules.public.reference_data.countries.models import Country
from api.v1.modules.a76.general_catalogs.sectors.models import Sector
from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
from api.v1.modules.a76.general_catalogs.company.models import Company
from api.v1.modules.a76.parts.models import Part
from .validators.partidas_impo_temp import validate_row_partidas_impo_temp
_fc = parse_footer_config(meta.get("footer_config"))
autonumerar = meta.get("autonumerar", True)
actualizar = meta.get("actualizar", False)
levantar_subpartidas = meta.get("levantar_subpartidas", False)
calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False)
validar_decimales_pza = meta.get("validar_decimales_pza", False)
if _fc:
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
elif _fc.get("autonumber_partidas", "true") is not None:
autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes")
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "levantar_subpartidas" in _fc:
levantar_subpartidas = bool(_fc["levantar_subpartidas"])
if "calcular_costo_unitario_en_base_a_valor_total" in _fc:
calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"])
if "validar_decimales_pza" in _fc:
validar_decimales_pza = bool(_fc["validar_decimales_pza"])
RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"}
RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"}
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in q_inv.all():
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
existing_line_keys_by_invoice: Dict[str, Set[str]] = {}
q_li = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
for num, ln in q_li.all():
if num is not None:
key = str(num).strip()
if key not in existing_line_keys_by_invoice:
existing_line_keys_by_invoice[key] = set()
existing_line_keys_by_invoice[key].add(str(ln).strip())
partidas_principales_bd: Set[Tuple[str, str]] = set()
try:
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem
q_pp = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(FaLineItem, FaLineItem.id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
FaLineItem.is_subitem == False,
FaLineItem.contains_subitems == True,
)
)
for num, ln in q_pp.all():
if num is not None:
partidas_principales_bd.add((str(num).strip(), str(ln).strip()))
except Exception:
pass
valid_class_codes: Set[str] = set()
class_um_by_code: Dict[str, str] = {}
class_fraction_by_code: Dict[str, str] = {}
class_desc_es_by_code: Dict[str, str] = {}
class_desc_en_by_code: Dict[str, str] = {}
for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all():
code = (c.class_code or "").strip().upper()
if code:
valid_class_codes.add(code)
class_um_by_code[code] = (c.unit_of_measure or "").strip().upper()
class_fraction_by_code[code] = (c.fraction or "").strip()
class_desc_es_by_code[code] = (c.description_es or "").strip()
class_desc_en_by_code[code] = (c.description_en or "").strip()
valid_uom_codes: Set[str] = set()
for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all():
if u[0]:
valid_uom_codes.add((u[0] or "").strip().upper())
valid_bulks_codes: Set[str] = set()
for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all():
if p[0]:
valid_bulks_codes.add((p[0] or "").strip())
valid_country_keys: Set[str] = set()
for row in session.query(Country.m3_key, Country.ame_key).all():
if row[0]:
valid_country_keys.add((row[0] or "").strip().upper())
if row[1]:
valid_country_keys.add((row[1] or "").strip().upper())
valid_fraction_ame: Set[str] = set()
for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all():
if row[0]:
valid_fraction_ame.add((row[0] or "").strip())
authorized_sectors: Set[str] = set()
for row in session.query(Sector.key).filter(
Sector.authorized == True,
Sector.tenant_id == tenant_id,
Sector.company_id == company_id,
).all():
if row[0]:
authorized_sectors.add((row[0] or "").strip().upper())
valid_payment_methods: Set[str] = set()
for row in session.query(PaymentMethod.key).all():
if row[0] is not None:
valid_payment_methods.add(str(row[0]).strip())
valid_valuation_methods: Set[str] = set()
for row in session.query(ValuationMethod.key).all():
if row[0]:
valid_valuation_methods.add((row[0] or "").strip())
company = session.query(Company).filter(Company.id == company_id).first()
company_has_prosec = bool(company.prosec) if company else False
company_rfc = (company.rfc or "").strip().upper() if company else ""
valid_part_numbers: Set[str] = set()
for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all():
if row[0]:
valid_part_numbers.add((row[0] or "").strip().upper())
rfc_exception_updated: Set[str] = set()
rfc_exception_num_parte: Set[str] = set()
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
invoice_numbers_from_csv = set()
for row in rows_list:
inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip()
if inv:
invoice_numbers_from_csv.add(inv)
if company_rfc in RFC_EXCEPTION_UPDATED:
rfc_exception_updated = invoice_numbers_from_csv
if company_rfc in RFC_EXCEPTION_NUM_PARTE:
rfc_exception_num_parte = invoice_numbers_from_csv
line_counts_csv: Dict[Tuple[str, str], int] = {}
partidas_principales_csv: Set[Tuple[str, str]] = set()
def _get_row(row_norm: Dict[str, Any], *keys: str) -> str:
for k in keys:
v = row_norm.get(k)
if v is not None and str(v).strip():
return str(v).strip()
return ""
for row in rows_list:
row_norm = row_from_template(row, "imp_temp_details", normalize_header)
inv = _get_row(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA")
linea = _get_row(row_norm, "LINEA", "RENGLON", "PARTIDA")
if inv and linea:
key = (inv, linea)
line_counts_csv[key] = line_counts_csv.get(key, 0) + 1
u = _get_row(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper()
if u == "P" and inv and linea:
partidas_principales_csv.add((inv, linea))
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_temp_details", normalize_header)
err = validate_row_partidas_impo_temp(
row_norm,
i,
autonumerar=autonumerar,
actualizar=actualizar,
levantar_subpartidas=levantar_subpartidas,
calcular_costo_en_base_a_total=calcular_costo_en_base_a_total,
validar_decimales_pza=validar_decimales_pza,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
rfc_exception_updated=rfc_exception_updated,
existing_line_keys_by_invoice=existing_line_keys_by_invoice,
line_counts_csv=line_counts_csv,
partidas_principales_csv=partidas_principales_csv,
partidas_principales_bd=partidas_principales_bd,
valid_class_codes=valid_class_codes,
class_um_by_code=class_um_by_code,
class_fraction_by_code=class_fraction_by_code,
class_desc_es_by_code=class_desc_es_by_code,
class_desc_en_by_code=class_desc_en_by_code,
valid_uom_codes=valid_uom_codes,
valid_bulks_codes=valid_bulks_codes,
valid_country_keys=valid_country_keys,
valid_fraction_ame=valid_fraction_ame,
valid_payment_methods=valid_payment_methods,
valid_valuation_methods=valid_valuation_methods,
authorized_sectors=authorized_sectors,
company_has_prosec=company_has_prosec,
rfc_exception_num_parte=rfc_exception_num_parte or None,
valid_part_numbers=valid_part_numbers,
warnings=None,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
except Exception as e:
logger.exception("Partidas import scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Partidas de Importación Definitiva: flujo específico (Clarion VALIDA_TODA_PARIMPO_DEF / VALIDA_PARCIAL) ---
# Estructura de columnas: misma que partidas TEM (imp_def_details resuelve a imp_temp_details). Facturas DEF/MATDE/EXDEF.
if model_target == "invoice_details" and template_id == "imp_def_details":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.classes.models import Class
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
from api.v1.modules.a76.general_catalogs.packages.models import Package
from api.v1.modules.public.reference_data.countries.models import Country
from api.v1.modules.a76.general_catalogs.sectors.models import Sector
from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
from api.v1.modules.a76.general_catalogs.company.models import Company
from api.v1.modules.a76.parts.models import Part
from .validators.partidas_impo_def import validate_row_partidas_impo_def
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF")
_fc = parse_footer_config(meta.get("footer_config"))
autonumerar = meta.get("autonumerar", True)
actualizar = meta.get("actualizar", False)
levantar_subpartidas = meta.get("levantar_subpartidas", False)
calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False)
validar_decimales_pza = meta.get("validar_decimales_pza", False)
if _fc:
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
elif _fc.get("autonumber_partidas", "true") is not None:
autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes")
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "levantar_subpartidas" in _fc:
levantar_subpartidas = bool(_fc["levantar_subpartidas"])
if "calcular_costo_unitario_en_base_a_valor_total" in _fc:
calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"])
if "validar_decimales_pza" in _fc:
validar_decimales_pza = bool(_fc["validar_decimales_pza"])
RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"}
RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"}
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in q_inv.all():
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
existing_line_keys_by_invoice: Dict[str, Set[str]] = {}
q_li = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
for num, ln in q_li.all():
if num is not None:
key = str(num).strip()
if key not in existing_line_keys_by_invoice:
existing_line_keys_by_invoice[key] = set()
existing_line_keys_by_invoice[key].add(str(ln).strip())
partidas_principales_bd: Set[Tuple[str, str]] = set()
try:
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem
q_pp = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(FaLineItem, FaLineItem.id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
FaLineItem.is_subitem == False,
FaLineItem.contains_subitems == True,
)
)
for num, ln in q_pp.all():
if num is not None:
partidas_principales_bd.add((str(num).strip(), str(ln).strip()))
except Exception:
pass
valid_class_codes: Set[str] = set()
class_um_by_code: Dict[str, str] = {}
class_fraction_by_code: Dict[str, str] = {}
class_desc_es_by_code: Dict[str, str] = {}
class_desc_en_by_code: Dict[str, str] = {}
for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all():
code = (c.class_code or "").strip().upper()
if code:
valid_class_codes.add(code)
class_um_by_code[code] = (c.unit_of_measure or "").strip().upper()
class_fraction_by_code[code] = (c.fraction or "").strip()
class_desc_es_by_code[code] = (c.description_es or "").strip()
class_desc_en_by_code[code] = (c.description_en or "").strip()
valid_uom_codes: Set[str] = set()
for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all():
if u[0]:
valid_uom_codes.add((u[0] or "").strip().upper())
valid_bulks_codes: Set[str] = set()
for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all():
if p[0]:
valid_bulks_codes.add((p[0] or "").strip())
valid_country_keys: Set[str] = set()
for row in session.query(Country.m3_key, Country.ame_key).all():
if row[0]:
valid_country_keys.add((row[0] or "").strip().upper())
if row[1]:
valid_country_keys.add((row[1] or "").strip().upper())
valid_fraction_ame: Set[str] = set()
for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all():
if row[0]:
valid_fraction_ame.add((row[0] or "").strip())
authorized_sectors: Set[str] = set()
for row in session.query(Sector.key).filter(
Sector.authorized == True,
Sector.tenant_id == tenant_id,
Sector.company_id == company_id,
).all():
if row[0]:
authorized_sectors.add((row[0] or "").strip().upper())
valid_payment_methods: Set[str] = set()
for row in session.query(PaymentMethod.key).all():
if row[0] is not None:
valid_payment_methods.add(str(row[0]).strip())
valid_valuation_methods: Set[str] = set()
for row in session.query(ValuationMethod.key).all():
if row[0]:
valid_valuation_methods.add((row[0] or "").strip())
company = session.query(Company).filter(Company.id == company_id).first()
company_has_prosec = bool(company.prosec) if company else False
company_rfc = (company.rfc or "").strip().upper() if company else ""
valid_part_numbers: Set[str] = set()
for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all():
if row[0]:
valid_part_numbers.add((row[0] or "").strip().upper())
rfc_exception_updated: Set[str] = set()
rfc_exception_num_parte: Set[str] = set()
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
invoice_numbers_from_csv = set()
for row in rows_list:
inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip()
if inv:
invoice_numbers_from_csv.add(inv)
if company_rfc in RFC_EXCEPTION_UPDATED:
rfc_exception_updated = invoice_numbers_from_csv
if company_rfc in RFC_EXCEPTION_NUM_PARTE:
rfc_exception_num_parte = invoice_numbers_from_csv
line_counts_csv: Dict[Tuple[str, str], int] = {}
partidas_principales_csv: Set[Tuple[str, str]] = set()
def _get_row_def(row_norm: Dict[str, Any], *keys: str) -> str:
for k in keys:
v = row_norm.get(k)
if v is not None and str(v).strip():
return str(v).strip()
return ""
for row in rows_list:
row_norm = row_from_template(row, "imp_def_details", normalize_header)
inv = _get_row_def(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA")
linea = _get_row_def(row_norm, "LINEA", "RENGLON", "PARTIDA")
if inv and linea:
key = (inv, linea)
line_counts_csv[key] = line_counts_csv.get(key, 0) + 1
u = _get_row_def(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper()
if u == "P" and inv and linea:
partidas_principales_csv.add((inv, linea))
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_def_details", normalize_header)
err = validate_row_partidas_impo_def(
row_norm,
i,
autonumerar=autonumerar,
actualizar=actualizar,
levantar_subpartidas=levantar_subpartidas,
calcular_costo_en_base_a_total=calcular_costo_en_base_a_total,
validar_decimales_pza=validar_decimales_pza,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
rfc_exception_updated=rfc_exception_updated,
existing_line_keys_by_invoice=existing_line_keys_by_invoice,
line_counts_csv=line_counts_csv,
partidas_principales_csv=partidas_principales_csv,
partidas_principales_bd=partidas_principales_bd,
valid_class_codes=valid_class_codes,
class_um_by_code=class_um_by_code,
class_fraction_by_code=class_fraction_by_code,
class_desc_es_by_code=class_desc_es_by_code,
class_desc_en_by_code=class_desc_en_by_code,
valid_uom_codes=valid_uom_codes,
valid_bulks_codes=valid_bulks_codes,
valid_country_keys=valid_country_keys,
valid_fraction_ame=valid_fraction_ame,
valid_payment_methods=valid_payment_methods,
valid_valuation_methods=valid_valuation_methods,
authorized_sectors=authorized_sectors,
company_has_prosec=company_has_prosec,
rfc_exception_num_parte=rfc_exception_num_parte or None,
valid_part_numbers=valid_part_numbers,
warnings=None,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
except Exception as e:
logger.exception("Partidas importación definitiva scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Partidas Exportación Definitiva: Clarion VALIDA_TODA_PAR_EXPO / VALIDA_PARCIAL_PAR_EXPO / VALIDACIONES_PAR_EXPO ---
if model_target == "invoice_details" and template_id == "exp_def_partidas":
logger.info("Partidas expo scan: running validation for job %s", job_id)
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
from api.v1.modules.a76.general_catalogs.packages.models import Package
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
from api.v1.modules.a76.general_catalogs.company.models import Company
from api.v1.modules.a76.parts.models import Part
from .validators.partidas_expo import validate_row_partidas_expo
_fc = parse_footer_config(meta.get("footer_config"))
autonumerar = meta.get("autonumerar", True)
actualizar = meta.get("actualizar", False)
levantar_subpartidas = meta.get("levantar_subpartidas", False)
validar_decimales_pza = meta.get("validar_decimales_pza", False)
if _fc:
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
elif _fc.get("autonumber_partidas", "true") is not None:
autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes")
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "levantar_subpartidas" in _fc:
levantar_subpartidas = bool(_fc["levantar_subpartidas"])
if "validar_decimales_pza" in _fc:
validar_decimales_pza = bool(_fc["validar_decimales_pza"])
RFC_EXCEPTION_UPDATED = set()
RFC_EXCEPTION_EGM = {"EGM0303257J1"}
with CoreSessionLocal() as session:
company = session.query(Company).filter(Company.id == company_id).first()
company_rfc = (company.rfc or "").strip().upper() if company else ""
if company_rfc in RFC_EXCEPTION_EGM:
rfc_exception_egm = True
else:
rfc_exception_egm = False
q_inv_expo = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in q_inv_expo.all():
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
existing_line_keys_by_invoice: Dict[str, Set[str]] = {}
q_li_expo = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
for num, ln in q_li_expo.all():
if num is not None:
key = str(num).strip()
if key not in existing_line_keys_by_invoice:
existing_line_keys_by_invoice[key] = set()
existing_line_keys_by_invoice[key].add(str(ln).strip())
partidas_principales_bd: Set[Tuple[str, str]] = set()
try:
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem
q_pp = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(FaLineItem, FaLineItem.id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
FaLineItem.is_subitem == False,
FaLineItem.contains_subitems == True,
)
)
for num, ln in q_pp.all():
if num is not None:
partidas_principales_bd.add((str(num).strip(), str(ln).strip()))
except Exception:
pass
factura_impo_tem_by_number: Dict[str, int] = {}
q_tem = session.query(InvoiceHeader.invoice_number, InvoiceHeader.id).filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
for num, iid in q_tem.all():
if num:
factura_impo_tem_by_number[str(num).strip()] = iid
factura_impo_def_by_number: Dict[str, int] = {}
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF")
q_def = session.query(InvoiceHeader.invoice_number, InvoiceHeader.id).filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
for num, iid in q_def.all():
if num:
factura_impo_def_by_number[str(num).strip()] = iid
line_exists_tem: Set[Tuple[int, str]] = set()
q_li_tem = (
session.query(LineItem.invoice_id, LineItem.line_number)
.join(InvoiceHeader, InvoiceHeader.id == LineItem.invoice_id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
for inv_id, ln in q_li_tem.all():
if inv_id is not None and ln is not None:
line_exists_tem.add((inv_id, str(ln).strip()))
line_exists_def: Set[Tuple[int, str]] = set()
q_li_def = (
session.query(LineItem.invoice_id, LineItem.line_number)
.join(InvoiceHeader, InvoiceHeader.id == LineItem.invoice_id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
for inv_id, ln in q_li_def.all():
if inv_id is not None and ln is not None:
line_exists_def.add((inv_id, str(ln).strip()))
valid_uom_codes: Set[str] = set()
for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all():
if u[0]:
valid_uom_codes.add((u[0] or "").strip().upper())
valid_bulks_codes: Set[str] = set()
for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all():
if p[0]:
valid_bulks_codes.add((p[0] or "").strip())
valid_payment_methods: Set[str] = set()
for row in session.query(PaymentMethod.key).all():
if row[0] is not None:
valid_payment_methods.add(str(row[0]).strip())
valid_fraction_ame: Set[str] = set()
for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all():
if row[0]:
valid_fraction_ame.add((row[0] or "").strip())
valid_part_numbers: Set[str] = set()
for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all():
if row[0]:
valid_part_numbers.add((row[0] or "").strip().upper())
invoice_numbers_from_csv = set()
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
for row in rows_list:
row_norm = row_from_template(row, "exp_def_partidas", normalize_header)
inv = (row_norm.get("NUMERO FACTURA EXPO") or row_norm.get("NUMERO FACTURA EXPO.") or row_norm.get("FACTURA EXPO") or "").strip()
if inv:
invoice_numbers_from_csv.add(inv)
if company_rfc in RFC_EXCEPTION_EGM:
rfc_exception_updated = invoice_numbers_from_csv
else:
rfc_exception_updated = set()
line_counts_csv: Dict[Tuple[str, str], int] = {}
partidas_principales_csv: Set[Tuple[str, str]] = set()
def _get_row_expo(row_norm: Dict[str, Any], *keys: str) -> str:
for k in keys:
v = row_norm.get(k)
if v is not None and str(v).strip():
return str(v).strip()
return ""
for row in rows_list:
row_norm = row_from_template(row, "exp_def_partidas", normalize_header)
inv = _get_row_expo(row_norm, "NUMERO FACTURA EXPO", "NUMERO FACTURA EXPO.", "FACTURA EXPO")
linea = _get_row_expo(row_norm, "LINEA EXPO", "LINEA EXPO.", "RENGLON EXPO")
if inv and linea:
key = (inv, linea)
line_counts_csv[key] = line_counts_csv.get(key, 0) + 1
u = _get_row_expo(row_norm, "ES PARTIDA/SUBPARTIDA", "ESSUBPARTIDA", "ES PARTIDA O SUBPARTIDA").upper()
if u == "P" and inv and linea:
partidas_principales_csv.add((inv, linea))
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
row_norm = row_from_template(row, "exp_def_partidas", normalize_header)
err = validate_row_partidas_expo(
row_norm,
i,
autonumerar=autonumerar,
actualizar=actualizar,
levantar_subpartidas=levantar_subpartidas,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
rfc_exception_updated=rfc_exception_updated,
existing_line_keys_by_invoice=existing_line_keys_by_invoice,
line_counts_csv=line_counts_csv,
partidas_principales_csv=partidas_principales_csv,
partidas_principales_bd=partidas_principales_bd,
valid_uom_codes=valid_uom_codes,
valid_bulks_codes=valid_bulks_codes,
valid_payment_methods=valid_payment_methods,
valid_fraction_ame=valid_fraction_ame,
valid_part_numbers=valid_part_numbers,
factura_impo_tem_by_number=factura_impo_tem_by_number,
factura_impo_def_by_number=factura_impo_def_by_number,
line_exists_tem=line_exists_tem,
line_exists_def=line_exists_def,
rfc_exception_egm=rfc_exception_egm,
validar_decimales_pza=validar_decimales_pza,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
logger.info(
"Partidas expo scan rechazo línea %s (col %s): %s",
err["line"],
err.get("col", ""),
err.get("msg", ""),
)
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Partidas exportación definitiva scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Partidas Compras Mexicanas: misma lógica que Impo Def, facturas MEX ---
if model_target == "invoice_details" and template_id == "cmex_details":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.classes.models import Class
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
from api.v1.modules.a76.general_catalogs.packages.models import Package
from api.v1.modules.public.reference_data.countries.models import Country
from api.v1.modules.a76.general_catalogs.sectors.models import Sector
from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod
from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod
from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction
from api.v1.modules.a76.general_catalogs.company.models import Company
from api.v1.modules.a76.parts.models import Part
from .validators.partidas_impo_def import validate_row_partidas_impo_def
_fc = parse_footer_config(meta.get("footer_config"))
autonumerar = meta.get("autonumerar", True)
actualizar = meta.get("actualizar", False)
levantar_subpartidas = meta.get("levantar_subpartidas", False)
calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False)
validar_decimales_pza = meta.get("validar_decimales_pza", False)
if _fc:
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
elif _fc.get("autonumber_partidas", "true") is not None:
autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes")
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "levantar_subpartidas" in _fc:
levantar_subpartidas = bool(_fc["levantar_subpartidas"])
if "calcular_costo_unitario_en_base_a_valor_total" in _fc:
calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"])
if "validar_decimales_pza" in _fc:
validar_decimales_pza = bool(_fc["validar_decimales_pza"])
RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"}
RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"}
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in q_inv.all():
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
existing_line_keys_by_invoice: Dict[str, Set[str]] = {}
q_li = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
for num, ln in q_li.all():
if num is not None:
key = str(num).strip()
if key not in existing_line_keys_by_invoice:
existing_line_keys_by_invoice[key] = set()
existing_line_keys_by_invoice[key].add(str(ln).strip())
partidas_principales_bd: Set[Tuple[str, str]] = set()
try:
from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem
q_pp = (
session.query(InvoiceHeader.invoice_number, LineItem.line_number)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(FaLineItem, FaLineItem.id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
FaLineItem.is_subitem == False,
FaLineItem.contains_subitems == True,
)
)
for num, ln in q_pp.all():
if num is not None:
partidas_principales_bd.add((str(num).strip(), str(ln).strip()))
except Exception:
pass
valid_class_codes: Set[str] = set()
class_um_by_code: Dict[str, str] = {}
class_fraction_by_code: Dict[str, str] = {}
class_desc_es_by_code: Dict[str, str] = {}
class_desc_en_by_code: Dict[str, str] = {}
for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all():
code = (c.class_code or "").strip().upper()
if code:
valid_class_codes.add(code)
class_um_by_code[code] = (c.unit_of_measure or "").strip().upper()
class_fraction_by_code[code] = (c.fraction or "").strip()
class_desc_es_by_code[code] = (c.description_es or "").strip()
class_desc_en_by_code[code] = (c.description_en or "").strip()
valid_uom_codes: Set[str] = set()
for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all():
if u[0]:
valid_uom_codes.add((u[0] or "").strip().upper())
valid_bulks_codes: Set[str] = set()
for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all():
if p[0]:
valid_bulks_codes.add((p[0] or "").strip())
valid_country_keys: Set[str] = set()
for row in session.query(Country.m3_key, Country.ame_key).all():
if row[0]:
valid_country_keys.add((row[0] or "").strip().upper())
if row[1]:
valid_country_keys.add((row[1] or "").strip().upper())
valid_fraction_ame: Set[str] = set()
for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all():
if row[0]:
valid_fraction_ame.add((row[0] or "").strip())
authorized_sectors: Set[str] = set()
for row in session.query(Sector.key).filter(
Sector.authorized == True,
Sector.tenant_id == tenant_id,
Sector.company_id == company_id,
).all():
if row[0]:
authorized_sectors.add((row[0] or "").strip().upper())
valid_payment_methods: Set[str] = set()
for row in session.query(PaymentMethod.key).all():
if row[0] is not None:
valid_payment_methods.add(str(row[0]).strip())
valid_valuation_methods: Set[str] = set()
for row in session.query(ValuationMethod.key).all():
if row[0]:
valid_valuation_methods.add((row[0] or "").strip())
company = session.query(Company).filter(Company.id == company_id).first()
company_has_prosec = bool(company.prosec) if company else False
company_rfc = (company.rfc or "").strip().upper() if company else ""
valid_part_numbers: Set[str] = set()
for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all():
if row[0]:
valid_part_numbers.add((row[0] or "").strip().upper())
rfc_exception_updated: Set[str] = set()
rfc_exception_num_parte: Set[str] = set()
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
invoice_numbers_from_csv = set()
for row in rows_list:
inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip()
if inv:
invoice_numbers_from_csv.add(inv)
if company_rfc in RFC_EXCEPTION_UPDATED:
rfc_exception_updated = invoice_numbers_from_csv
if company_rfc in RFC_EXCEPTION_NUM_PARTE:
rfc_exception_num_parte = invoice_numbers_from_csv
line_counts_csv: Dict[Tuple[str, str], int] = {}
partidas_principales_csv: Set[Tuple[str, str]] = set()
def _get_row_cmex(row_norm: Dict[str, Any], *keys: str) -> str:
for k in keys:
v = row_norm.get(k)
if v is not None and str(v).strip():
return str(v).strip()
return ""
for row in rows_list:
row_norm = row_from_template(row, "cmex_details", normalize_header)
inv = _get_row_cmex(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA")
linea = _get_row_cmex(row_norm, "LINEA", "RENGLON", "PARTIDA")
if inv and linea:
key = (inv, linea)
line_counts_csv[key] = line_counts_csv.get(key, 0) + 1
u = _get_row_cmex(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper()
if u == "P" and inv and linea:
partidas_principales_csv.add((inv, linea))
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "cmex_details", normalize_header)
err = validate_row_partidas_impo_def(
row_norm,
i,
autonumerar=autonumerar,
actualizar=actualizar,
levantar_subpartidas=levantar_subpartidas,
calcular_costo_en_base_a_total=calcular_costo_en_base_a_total,
validar_decimales_pza=validar_decimales_pza,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
rfc_exception_updated=rfc_exception_updated,
existing_line_keys_by_invoice=existing_line_keys_by_invoice,
line_counts_csv=line_counts_csv,
partidas_principales_csv=partidas_principales_csv,
partidas_principales_bd=partidas_principales_bd,
valid_class_codes=valid_class_codes,
class_um_by_code=class_um_by_code,
class_fraction_by_code=class_fraction_by_code,
class_desc_es_by_code=class_desc_es_by_code,
class_desc_en_by_code=class_desc_en_by_code,
valid_uom_codes=valid_uom_codes,
valid_bulks_codes=valid_bulks_codes,
valid_country_keys=valid_country_keys,
valid_fraction_ame=valid_fraction_ame,
valid_payment_methods=valid_payment_methods,
valid_valuation_methods=valid_valuation_methods,
authorized_sectors=authorized_sectors,
company_has_prosec=company_has_prosec,
rfc_exception_num_parte=rfc_exception_num_parte or None,
valid_part_numbers=valid_part_numbers,
warnings=None,
catalog_label="Compras Mexicanas",
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
except Exception as e:
logger.exception("Partidas Compras Mexicanas scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Encabezados de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) ---
if model_target == "invoice_header" and template_id == "imp_temp_header":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
from api.v1.modules.a76.transportation.transporters.models import Transporter
from .validators.encabezados_impo_temp import (
validate_row_encabezados_impo_temp,
parse_pedimento_col_a,
_pedimento_key_from_parsed,
)
def _ped_key_from_row(ped_str: str) -> Optional[str]:
parsed = parse_pedimento_col_a(ped_str)
if not parsed:
return None
return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2])
_fc = parse_footer_config(meta.get("footer_config"))
actualizar = meta.get("actualizar", False)
autonumerar_remesas = meta.get("autonumerar_remesas", False)
control_remesa = bool(_fc.get("control_remesa", False))
remesa_inicio = _fc.get("remesa_inicio")
remesa_fin = _fc.get("remesa_fin")
if remesa_inicio is not None:
try:
remesa_inicio = int(remesa_inicio)
except (TypeError, ValueError):
remesa_inicio = None
if remesa_fin is not None:
try:
remesa_fin = int(remesa_fin)
except (TypeError, ValueError):
remesa_fin = None
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "autonumerar_remesas" in _fc:
autonumerar_remesas = bool(_fc["autonumerar_remesas"])
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
invoice_exists_by_number: Dict[str, bool] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in q_inv.all():
if num:
n = str(num).strip()
invoice_exists_by_number[n] = True
invoice_processed_by_number[n] = bool(is_upd)
pedimento_data_by_key: Dict[str, List[Dict[str, Any]]] = {}
for p in (
session.query(
Pedimentos.id,
Pedimentos.year,
Pedimentos.customs_office,
Pedimentos.license,
Pedimentos.pedimento_number,
Pedimentos.operation_type,
Pedimentos.regime,
Pedimentos.pedimento_type,
)
.filter(
Pedimentos.tenant_id == tenant_id,
Pedimentos.company_id == company_id,
)
.all()
):
co = (p.customs_office or "").strip()
lic = (p.license or "").strip()
num = (p.pedimento_number or "").strip()
if not co or not lic or not num:
continue
key = _pedimento_key_from_parsed(co, lic, num)
entry_date = None
end_date = None
pd = (
session.query(PedimentoDates.entry_date, PedimentoDates.end_date)
.filter(PedimentoDates.pedimento_id == p.id).first()
)
if pd:
entry_date = pd[0]
end_date = pd[1]
info = {
"id": p.id,
"regime": (p.regime or "").strip(),
"operation_type": (p.operation_type or "").strip(),
"pedimento_type": (p.pedimento_type or "").strip(),
"entry_date": entry_date,
"end_date": end_date,
}
if key not in pedimento_data_by_key:
pedimento_data_by_key[key] = []
pedimento_data_by_key[key].append(info)
remesa_por_pedimento_bd: Dict[str, Set[int]] = {}
q_rem = (
session.query(
InvoiceComplianceMx.remesa,
Pedimentos.year,
Pedimentos.customs_office,
Pedimentos.license,
Pedimentos.pedimento_number,
)
.join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id)
.filter(
InvoiceComplianceMx.tenant_id == tenant_id,
InvoiceComplianceMx.company_id == company_id,
InvoiceComplianceMx.pedimento_id.isnot(None),
InvoiceComplianceMx.remesa.isnot(None),
)
)
for rem, y, co, lic, num in q_rem.all():
if co and lic and num and rem is not None:
key = _pedimento_key_from_parsed(
(co or "").strip(),
(lic or "").strip(),
(num or "").strip(),
)
if key not in remesa_por_pedimento_bd:
remesa_por_pedimento_bd[key] = set()
remesa_por_pedimento_bd[key].add(int(rem))
valid_provider_ids: Set[int] = set()
valid_sold_to_ids: Set[int] = set()
valid_shipped_to_ids: Set[int] = set()
valid_provider_short_names: Set[str] = set()
valid_sold_to_short_names: Set[str] = set()
valid_shipped_to_short_names: Set[str] = set()
for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
).all():
valid_provider_ids.add(cp[0])
valid_sold_to_ids.add(cp[0])
valid_shipped_to_ids.add(cp[0])
if cp[1] and str(cp[1]).strip():
sn_upper = str(cp[1]).strip().upper()
valid_provider_short_names.add(sn_upper)
valid_sold_to_short_names.add(sn_upper)
valid_shipped_to_short_names.add(sn_upper)
valid_broker_ids: Set[int] = set()
valid_broker_claves: Set[str] = set()
for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter(
CustomsBroker.tenant_id == tenant_id,
CustomsBroker.company_id == company_id,
).all():
valid_broker_ids.add(cb[0])
if cb[1] and str(cb[1]).strip():
valid_broker_claves.add(str(cb[1]).strip())
valid_transporter_keys: Set[str] = set()
for t in session.query(Transporter.transporter_key).filter(
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
).all():
if t[0]:
valid_transporter_keys.add((t[0] or "").strip().upper())
valid_incoterms: Set[str] = set()
for inc in session.query(Incoterm.code).all():
if inc[0]:
valid_incoterms.add((inc[0] or "").strip().upper())
valid_aduana_codes: Set[str] = set()
for cs in session.query(CustomsSection.customs_code).all():
if cs[0]:
valid_aduana_codes.add((cs[0] or "").strip())
valid_currency_codes: Set[str] = set()
for ct in session.query(CurrencyType.code).all():
if ct[0]:
valid_currency_codes.add((ct[0] or "").strip().upper())
exchange_rate_by_date: Dict[str, Any] = {}
for er in session.query(ExchangeRate.date, ExchangeRate.value).filter(
ExchangeRate.tenant_id == tenant_id,
ExchangeRate.company_id == company_id,
).all():
if er[0] and er[1] is not None:
dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10]
exchange_rate_by_date[dk] = er[1]
invoice_has_partidas_by_number: Dict[str, bool] = {}
existing_tipo_moneda_by_number: Dict[str, str] = {}
q_li_count = (
session.query(InvoiceHeader.invoice_number, func.count(LineItem.id))
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
.group_by(InvoiceHeader.invoice_number)
)
for num, cnt in q_li_count.all():
if num:
invoice_has_partidas_by_number[str(num).strip()] = cnt > 0
q_fin = (
session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency)
.join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
for num, cur in q_fin.all():
if num and cur:
cur_str = (cur or "").strip().lower()
if cur_str == "foreign":
existing_tipo_moneda_by_number[str(num).strip()] = "ME"
elif cur_str == "local":
existing_tipo_moneda_by_number[str(num).strip()] = "MN"
else:
existing_tipo_moneda_by_number[str(num).strip()] = cur_str.upper()[:2]
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
remesa_por_pedimento_csv: Dict[str, Dict[int, str]] = {}
for row in rows_list:
row_norm = row_from_template(row, "imp_temp_header", normalize_header)
ped = (row_norm.get("PEDIMENTO") or "").strip()
rem = row_norm.get("REMESA")
factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip()
if not ped or not factura:
continue
key = _ped_key_from_row(ped)
if not key:
continue
try:
rem_int = int(rem) if rem is not None and str(rem).strip() else None
except (TypeError, ValueError):
rem_int = None
if rem_int is not None:
if key not in remesa_por_pedimento_csv:
remesa_por_pedimento_csv[key] = {}
if rem_int not in remesa_por_pedimento_csv[key]:
remesa_por_pedimento_csv[key][rem_int] = factura
error_count = 0
processed_rows = 0
error_lines_list: List[int] = []
errors_detail: List[Dict[str, Any]] = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_temp_header", normalize_header)
warnings_row: List[Dict[str, Any]] = []
err = validate_row_encabezados_impo_temp(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
pedimento_data_by_key=pedimento_data_by_key,
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
valid_broker_ids=valid_broker_ids,
valid_broker_claves=valid_broker_claves,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_aduana_codes=valid_aduana_codes,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
autonumerar_remesas=autonumerar_remesas,
control_remesa=control_remesa,
remesa_inicio=remesa_inicio,
remesa_fin=remesa_fin,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
for w in warnings_row:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Encabezados importación temporal scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Encabezados de Importación Definitiva: flujo específico (Clarion VALIDA_TODA_FACIMPO_DEF / VALIDA_PARCIAL) ---
if model_target == "invoice_header" and template_id == "imp_def_header":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
from api.v1.modules.a76.transportation.transporters.models import Transporter
from .validators.encabezados_impo_def import (
validate_row_encabezados_impo_def,
parse_pedimento_col_a_impo_def,
)
from .validators.encabezados_impo_temp import _pedimento_key_from_parsed
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF")
def _ped_key_from_row_def(ped_str: str) -> Optional[str]:
parsed = parse_pedimento_col_a_impo_def(ped_str)
if not parsed:
return None
return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2])
_fc = parse_footer_config(meta.get("footer_config"))
actualizar = meta.get("actualizar", False)
autonumerar_remesas = meta.get("autonumerar_remesas", False)
control_remesa = bool(_fc.get("control_remesa", False))
remesa_inicio = _fc.get("remesa_inicio")
remesa_fin = _fc.get("remesa_fin")
if remesa_inicio is not None:
try:
remesa_inicio = int(remesa_inicio)
except (TypeError, ValueError):
remesa_inicio = None
if remesa_fin is not None:
try:
remesa_fin = int(remesa_fin)
except (TypeError, ValueError):
remesa_fin = None
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "autonumerar_remesas" in _fc:
autonumerar_remesas = bool(_fc["autonumerar_remesas"])
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
invoice_exists_by_number = {}
invoice_processed_by_number = {}
for num, iid, is_upd in q_inv.all():
if num:
n = str(num).strip()
invoice_exists_by_number[n] = True
invoice_processed_by_number[n] = bool(is_upd)
pedimento_data_by_key = {}
for p in (
session.query(
Pedimentos.id,
Pedimentos.year,
Pedimentos.customs_office,
Pedimentos.license,
Pedimentos.pedimento_number,
Pedimentos.operation_type,
Pedimentos.regime,
Pedimentos.pedimento_type,
)
.filter(
Pedimentos.tenant_id == tenant_id,
Pedimentos.company_id == company_id,
Pedimentos.operation_type == "imp",
Pedimentos.regime == "IMD",
)
.all()
):
co = (p.customs_office or "").strip()
lic = (p.license or "").strip()
num = (p.pedimento_number or "").strip()
if not co or not lic or not num:
continue
key = _pedimento_key_from_parsed(co, lic, num)
entry_date = None
end_date = None
pd = (
session.query(PedimentoDates.entry_date, PedimentoDates.end_date)
.filter(PedimentoDates.pedimento_id == p.id).first()
)
if pd:
entry_date = pd[0]
end_date = pd[1]
info = {
"id": p.id,
"regime": (p.regime or "").strip(),
"operation_type": (p.operation_type or "").strip(),
"pedimento_type": (p.pedimento_type or "").strip(),
"entry_date": entry_date,
"end_date": end_date,
}
if key not in pedimento_data_by_key:
pedimento_data_by_key[key] = []
pedimento_data_by_key[key].append(info)
remesa_por_pedimento_bd = {}
q_rem = (
session.query(
InvoiceComplianceMx.remesa,
Pedimentos.customs_office,
Pedimentos.license,
Pedimentos.pedimento_number,
)
.join(InvoiceHeader, InvoiceHeader.id == InvoiceComplianceMx.invoice_id)
.join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id)
.filter(
InvoiceComplianceMx.tenant_id == tenant_id,
InvoiceComplianceMx.company_id == company_id,
InvoiceComplianceMx.pedimento_id.isnot(None),
InvoiceComplianceMx.remesa.isnot(None),
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
for rem, co, lic, num in q_rem.all():
if co and lic and num and rem is not None:
key = _pedimento_key_from_parsed(
(co or "").strip(),
(lic or "").strip(),
(num or "").strip(),
)
if key not in remesa_por_pedimento_bd:
remesa_por_pedimento_bd[key] = set()
remesa_por_pedimento_bd[key].add(int(rem))
valid_provider_ids = set()
valid_sold_to_ids = set()
valid_shipped_to_ids = set()
valid_provider_short_names = set()
valid_sold_to_short_names = set()
valid_shipped_to_short_names = set()
for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
).all():
valid_provider_ids.add(cp[0])
valid_sold_to_ids.add(cp[0])
valid_shipped_to_ids.add(cp[0])
if cp[1] and str(cp[1]).strip():
sn_upper = str(cp[1]).strip().upper()
valid_provider_short_names.add(sn_upper)
valid_sold_to_short_names.add(sn_upper)
valid_shipped_to_short_names.add(sn_upper)
valid_broker_ids = set()
valid_broker_claves = set()
for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter(
CustomsBroker.tenant_id == tenant_id,
CustomsBroker.company_id == company_id,
).all():
valid_broker_ids.add(cb[0])
if cb[1] and str(cb[1]).strip():
valid_broker_claves.add(str(cb[1]).strip())
valid_transporter_keys = set()
for t in session.query(Transporter.transporter_key).filter(
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
).all():
if t[0]:
valid_transporter_keys.add((t[0] or "").strip().upper())
valid_incoterms = set()
for inc in session.query(Incoterm.code).all():
if inc[0]:
valid_incoterms.add((inc[0] or "").strip().upper())
valid_aduana_codes = set()
for cs in session.query(CustomsSection.customs_code).all():
if cs[0]:
valid_aduana_codes.add((cs[0] or "").strip())
valid_currency_codes = set()
for ct in session.query(CurrencyType.code).all():
if ct[0]:
valid_currency_codes.add((ct[0] or "").strip().upper())
exchange_rate_by_date = {}
for er in session.query(ExchangeRate.date, ExchangeRate.value).filter(
ExchangeRate.tenant_id == tenant_id,
ExchangeRate.company_id == company_id,
).all():
if er[0] and er[1] is not None:
dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10]
exchange_rate_by_date[dk] = er[1]
invoice_has_partidas_by_number = {}
existing_tipo_moneda_by_number = {}
q_li_count = (
session.query(InvoiceHeader.invoice_number, func.count(LineItem.id))
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
.group_by(InvoiceHeader.invoice_number)
)
for num, cnt in q_li_count.all():
if num:
invoice_has_partidas_by_number[str(num).strip()] = cnt > 0
q_fin = (
session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency)
.join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES),
)
)
for num, cur in q_fin.all():
if num and cur:
cur_str = (cur or "").strip().lower()
if cur_str == "foreign":
existing_tipo_moneda_by_number[str(num).strip()] = "ME"
elif cur_str == "local":
existing_tipo_moneda_by_number[str(num).strip()] = "MN"
else:
existing_tipo_moneda_by_number[str(num).strip()] = (cur_str or "").upper()[:2]
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
remesa_por_pedimento_csv = {}
for row in rows_list:
row_norm = row_from_template(row, "imp_def_header", normalize_header)
ped = (row_norm.get("PEDIMENTO") or "").strip()
rem = row_norm.get("REMESA")
factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip()
if not ped or not factura:
continue
key = _ped_key_from_row_def(ped)
if not key:
continue
try:
rem_int = int(rem) if rem is not None and str(rem).strip() else None
except (TypeError, ValueError):
rem_int = None
if rem_int is not None:
if key not in remesa_por_pedimento_csv:
remesa_por_pedimento_csv[key] = {}
if rem_int not in remesa_por_pedimento_csv[key]:
remesa_por_pedimento_csv[key][rem_int] = factura
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "imp_def_header", normalize_header)
warnings_row = []
err = validate_row_encabezados_impo_def(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
pedimento_data_by_key=pedimento_data_by_key,
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
valid_broker_ids=valid_broker_ids,
valid_broker_claves=valid_broker_claves,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_aduana_codes=valid_aduana_codes,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
autonumerar_remesas=autonumerar_remesas,
control_remesa=control_remesa,
remesa_inicio=remesa_inicio,
remesa_fin=remesa_fin,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
for w in warnings_row:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Encabezados importación definitiva scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Encabezados Exportación (Expo Def) y Cambio de Régimen: flujo específico (Clarion VALIDA_TODA_FAC_EXPO / VALIDA_PARCIAL) ---
if model_target == "invoice_header" and template_id == "exp_def_header":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
from api.v1.modules.a76.transportation.transporters.models import Transporter
from api.v1.modules.a76.manifests.manifest.models import Manifest
from .validators.encabezados_expo import validate_row_encabezados_expo
from .validators.encabezados_impo_temp import _pedimento_key_from_parsed
from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def
_fc = parse_footer_config(meta.get("footer_config"))
actualizar = meta.get("actualizar", False)
autonumerar_remesas = meta.get("autonumerar_remesas", False)
recalcular_fecha_pedimentos = meta.get("recalcular_fecha_pedimentos", False)
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "autonumerar_remesas" in _fc:
autonumerar_remesas = bool(_fc["autonumerar_remesas"])
if "recalcular_fecha_pedimentos" in _fc:
recalcular_fecha_pedimentos = bool(_fc["recalcular_fecha_pedimentos"])
cambio_regimen_raw = (meta.get("cambio_regimen") or _fc.get("cambio_regimen") or "NO").strip().upper()
cambio_regimen = cambio_regimen_raw == "SI"
tipo_factura = (meta.get("tipo_factura") or _fc.get("tipo_factura") or "AFIJO").strip().upper()
TIPOS_FACTURA_EXPO_VALIDOS = frozenset({"NODES", "AFIJO", "DONAC", "SCRAP"})
if tipo_factura not in TIPOS_FACTURA_EXPO_VALIDOS:
return {
"status": "failed",
"error": f"Tipo de factura '{tipo_factura}' no válido para Exportación. Debe ser uno de: NODES, AFIJO, DONAC, SCRAP.",
}
if cambio_regimen and tipo_factura != "AFIJO":
return {
"status": "failed",
"error": "Este tipo de factura no es compatible para Cambio de Régimen, seleccionar AFIJO.",
}
def _ped_key_from_row_expo(ped_str: str):
parsed = parse_pedimento_col_a_impo_def(ped_str)
if not parsed:
return None
return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2])
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status, InvoiceHeader.status_rep)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
invoice_exists_by_number = {}
invoice_processed_by_number = {}
invoice_in_report_by_number = {}
for num, iid, is_upd, is_rep in q_inv.all():
if num:
n = str(num).strip()
invoice_exists_by_number[n] = True
invoice_processed_by_number[n] = bool(is_upd)
invoice_in_report_by_number[n] = bool(is_rep) if is_rep is not None else False
if cambio_regimen:
ped_filter_op = "imp"
ped_filter_regimes = ["IMD"]
else:
ped_filter_op = "exp"
ped_filter_regimes = ["EXD", "ETE", "ETR"]
pedimento_data_by_key = {}
for p in (
session.query(
Pedimentos.id,
Pedimentos.customs_office,
Pedimentos.license,
Pedimentos.pedimento_number,
Pedimentos.operation_type,
Pedimentos.regime,
Pedimentos.pedimento_type,
Pedimentos.pedimento_code,
)
.filter(
Pedimentos.tenant_id == tenant_id,
Pedimentos.company_id == company_id,
Pedimentos.operation_type == ped_filter_op,
Pedimentos.regime.in_(ped_filter_regimes),
)
.all()
):
co = (p.customs_office or "").strip()
lic = (p.license or "").strip()
num = (p.pedimento_number or "").strip()
if not co or not lic or not num:
continue
key = _pedimento_key_from_parsed(co, lic, num)
entry_date = None
end_date = None
pd = (
session.query(PedimentoDates.entry_date, PedimentoDates.end_date)
.filter(PedimentoDates.pedimento_id == p.id).first()
)
if pd:
entry_date = pd[0]
end_date = pd[1]
info = {
"id": p.id,
"regime": (p.regime or "").strip(),
"operation_type": (p.operation_type or "").strip().upper()[:3],
"pedimento_type": (p.pedimento_type or "").strip(),
"pedimento_code": (p.pedimento_code or "").strip().upper(),
"entry_date": entry_date,
"end_date": end_date,
}
if key not in pedimento_data_by_key:
pedimento_data_by_key[key] = []
pedimento_data_by_key[key].append(info)
remesa_por_pedimento_bd = {}
q_rem = (
session.query(
InvoiceComplianceMx.remesa,
Pedimentos.customs_office,
Pedimentos.license,
Pedimentos.pedimento_number,
)
.join(InvoiceHeader, InvoiceHeader.id == InvoiceComplianceMx.invoice_id)
.join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id)
.filter(
InvoiceComplianceMx.tenant_id == tenant_id,
InvoiceComplianceMx.company_id == company_id,
InvoiceComplianceMx.pedimento_id.isnot(None),
InvoiceComplianceMx.remesa.isnot(None),
InvoiceHeader.operation_type == "exp",
)
)
for rem, co, lic, num in q_rem.all():
if co and lic and num and rem is not None:
key = _pedimento_key_from_parsed(
(co or "").strip()[:2],
(lic or "").strip(),
(num or "").strip(),
)
if key not in remesa_por_pedimento_bd:
remesa_por_pedimento_bd[key] = set()
remesa_por_pedimento_bd[key].add(int(rem))
valid_provider_ids = set()
valid_sold_to_ids = set()
valid_shipped_to_ids = set()
valid_provider_short_names = set()
valid_sold_to_short_names = set()
valid_shipped_to_short_names = set()
for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
).all():
valid_provider_ids.add(cp[0])
valid_sold_to_ids.add(cp[0])
valid_shipped_to_ids.add(cp[0])
if cp[1] and str(cp[1]).strip():
sn_upper = str(cp[1]).strip().upper()
valid_provider_short_names.add(sn_upper)
valid_sold_to_short_names.add(sn_upper)
valid_shipped_to_short_names.add(sn_upper)
valid_broker_ids = set()
valid_broker_claves = set()
for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter(
CustomsBroker.tenant_id == tenant_id,
CustomsBroker.company_id == company_id,
).all():
valid_broker_ids.add(cb[0])
if cb[1] and str(cb[1]).strip():
valid_broker_claves.add(str(cb[1]).strip())
valid_transporter_keys = set()
for t in session.query(Transporter.transporter_key).filter(
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
).all():
if t[0]:
valid_transporter_keys.add((t[0] or "").strip().upper())
valid_incoterms = set()
for inc in session.query(Incoterm.code).all():
if inc[0]:
valid_incoterms.add((inc[0] or "").strip().upper())
valid_aduana_codes = set()
for cs in session.query(CustomsSection.customs_code).all():
if cs[0]:
valid_aduana_codes.add((cs[0] or "").strip())
valid_currency_codes = set()
for ct in session.query(CurrencyType.code).all():
if ct[0]:
valid_currency_codes.add((ct[0] or "").strip().upper())
valid_manifiesto_codes = set()
for m in session.query(Manifest.manifest_number).filter(
Manifest.tenant_id == tenant_id,
Manifest.company_id == company_id,
).all():
if m[0] and str(m[0]).strip():
valid_manifiesto_codes.add(str(m[0]).strip())
exchange_rate_by_date = {}
for er in session.query(ExchangeRate.date, ExchangeRate.value).filter(
ExchangeRate.tenant_id == tenant_id,
ExchangeRate.company_id == company_id,
).all():
if er[0] and er[1] is not None:
dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10]
exchange_rate_by_date[dk] = er[1]
invoice_has_partidas_by_number = {}
existing_tipo_moneda_by_number = {}
q_li_count = (
session.query(InvoiceHeader.invoice_number, func.count(LineItem.id))
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
.group_by(InvoiceHeader.invoice_number)
)
for num, cnt in q_li_count.all():
if num:
invoice_has_partidas_by_number[str(num).strip()] = cnt > 0
q_fin = (
session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency)
.join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
for num, cur in q_fin.all():
if num and cur:
cur_str = (cur or "").strip().lower()
if cur_str == "foreign":
existing_tipo_moneda_by_number[str(num).strip()] = "ME"
elif cur_str == "local":
existing_tipo_moneda_by_number[str(num).strip()] = "MN"
else:
existing_tipo_moneda_by_number[str(num).strip()] = (cur_str or "").upper()[:2]
date_format = _fc.get("dateFormat") or meta.get("date_format")
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
remesa_por_pedimento_csv = {}
for row in rows_list:
row_norm = row_from_template(row, "exp_def_header", normalize_header)
ped = (row_norm.get("PEDIMENTO") or "").strip()
rem = row_norm.get("REMESA")
factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip()
if not ped or not factura:
continue
key = _ped_key_from_row_expo(ped)
if not key:
continue
try:
rem_int = int(rem) if rem is not None and str(rem).strip() else None
except (TypeError, ValueError):
rem_int = None
if rem_int is not None:
if key not in remesa_por_pedimento_csv:
remesa_por_pedimento_csv[key] = {}
if rem_int not in remesa_por_pedimento_csv[key]:
remesa_por_pedimento_csv[key][rem_int] = factura
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "exp_def_header", normalize_header)
warnings_row = []
err = validate_row_encabezados_expo(
row_norm,
i,
actualizar=actualizar,
cambio_regimen=cambio_regimen,
tipo_factura=tipo_factura,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
invoice_in_report_by_number=invoice_in_report_by_number,
pedimento_data_by_key=pedimento_data_by_key,
remesa_por_pedimento_bd=remesa_por_pedimento_bd,
remesa_por_pedimento_csv=remesa_por_pedimento_csv,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_broker_ids=valid_broker_ids,
valid_broker_claves=valid_broker_claves,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_aduana_codes=valid_aduana_codes,
valid_currency_codes=valid_currency_codes,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
valid_manifiesto_codes=valid_manifiesto_codes,
valid_enviado_por_ids=valid_provider_ids,
valid_enviado_por_short_names=valid_provider_short_names,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
autonumerar_remesas=autonumerar_remesas,
recalcular_fecha_pedimentos=recalcular_fecha_pedimentos,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
for w in warnings_row:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Encabezados exportación scan failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Encabezados Compras Mexicanas: flujo específico (Clarion VALIDA_TODA_FAC_COM_MEX / VALIDA_PARCIAL) ---
if model_target == "invoice_header" and template_id == "cmex_header":
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceFinancials
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate
from api.v1.modules.a76.transportation.transporters.models import Transporter
from .validators.encabezados_cmex import validate_row_encabezados_cmex
_fc = parse_footer_config(meta.get("footer_config"))
actualizar = meta.get("actualizar", False)
if _fc and "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
invoice_exists_by_number = {}
invoice_processed_by_number = {}
for num, iid, is_upd in q_inv.all():
if num:
n = str(num).strip()
invoice_exists_by_number[n] = True
invoice_processed_by_number[n] = bool(is_upd)
valid_provider_ids = set()
valid_sold_to_ids = set()
valid_shipped_to_ids = set()
valid_provider_short_names = set()
valid_sold_to_short_names = set()
valid_shipped_to_short_names = set()
for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter(
ClientProvider.tenant_id == tenant_id,
ClientProvider.company_id == company_id,
).all():
valid_provider_ids.add(cp[0])
valid_sold_to_ids.add(cp[0])
valid_shipped_to_ids.add(cp[0])
if cp[1] and str(cp[1]).strip():
sn_upper = str(cp[1]).strip().upper()
valid_provider_short_names.add(sn_upper)
valid_sold_to_short_names.add(sn_upper)
valid_shipped_to_short_names.add(sn_upper)
valid_transporter_keys = set()
for t in session.query(Transporter.transporter_key).filter(
Transporter.tenant_id == tenant_id,
Transporter.company_id == company_id,
).all():
if t[0]:
valid_transporter_keys.add((t[0] or "").strip().upper())
valid_incoterms = set()
for inc in session.query(Incoterm.code).all():
if inc[0]:
valid_incoterms.add((inc[0] or "").strip().upper())
valid_currency_codes = set()
for ct in session.query(CurrencyType.code).all():
if ct[0]:
valid_currency_codes.add((ct[0] or "").strip().upper())
exchange_rate_by_date = {}
for er in session.query(ExchangeRate.date, ExchangeRate.value).filter(
ExchangeRate.tenant_id == tenant_id,
ExchangeRate.company_id == company_id,
).all():
if er[0] and er[1] is not None:
dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10]
exchange_rate_by_date[dk] = er[1]
invoice_has_partidas_by_number = {}
existing_tipo_moneda_by_number = {}
q_li_count = (
session.query(InvoiceHeader.invoice_number, func.count(LineItem.id))
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
.group_by(InvoiceHeader.invoice_number)
)
for num, cnt in q_li_count.all():
if num:
invoice_has_partidas_by_number[str(num).strip()] = cnt > 0
q_fin = (
session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency)
.join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "MEX",
)
)
for num, cur in q_fin.all():
if num and cur:
cur_str = (cur or "").strip().lower()
if cur_str == "foreign":
existing_tipo_moneda_by_number[str(num).strip()] = "ME"
elif cur_str == "local":
existing_tipo_moneda_by_number[str(num).strip()] = "MN"
else:
existing_tipo_moneda_by_number[str(num).strip()] = (cur_str or "").upper()[:2]
with open(file_path, "r", encoding="utf-8-sig") as f_in:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
rows_list = list(reader)
error_count = 0
processed_rows = 0
error_lines_list = []
errors_detail = []
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in enumerate(rows_list, start=1):
if i % 1000 == 0:
self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count})
row_norm = row_from_template(row, "cmex_header", normalize_header)
warnings_row = []
err = validate_row_encabezados_cmex(
row_norm,
i,
actualizar=actualizar,
invoice_exists_by_number=invoice_exists_by_number,
invoice_processed_by_number=invoice_processed_by_number,
valid_provider_ids=valid_provider_ids,
valid_sold_to_ids=valid_sold_to_ids,
valid_shipped_to_ids=valid_shipped_to_ids,
valid_transporter_keys=valid_transporter_keys,
valid_incoterms=valid_incoterms,
valid_currency_codes=valid_currency_codes,
exchange_rate_by_date=exchange_rate_by_date,
invoice_has_partidas_by_number=invoice_has_partidas_by_number,
existing_tipo_moneda_by_number=existing_tipo_moneda_by_number,
valid_provider_short_names=valid_provider_short_names,
valid_sold_to_short_names=valid_sold_to_short_names,
valid_shipped_to_short_names=valid_shipped_to_short_names,
date_format=date_format,
parse_date_fn=parse_date,
warnings=warnings_row,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n")
if len(errors_detail) < 500:
errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")})
for w in warnings_row:
if len(errors_detail) < 500:
errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True})
processed_rows += 1
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows
)
except Exception as e:
logger.exception("Encabezados Compras Mexicanas scan failed: %s", e)
return {"status": "failed", "error": str(e)}
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import (
CodePedimentoRegimen,
)
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from api.v1.modules.a76.parts.models import Part
models = {
"InvoiceHeader": InvoiceHeader,
"InvoiceType": InvoiceType,
"ClientProvider": ClientProvider,
"CustomsBroker": CustomsBroker,
"RegimenPedimento": RegimenPedimento,
"CodePedimentoRegimen": CodePedimentoRegimen,
"PedimentoCode": PedimentoCode,
"CurrencyType": CurrencyType,
"CustomsSection": CustomsSection,
"Incoterm": Incoterm,
"Part": Part,
}
with CoreSessionLocal() as session, \
open(file_path, 'r', encoding='utf-8-sig') as f_in, \
open(error_path, 'w', encoding='utf-8') as f_err:
validator = ForeignKeyValidator(session, tenant_id, company_id)
invoice_id_cache: Dict[str, Optional[int]] = {}
# Detect Delimiter
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except:
dialect = 'excel'
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
# Check for Progress Update
if i % 1000 == 0:
self.update_state(state='PROGRESS', meta={
'current': i,
'total': total_rows,
'errors': error_count
})
# Solo columnas de la plantilla (respetar plantilla tal cual)
row_norm = row_from_template(row, template_id, normalize_header)
errors = validate_row_strict(
row_norm,
model_target,
i,
date_format,
validator,
inv_type_value,
invoice_id_cache,
models,
)
if errors:
error_count += 1
# Write simple JSON error
f_err.write(json.dumps(errors) + "\n")
processed_rows += 1
except Exception as e:
logger.error(f"Scan failed: {e}")
return {"status": "failed", "error": str(e)}
# 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker)
error_lines_list = []
errors_detail: List[Dict[str, Any]] = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
if len(errors_detail) < 500:
errors_detail.append(
{
"line": err["line"],
"col": err.get("col", ""),
"msg": err.get("msg", ""),
}
)
except Exception:
pass
common_storage.store_error_lines(effective_job_type, job_id, error_lines_list)
except Exception as e:
logger.warning(f"Failed to store error lines in Redis: {e}")
return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail)
def validate_row_phase_1(
row: Dict[str, Any],
target: str,
line_num: int,
date_format: Optional[str],
) -> Optional[Dict[str, Any]]:
"""
Validation: Unique IDs, Dates, and Numeric constraint checks.
Target: 'invoice_header' or 'invoice_details'
"""
def check_decimal(col_name):
val = row.get(col_name)
if val and str(val).strip():
if parse_decimal(val) is None:
return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"}
return None
def check_int(col_name):
val = row.get(col_name)
if val and str(val).strip():
if parse_int(val) is None:
return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"}
return None
def check_date(col_name):
date_str = row.get(col_name)
if date_str and str(date_str).strip():
if not is_valid_date(date_str, date_format):
expected = display_date_format(date_format)
return {
"line": line_num,
"col": col_name,
"msg": f"Formato de fecha inválido ({expected})",
}
return None
def check_weight(col_name):
val = row.get(col_name)
if val and str(val).strip():
if parse_weight_unit(val) is None:
return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"}
return None
def check_currency(col_name):
val = row.get(col_name)
if val and str(val).strip():
parsed_currency = parse_currency(val, None)
val_norm = normalize_header(val)
# parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid
if parsed_currency.value == "manual" and "MANUAL" not in val_norm:
return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"}
return None
def check_transport_type(col_name):
val = row.get(col_name)
if val and str(val).strip():
if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES:
return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"}
return None
# A. Invoice Header
if target == 'invoice_header':
# 1. Unique ID
if not row.get('NUMERO FACTURA') and not row.get('NUM FACTURA') and not row.get('ID'):
return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"}
# 2. Date Format
date_str = row.get('FECHA FACTURA') or row.get('FECHA')
if not date_str or not str(date_str).strip():
return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"}
err = check_date('FECHA FACTURA') or check_date('FECHA')
if err: return err
err = check_date('FECHA EMISION')
if err: return err
# 3. Numeric Fields
for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']:
err = check_decimal(col)
if err: return err
# 4. Integer FKs (CLAVE PROVEEDOR/VENDIDO/ENVIADO aceptan short_name; AGENTE ADUANAL acepta clave; solo REMESA exige entero)
int_fk_cols = ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA']
if target == 'invoice_header':
int_fk_cols = ['REMESA'] # proveedor/vendido/enviado por short_name; agente aduanal por clave
for col in int_fk_cols:
err = check_int(col)
if err: return err
# 5. Enums
for col in ['TIPO PESO']:
err = check_weight(col)
if err: return err
for col in ['TIPO MONEDA']:
err = check_currency(col)
if err: return err
for col in ['TIPO TRANSPORTE']:
err = check_transport_type(col)
if err: return err
# B. Invoice Details (Parts)
elif target == 'invoice_details':
# 1. Line Number
if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'):
return {"line": line_num, "col": "LINEA", "msg": "Requerido"}
# 2. Parent Link (Invoice Number)
if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')):
return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"}
# 3. Numeric Fields
for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']:
err = check_decimal(col)
if err: return err
for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']:
err = check_int(col)
if err: return err
return None
def validate_row_strict(
row: Dict[str, Any],
target: str,
line_num: int,
date_format: Optional[str],
validator: ForeignKeyValidator,
inv_type_value: str,
invoice_id_cache: Dict[str, Optional[int]],
models: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
err = validate_row_phase_1(row, target, line_num, date_format)
if err:
return err
InvoiceHeader = models["InvoiceHeader"]
InvoiceType = models["InvoiceType"]
ClientProvider = models["ClientProvider"]
CustomsBroker = models["CustomsBroker"]
RegimenPedimento = models["RegimenPedimento"]
CurrencyType = models["CurrencyType"]
CustomsSection = models["CustomsSection"]
Incoterm = models["Incoterm"]
Part = models["Part"]
if target == "invoice_header":
if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True):
return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"}
err = _validate_client_provider_ref(
validator, ClientProvider, row.get("CLAVE PROVEEDOR"), line_num, "CLAVE PROVEEDOR", required=True
)
if err:
return err
err = _validate_client_provider_ref(
validator, ClientProvider, row.get("CLAVE VENDIDO A"), line_num, "CLAVE VENDIDO A", required=True
)
if err:
return err
err = _validate_client_provider_ref(
validator, ClientProvider, row.get("CLAVE ENVIADO A"), line_num, "CLAVE ENVIADO A", required=True
)
if err:
return err
err = _validate_customs_broker_ref(
validator, CustomsBroker, row.get("AGENTE ADUANAL"), line_num, "AGENTE ADUANAL", required=False
)
if err:
return err
err = validate_public_code(
validator,
RegimenPedimento,
row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"),
line_num,
"CLAVEDOCUMENTO",
)
if err:
return err
err = validate_public_code(
validator,
CustomsSection,
row.get("ADUANA DE CRUCE"),
line_num,
"ADUANA DE CRUCE",
field_name="customs_code",
)
if err:
return err
err = validate_public_code(
validator,
CurrencyType,
row.get("CLAVE MONEDA"),
line_num,
"CLAVE MONEDA",
)
if err:
return err
err = validate_public_code(
validator,
Incoterm,
row.get("CLAVE INCOTERM"),
line_num,
"CLAVE INCOTERM",
)
if err:
return err
elif target == "invoice_details":
invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip()
if not invoice_number:
return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"}
cache_key = f"{invoice_number}|{inv_type_value}"
if cache_key in invoice_id_cache:
invoice_id = invoice_id_cache[cache_key]
else:
invoice_id = (
validator.session.query(InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == validator.tenant_id,
InvoiceHeader.company_id == validator.company_id,
InvoiceHeader.invoice_number == invoice_number,
InvoiceHeader.invoice_type == inv_type_value,
)
.scalar()
)
invoice_id_cache[cache_key] = invoice_id
if not invoice_id:
return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"}
part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip()
if not part_num:
return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"}
if not validator.check_exists(Part, part_num, field_name="part_number"):
return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"}
return None
def parse_footer_config(config: Optional[str]) -> Dict[str, Any]:
if not config:
return {}
try:
if isinstance(config, str):
return json.loads(config)
if isinstance(config, dict):
return config
except Exception:
return {}
return {}
def display_date_format(date_format: Optional[str]) -> str:
if not date_format:
return "YYYY-MM-DD"
return date_format.upper()
def parse_date(date_text: Optional[str], date_format: Optional[str]) -> Optional[datetime.date]:
if not date_text:
return None
candidates = []
fmt_map = {
"dd/mm/yyyy": "%d/%m/%Y",
"mm/dd/yyyy": "%m/%d/%Y",
"yyyy-mm-dd": "%Y-%m-%d",
}
if date_format and date_format in fmt_map:
candidates.append(fmt_map[date_format])
candidates.extend(["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y"])
for fmt in candidates:
try:
return datetime.strptime(str(date_text).strip(), fmt).date()
except ValueError:
continue
return None
def is_valid_date(date_text: Optional[str], date_format: Optional[str]) -> bool:
return parse_date(date_text, date_format) is not None
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def normalize_row(row: Dict[str, Any]) -> Dict[str, Any]:
return {normalize_header(k): v for k, v in row.items()}
def parse_int(value: Any) -> Optional[int]:
if value is None:
return None
text = str(value).strip()
if not text:
return None
try:
return int(text)
except ValueError:
pass
try:
f = float(text.replace(",", ""))
if f == int(f):
return int(f)
return None
except ValueError:
return None
def parse_decimal(value: Any) -> Optional[Decimal]:
if value is None:
return None
text = str(value).strip()
if not text:
return None
text = text.replace(",", "")
try:
return Decimal(text)
except Exception:
return None
def decimal_or_zero(value: Any) -> Decimal:
"""Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default)."""
return parse_decimal(value) or Decimal("0")
def int_or_zero(value: Any) -> int:
"""Return parsed int or 0 for CSV nulls/empty (vanilla default)."""
return parse_int(value) if parse_int(value) is not None else 0
def parse_currency(value: Optional[str], currency_type: Optional[str]):
from api.v1.modules.a76.invoices.models import Currency
if value:
normalized = normalize_header(value)
if normalized in {"MN", "M N", "NACIONAL", "LOCAL", "PESOS", "PESO"}:
return Currency.LOCAL
if normalized in {"ME", "M E", "EXTRANJERA", "EXTRANJERO", "FOREIGN", "USD", "DOLAR", "DOLARES"}:
return Currency.FOREIGN
if "MANUAL" in normalized:
return Currency.MANUAL
if currency_type and str(currency_type).strip().upper() == "MXN":
return Currency.LOCAL
if currency_type:
return Currency.FOREIGN
return Currency.MANUAL
def parse_weight_unit(value: Optional[str]):
from api.v1.modules.a76.invoices.models import WeightUnit
if not value:
return None
normalized = normalize_header(value)
if normalized in {"KG", "KGS", "KILOS", "KILOGRAMOS"}:
return WeightUnit.KGS
if normalized in {"LB", "LBS", "LIBRAS"}:
return WeightUnit.LBS
return None
def resolve_tenant_fk_id(
session: CoreSessionLocal,
model,
value: Optional[int],
tenant_id: int,
company_id: int,
cache: Dict[int, Optional[int]],
) -> Optional[int]:
if value is None:
return None
if value in cache:
return cache[value]
exists = (
session.query(model.id)
.filter(
model.id == value,
model.tenant_id == tenant_id,
model.company_id == company_id,
)
.scalar()
)
cache[value] = value if exists is not None else None
return cache[value]
def resolve_client_provider_id(
session: CoreSessionLocal,
model,
value: Any,
tenant_id: int,
company_id: int,
cache: Dict[Any, Optional[int]],
) -> Optional[int]:
"""Resuelve ID de ClientProvider por id (entero) o por short_name (texto). value puede ser int o str."""
if value is None or (isinstance(value, str) and not value.strip()):
return None
if value in cache:
return cache[value]
pid = parse_int(value)
if pid is not None:
found = (
session.query(model.id)
.filter(
model.id == pid,
model.tenant_id == tenant_id,
model.company_id == company_id,
)
.scalar()
)
cache[value] = found
return found
short_norm = str(value).strip().upper()
if short_norm in cache:
return cache[short_norm]
found = (
session.query(model.id)
.filter(
func.upper(model.short_name) == short_norm,
model.tenant_id == tenant_id,
model.company_id == company_id,
)
.scalar()
)
cache[value] = found
cache[short_norm] = found
return found
def resolve_customs_broker_id(
session: CoreSessionLocal,
model,
value: Any,
tenant_id: int,
company_id: int,
cache: Dict[Any, Optional[int]],
) -> Optional[int]:
"""Resuelve ID de CustomsBroker por id (entero) o por broker_key (clave). value puede ser int o str."""
if value is None or (isinstance(value, str) and not value.strip()):
return None
if value in cache:
return cache[value]
pid = parse_int(value)
if pid is not None:
found = (
session.query(model.id)
.filter(
model.id == pid,
model.tenant_id == tenant_id,
model.company_id == company_id,
)
.scalar()
)
cache[value] = found
return found
clave = str(value).strip()
if clave in cache:
return cache[clave]
found = (
session.query(model.id)
.filter(
model.broker_key == clave,
model.tenant_id == tenant_id,
model.company_id == company_id,
)
.scalar()
)
cache[value] = found
cache[clave] = found
return found
def resolve_public_code(
session: CoreSessionLocal,
model,
column,
value: Optional[str],
cache: Dict[str, Optional[str]],
) -> Optional[str]:
if not value:
return None
normalized = str(value).strip().upper()
if not normalized:
return None
if normalized in cache:
return cache[normalized]
exists = session.query(column).filter(column == normalized).scalar()
cache[normalized] = normalized if exists is not None else None
return cache[normalized]
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str, model_target: str, job_type_override: Optional[str] = None):
"""Pass 2: Re-read CSV, Skip Errors, Bulk Insert. Delegates to _do_insert_valid_rows."""
return _do_insert_valid_rows(job_id, model_target, job_type_override)
def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Optional[str] = None) -> Dict[str, Any]:
"""
Pass 2: Re-read CSV, Skip Errors, Bulk Insert.
File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem.
When job_type_override is set (e.g. "exp" for Exportación), storage keys use that prefix.
"""
effective_job_type = job_type_override if job_type_override is not None else JOB_TYPE
log_prefix = "Exportación import" if effective_job_type else "Invoices import"
logger.info(f"Starting Commit for {job_id} target {model_target}")
# Ensure we have the file on this worker: prefer Redis (so any worker can run commit)
file_path = common_storage.ensure_file_from_redis(effective_job_type, job_id, log_prefix)
if not file_path:
alt_path = common_storage.file_path_for_job(effective_job_type, job_id)
if not os.path.exists(alt_path):
return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."}
file_path = alt_path
else:
common_storage.ensure_meta_from_redis(effective_job_type, job_id, file_path, log_prefix)
try:
tenant_id, company_id = common_meta.require_tenant_context(file_path)
except ValueError as e:
return {"status": "failed", "error": str(e)}
meta = common_meta.load_meta(file_path)
meta_path = common_meta.get_meta_path(file_path)
error_path = common_storage.error_path_for_job(effective_job_type, job_id)
error_lines = common_storage.get_error_lines(effective_job_type, job_id, error_path)
# Si el upload fue de series (template_id imp_temp_series o imp_def_series), usar flujo series aunque model_target venga mal
use_series_flow = (
model_target == "invoice_series"
or meta.get("template_id") in ("imp_temp_series", "imp_def_series", "cmex_series", "exp_def_series")
)
_footer_for_series = parse_footer_config(meta.get("footer_config")) or {}
_inv_type_series = normalize_public_code(_footer_for_series.get("invoice_type") or meta.get("invoice_type") or "")
use_def_series_commit = (
use_series_flow
and (
meta.get("template_id") == "imp_def_series"
or meta.get("template_id") == "cmex_series"
or _inv_type_series in ("DEF", "MATDE", "EXDEF")
)
)
use_expo_series_commit = use_series_flow and meta.get("template_id") == "exp_def_series"
# --- Series de Exportación Definitiva: commit (INSERT/UPDATE item_line_series para facturas exp) ---
if use_expo_series_commit:
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from .validators.series_expo import row_to_series_normalized_expo
actualizar = meta.get("actualizar", False)
autonumerar = meta.get("autonumerar", True)
_fc = parse_footer_config(meta.get("footer_config"))
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
else:
as_val = _fc.get("autonumber_series", "true")
autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes")
with CoreSessionLocal() as session:
q_inv = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "exp",
)
)
invoice_id_by_number: Dict[str, int] = {}
for num, iid in q_inv.all():
if num:
invoice_id_by_number[str(num).strip()] = iid
inserted_count = 0
updated_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
skipped_invalid += 1
continue
row_norm = row_from_template(row, "exp_def_series", normalize_header)
data = row_to_series_normalized_expo(row_norm)
invoice_number = data["NUMERO FACTURA"]
linea_factura = data["LINEA FACTURA"]
linea_serie = data["LINEA SERIE"]
if not invoice_number or invoice_number not in invoice_id_by_number:
skipped_invalid += 1
skipped_details.append({
"line": i,
"invoice": invoice_number or "(vacío)",
"reason": "Factura de exportación no encontrada.",
})
continue
invoice_id = invoice_id_by_number[invoice_number]
line_number_val = parse_int(linea_factura)
if line_number_val is None:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."})
continue
line_item = (
session.query(LineItem)
.filter(
LineItem.invoice_id == invoice_id,
LineItem.line_number == line_number_val,
LineItem.tenant_id == tenant_id,
LineItem.company_id == company_id,
)
.first()
)
if not line_item:
line_numbers = [
r[0] for r in
session.query(LineItem.line_number)
.filter(
LineItem.invoice_id == invoice_id,
LineItem.tenant_id == tenant_id,
LineItem.company_id == company_id,
)
.order_by(LineItem.line_number)
.all()
]
existing_str = ", ".join(str(n) for n in line_numbers) if line_numbers else "ninguna"
skipped_invalid += 1
skipped_details.append({
"line": i,
"invoice": invoice_number,
"reason": f"Partida línea {linea_factura} no existe en la factura. Partidas existentes: {existing_str}.",
})
continue
if autonumerar:
max_row = (
session.query(Serie.row)
.filter(Serie.line_item_id == line_item.id)
.order_by(Serie.row.desc())
.limit(1)
.scalar()
)
row_num = (max_row or 0) + 1
else:
row_num = parse_int(linea_serie)
if row_num is None:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."})
continue
existing_serie = (
session.query(Serie)
.filter(
Serie.line_item_id == line_item.id,
Serie.row == row_num,
)
.first()
)
if existing_serie:
if actualizar:
existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers
existing_serie.model = data["MODELO"] or existing_serie.model
existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model
existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id
session.add(existing_serie)
updated_count += 1
else:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."})
else:
new_serie = Serie(
tenant_id=tenant_id,
company_id=company_id,
line_item_id=line_item.id,
row=row_num,
serial_numbers=data["SERIE"] or None,
model=data["MODELO"] or None,
sub_model=data["SUB MODELO"] or None,
number_id=data["NUMERO ID"] or None,
)
session.add(new_serie)
inserted_count += 1
session.commit()
common_storage.cleanup_import_job(effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path)
status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed")
out = {
"status": status,
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_duplicate": 0,
"skipped_details": skipped_details,
}
if status == "failed":
out["error"] = "No hay registros válidos en el archivo CSV."
elif status == "warning" and skipped_invalid:
out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados."
return out
except Exception as e:
logger.exception("Series exportación definitiva commit failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Series de Importación Definitiva: commit (INSERT/UPDATE item_line_series para facturas DEF o MEX) ---
if use_def_series_commit:
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
from .validators.series_impo_def import (
validate_row_series_impo_def,
row_to_series_normalized_def,
)
_series_template_id = meta.get("template_id")
is_cmex_series = _series_template_id == "cmex_series"
SERIES_INV_TYPES = ("MEX",) if is_cmex_series else ("DEF", "MATDE", "EXDEF")
series_row_template_id = "cmex_series" if is_cmex_series else "imp_def_series"
DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") # keep for any legacy reference
actualizar = meta.get("actualizar", False)
autonumerar = meta.get("autonumerar", True)
validar_series_exception = meta.get("validar_series", False)
_fc = parse_footer_config(meta.get("footer_config"))
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
elif _fc.get("mode") == "update":
actualizar = True
elif _fc.get("mode") == "replace":
actualizar = False
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
else:
as_val = _fc.get("autonumber_series", "true")
autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes")
if "validar_series" in _fc:
validar_series_exception = bool(_fc["validar_series"])
with CoreSessionLocal() as session:
q = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(SERIES_INV_TYPES),
)
)
rows_inv = q.all()
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in rows_inv:
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
partida_max_series = {}
q_qty = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
LineQuantity.quantity,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(SERIES_INV_TYPES),
)
)
for num, ln, qty in q_qty.all():
if num is not None and ln is not None:
key = (str(num).strip(), str(ln).strip())
partida_max_series[key] = int(qty) if qty else 0
existing_series_keys = set()
existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
if actualizar and not autonumerar:
q_ser = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
Serie.row,
Serie.serial_numbers,
Serie.model,
Serie.sub_model,
Serie.number_id,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(Serie, Serie.line_item_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type.in_(SERIES_INV_TYPES),
)
)
for num, ln, rw, sn, md, sm, nid in q_ser.all():
if num is not None:
k = (str(num).strip(), str(ln).strip(), str(rw).strip())
existing_series_keys.add(k)
existing_series_data.setdefault(k, {
"serial_numbers": sn or "",
"model": md or "",
"sub_model": sm or "",
"number_id": nid or "",
})
csv_series_count_so_far: Dict[Tuple[str, str], int] = {}
inserted_count = 0
updated_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, series_row_template_id, normalize_header)
err = validate_row_series_impo_def(
row_norm,
i,
actualizar=actualizar,
autonumerar=autonumerar,
validar_series_exception=validar_series_exception,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
partida_max_series=partida_max_series,
csv_series_count_so_far=csv_series_count_so_far,
existing_series_keys=existing_series_keys,
existing_series_data=existing_series_data,
warnings=None,
catalog_label="Compras Mexicanas" if is_cmex_series else "Importación Definitiva",
)
if err:
skipped_invalid += 1
skipped_details.append({
"line": i,
"invoice": (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip(),
"reason": err.get("msg", ""),
})
continue
data = row_to_series_normalized_def(row_norm)
invoice_number = data["NUMERO FACTURA"]
linea_factura = data["LINEA FACTURA"]
linea_serie = data["LINEA SERIE"]
if not invoice_number or invoice_number not in invoice_id_by_number:
skipped_invalid += 1
continue
invoice_id = invoice_id_by_number[invoice_number]
line_number_val = parse_int(linea_factura)
if line_number_val is None:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."})
continue
line_item = (
session.query(LineItem)
.filter(
LineItem.invoice_id == invoice_id,
LineItem.line_number == line_number_val,
LineItem.tenant_id == tenant_id,
LineItem.company_id == company_id,
)
.first()
)
if not line_item:
line_numbers = [
r[0] for r in
session.query(LineItem.line_number)
.filter(
LineItem.invoice_id == invoice_id,
LineItem.tenant_id == tenant_id,
LineItem.company_id == company_id,
)
.order_by(LineItem.line_number)
.all()
]
existing_str = ", ".join(str(n) for n in line_numbers) if line_numbers else "ninguna"
skipped_invalid += 1
skipped_details.append({
"line": i,
"invoice": invoice_number,
"reason": f"Partida línea {linea_factura} no existe en la factura. Partidas existentes en la factura: {existing_str}.",
})
continue
if autonumerar:
max_row = (
session.query(Serie.row)
.filter(Serie.line_item_id == line_item.id)
.order_by(Serie.row.desc())
.limit(1)
.scalar()
)
row_num = (max_row or 0) + 1
else:
row_num = parse_int(linea_serie)
if row_num is None:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."})
continue
existing_serie = (
session.query(Serie)
.filter(
Serie.line_item_id == line_item.id,
Serie.row == row_num,
)
.first()
)
if existing_serie:
if actualizar:
existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers
existing_serie.model = data["MODELO"] or existing_serie.model
existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model
existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id
session.add(existing_serie)
updated_count += 1
else:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."})
else:
new_serie = Serie(
tenant_id=tenant_id,
company_id=company_id,
line_item_id=line_item.id,
row=row_num,
serial_numbers=data["SERIE"] or None,
model=data["MODELO"] or None,
sub_model=data["SUB MODELO"] or None,
number_id=data["NUMERO ID"] or None,
)
session.add(new_serie)
inserted_count += 1
key_csv = (invoice_number.strip(), (linea_factura or "").strip())
if key_csv[0] and key_csv[1]:
csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1
session.commit()
common_storage.cleanup_import_job(effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path)
status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed")
out = {
"status": status,
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_duplicate": 0,
"skipped_details": skipped_details,
}
if status == "failed":
out["error"] = "No hay registros válidos en el archivo CSV."
elif status == "warning" and skipped_invalid:
out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados."
return out
except Exception as e:
logger.exception("Series importación definitiva commit failed: %s", e)
return {"status": "failed", "error": str(e)}
# --- Series de Importación Temporal: commit (INSERT/UPDATE item_line_series) ---
# Paridad CSV: campos no presentes en CSV se persisten como null; no se exigen campos que no están en la plantilla.
if use_series_flow:
try:
from api.v1.modules.a76.invoices.models import InvoiceHeader
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.series.models import Serie
from .validators.series_impo_temp import (
validate_row_series_impo_temp,
row_to_series_normalized,
)
actualizar = meta.get("actualizar", False)
autonumerar = meta.get("autonumerar", True)
validar_series_exception = meta.get("validar_series", False)
_fc = parse_footer_config(meta.get("footer_config"))
if _fc:
if "actualizar" in _fc:
actualizar = bool(_fc["actualizar"])
elif _fc.get("mode") == "update":
actualizar = True
elif _fc.get("mode") == "replace":
actualizar = False
if "autonumerar" in _fc:
autonumerar = bool(_fc["autonumerar"])
else:
as_val = _fc.get("autonumber_series", "true")
autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes")
if "validar_series" in _fc:
validar_series_exception = bool(_fc["validar_series"])
with CoreSessionLocal() as session:
q = (
session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
rows_inv = q.all()
invoice_id_by_number: Dict[str, int] = {}
invoice_processed_by_number: Dict[str, bool] = {}
for num, iid, is_upd in rows_inv:
if num:
invoice_id_by_number[str(num).strip()] = iid
invoice_processed_by_number[str(num).strip()] = bool(is_upd)
existing_series_keys: Set[Tuple[str, str, str]] = set()
existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {}
if actualizar and not autonumerar:
q_ser = (
session.query(
InvoiceHeader.invoice_number,
LineItem.line_number,
Serie.row,
Serie.serial_numbers,
Serie.model,
Serie.sub_model,
Serie.number_id,
)
.join(LineItem, LineItem.invoice_id == InvoiceHeader.id)
.join(Serie, Serie.line_item_id == LineItem.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.operation_type == "imp",
InvoiceHeader.invoice_type == "TEM",
)
)
for num, ln, rw, sn, md, sm, nid in q_ser.all():
if num is not None:
k = (str(num).strip(), str(ln).strip(), str(rw).strip())
existing_series_keys.add(k)
existing_series_data.setdefault(k, {
"serial_numbers": sn or "",
"model": md or "",
"sub_model": sm or "",
"number_id": nid or "",
})
inserted_count = 0
updated_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, "imp_temp_series", normalize_header)
err = validate_row_series_impo_temp(
row_norm,
i,
actualizar=actualizar,
autonumerar=autonumerar,
validar_series_exception=validar_series_exception,
invoice_id_by_number=invoice_id_by_number,
invoice_processed_by_number=invoice_processed_by_number,
existing_series_keys=existing_series_keys,
existing_series_data=existing_series_data,
warnings=None,
)
if err:
skipped_invalid += 1
skipped_details.append({
"line": i,
"invoice": (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip(),
"reason": err.get("msg", ""),
})
continue
data = row_to_series_normalized(row_norm)
invoice_number = data["NUMERO FACTURA"]
linea_factura = data["LINEA FACTURA"]
linea_serie = data["LINEA SERIE"]
if not invoice_number or invoice_number not in invoice_id_by_number:
skipped_invalid += 1
continue
invoice_id = invoice_id_by_number[invoice_number]
line_number_val = parse_int(linea_factura)
if line_number_val is None:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."})
continue
line_item = (
session.query(LineItem)
.filter(
LineItem.invoice_id == invoice_id,
LineItem.line_number == line_number_val,
LineItem.tenant_id == tenant_id,
LineItem.company_id == company_id,
)
.first()
)
if not line_item:
skipped_invalid += 1
skipped_details.append({
"line": i,
"invoice": invoice_number,
"reason": f"Partida línea {linea_factura} no existe en la factura.",
})
continue
if autonumerar:
max_row = (
session.query(Serie.row)
.filter(Serie.line_item_id == line_item.id)
.order_by(Serie.row.desc())
.limit(1)
.scalar()
)
row_num = (max_row or 0) + 1
else:
row_num = parse_int(linea_serie)
if row_num is None:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."})
continue
existing_serie = (
session.query(Serie)
.filter(
Serie.line_item_id == line_item.id,
Serie.row == row_num,
)
.first()
)
if existing_serie:
if actualizar:
existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers
existing_serie.model = data["MODELO"] or existing_serie.model
existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model
existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id
session.add(existing_serie)
updated_count += 1
else:
skipped_invalid += 1
skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."})
else:
new_serie = Serie(
tenant_id=tenant_id,
company_id=company_id,
line_item_id=line_item.id,
row=row_num,
serial_numbers=data["SERIE"] or None,
model=data["MODELO"] or None,
sub_model=data["SUB MODELO"] or None,
number_id=data["NUMERO ID"] or None,
)
session.add(new_serie)
inserted_count += 1
session.commit()
common_storage.cleanup_import_job(effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path)
status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed")
out = {
"status": status,
"inserted": inserted_count,
"updated": updated_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_duplicate": 0,
"skipped_details": skipped_details,
}
if status == "failed":
out["error"] = "No hay registros válidos en el archivo CSV."
elif status == "warning" and skipped_invalid:
out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados."
return out
except Exception as e:
logger.exception("Series import commit failed: %s", e)
return {"status": "failed", "error": str(e)}
try:
from api.v1.modules.a76.invoices.models import (
InvoiceHeader,
InvoiceComplianceMx,
InvoiceFinancials,
InvoiceLogistics,
InvoiceSalesDetails,
InvoiceStatus,
OperationType,
TransportType,
WeightUnit,
)
from api.v1.modules.a76.clients_and_providers.models import ClientProvider
from api.v1.modules.a76.customs_brokers.models import CustomsBroker
from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos
from api.v1.modules.public.reference_data.currency_types.models import CurrencyType
from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode
from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen
from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento
from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType
from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection
from api.v1.modules.public.reference_data.incoterms.models import Incoterm
from .validators.encabezados_impo_temp import (
parse_pedimento_col_a,
_pedimento_key_from_parsed,
row_to_transport_type_clarion,
)
from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def
from api.v1.modules.a76.items.models import LineItem
from api.v1.modules.a76.items.line_financials.models import LineFinancial
from api.v1.modules.a76.items.line_quantities.models import LineQuantity
from api.v1.modules.a76.items.line_customs.models import LineCustom
from api.v1.modules.a76.items.line_descriptions.models import LineDescription
from api.v1.modules.a76.items.schemas import LineItemCreate
from api.v1.modules.a76.items.line_financials.schemas import LineFinancialCreate
from api.v1.modules.a76.items.line_quantities.schemas import LineQuantityCreate
from api.v1.modules.a76.items.line_customs.schemas import LineCustomCreate
from api.v1.modules.a76.items.line_descriptions.schemas import LineDescriptionCreate
from api.v1.modules.a24.fa.fa_item_lines.dto import FaLineItemCreateDTO
from api.v1.modules.a76.layouts_csv.facturas.line_item_enrichment import (
apply_import_defaults_and_calculations_for_csv,
apply_export_defaults_and_calculations_for_csv,
)
from api.v1.modules.a76.items.service import ItemService
from api.v1.modules.a76.parts.models import Part
from api.v1.modules.a76.classes.models import Class
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
from api.v1.modules.a76.general_catalogs.packages.models import Package
footer_config = parse_footer_config(meta.get("footer_config"))
date_format = footer_config.get("dateFormat")
# Validate and set default date_format if not provided
if not date_format:
date_format = "yyyy-mm-dd" # Default to ISO format
logger.info(f"No date_format specified in config, using default: {date_format}")
else:
logger.info(f"Using date_format from config: {date_format}")
# Default types from config or fallback
op_type_value = OperationType(meta.get('operation_type', 'imp').lower())
inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM'
_template_id_insert = meta.get("template_id") or (
"imp_temp_header" if model_target == "invoice_header" else "imp_temp_details"
)
if job_type_override == "exp" and model_target == "invoice_details":
_template_id_insert = "exp_def_partidas"
if job_type_override == "exp" and model_target == "invoice_series":
_template_id_insert = "exp_def_series"
if model_target == "invoice_header" and _template_id_insert == "imp_def_header":
inv_type_value = "DEF"
if model_target == "invoice_header" and _template_id_insert == "cmex_header":
inv_type_value = "MEX"
if model_target == "invoice_header" and _template_id_insert == "exp_def_header":
op_type_value = OperationType("exp")
inv_type_value = normalize_public_code(
meta.get("tipo_factura") or footer_config.get("tipo_factura") or "AFIJO"
) or "AFIJO"
_es_cambio_regimen = None
if model_target == "invoice_header" and _template_id_insert == "exp_def_header":
cambio_regimen_raw = (
str(meta.get("cambio_regimen") or footer_config.get("cambio_regimen") or "NO").strip().upper()
)
_es_cambio_regimen = "S" if cambio_regimen_raw == "SI" else "N"
if model_target == "invoice_details" and _template_id_insert == "imp_def_details":
inv_type_value = "DEF"
if model_target == "invoice_details" and _template_id_insert == "cmex_details":
inv_type_value = "MEX"
if model_target == "invoice_details" and _template_id_insert == "exp_def_partidas":
op_type_value = OperationType("exp")
inv_type_value = normalize_public_code(
meta.get("tipo_factura") or footer_config.get("tipo_factura") or "AFIJO"
) or "AFIJO"
if _template_id_insert == "cmex_series":
inv_type_value = "MEX"
logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}")
headers_to_insert = []
details_to_insert = []
skipped_invalid = 0
skipped_missing_invoice = 0
skipped_missing_fk = 0
skipped_fk_details = []
inserted_count = 0
response = None
with CoreSessionLocal() as session:
invoice_id_cache = {}
cleared_invoices = set() # Track invoices where we've already cleared items in this job
provider_cache: Dict[Any, Optional[int]] = {}
sold_to_cache: Dict[Any, Optional[int]] = {}
shipped_to_cache: Dict[Any, Optional[int]] = {}
broker_cache: Dict[Any, Optional[int]] = {}
regimen_cache: Dict[str, Optional[str]] = {}
currency_type_cache: Dict[str, Optional[str]] = {}
customs_section_cache: Dict[str, Optional[str]] = {}
part_cache: Dict[str, Optional[int]] = {}
pedimento_id_cache: Dict[str, Optional[int]] = {}
shipped_by_cache: Dict[Any, Optional[int]] = {}
_fc_insert = parse_footer_config(meta.get("footer_config"))
autonumerar_remesas_insert = _fc_insert.get("autonumerar_remesas", False)
class_id_by_code: Dict[str, int] = {}
class_uom_by_code: Dict[str, Optional[str]] = {}
class_fraction_by_code: Dict[str, Optional[str]] = {}
class_desc_es_by_code: Dict[str, Optional[str]] = {}
class_desc_en_by_code: Dict[str, Optional[str]] = {}
uom_id_by_code: Dict[str, int] = {}
package_id_by_key: Dict[str, int] = {}
if model_target == 'invoice_details':
for c in session.query(
Class.id,
Class.class_code,
Class.unit_of_measure,
Class.fraction,
Class.description_es,
Class.description_en,
).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all():
if c[1]:
class_code_key = (c[1] or "").strip().upper()
class_id_by_code[class_code_key] = c[0]
class_uom_by_code[class_code_key] = (c[2] or "").strip().upper() or None
class_fraction_by_code[class_code_key] = (c[3] or "").strip() or None
class_desc_es_by_code[class_code_key] = (c[4] or "").strip() or None
class_desc_en_by_code[class_code_key] = (c[5] or "").strip() or None
for u in session.query(UnitOfMeasure.id, UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all():
if u[1]:
uom_id_by_code[(u[1] or "").strip().upper()] = u[0]
for p in session.query(Package.id, Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all():
if p[1]:
package_id_by_key[(p[1] or "").strip()] = p[0]
validator = ForeignKeyValidator(session, tenant_id, company_id)
error_msg_by_line: Dict[int, str] = {}
if error_path and os.path.exists(error_path):
try:
with open(error_path, "r", encoding="utf-8") as f_err:
for line in f_err:
line = line.strip()
if not line:
continue
try:
rec = json.loads(line)
if "line" in rec and "msg" in rec:
error_msg_by_line[int(rec["line"])] = str(rec["msg"]).strip()
except (json.JSONDecodeError, ValueError, TypeError):
pass
except Exception:
pass
with open(file_path, 'r', encoding='utf-8-sig') as f:
# Detect Delimiter
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except:
dialect = 'excel'
reader = csv.DictReader(f, dialect=dialect)
template_id = _template_id_insert
for i, row in enumerate(reader, start=1):
row_norm = row_from_template(row, template_id, normalize_header)
if i in error_lines:
skipped_invalid += 1
if model_target == 'invoice_header':
inv_for_detail = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip()
elif _template_id_insert == "exp_def_partidas":
inv_for_detail = (row_norm.get('NUMERO FACTURA EXPO') or row_norm.get('NUMERO FACTURA EXPO.') or row_norm.get('FACTURA EXPO') or '').strip()
else:
inv_for_detail = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip()
reason = error_msg_by_line.get(i, "Línea marcada con error en el escaneo previo (revisar reporte de validación).")
skipped_fk_details.append({
"line": i,
"invoice": inv_for_detail or "(vacío)",
"reason": reason,
})
continue
# Mapping Logic (solo campos que acepta el modelo de facturas)
if model_target == 'invoice_header':
invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip()
invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format)
if not invoice_number or not invoice_date:
skipped_invalid += 1
reason = "Número de factura o fecha faltante/inválida"
skipped_fk_details.append({"line": i, "invoice": invoice_number or "(vacío)", "reason": reason})
logger.debug(f"Row {i}: Skipped - missing invoice_number or invalid invoice_date. "
f"Invoice: {invoice_number}, Date: {row_norm.get('FECHA FACTURA') or row_norm.get('FECHA')}")
continue
# --- NEW: Foreign Key Validations ---
# 1. Invoice Type (Public)
if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True):
skipped_missing_fk += 1
reason = f"Tipo de factura '{inv_type_value}' no existe"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
err = _validate_client_provider_ref(
validator,
ClientProvider,
row_norm.get('CLAVE PROVEEDOR'),
i,
"CLAVE PROVEEDOR",
required=True,
)
if err:
skipped_invalid += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
err = _validate_client_provider_ref(
validator,
ClientProvider,
row_norm.get('CLAVE VENDIDO A'),
i,
"CLAVE VENDIDO A",
required=True,
)
if err:
skipped_invalid += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
err = _validate_client_provider_ref(
validator,
ClientProvider,
row_norm.get('CLAVE ENVIADO A'),
i,
"CLAVE ENVIADO A",
required=True,
)
if err:
skipped_invalid += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
if inv_type_value != "MEX":
err = _validate_customs_broker_ref(
validator,
CustomsBroker,
row_norm.get('AGENTE ADUANAL'),
i,
"AGENTE ADUANAL",
required=False,
)
if err:
skipped_missing_fk += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
err = validate_public_code(
validator,
RegimenPedimento,
row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'),
i,
"CLAVEDOCUMENTO",
)
if err:
skipped_missing_fk += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
err = validate_public_code(
validator,
CustomsSection,
row_norm.get('ADUANA DE CRUCE'),
i,
"ADUANA DE CRUCE",
field_name="customs_code",
)
if err:
skipped_missing_fk += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
if inv_type_value == "MEX":
err = validate_public_code(
validator,
CurrencyType,
row_norm.get('CLAVE MONEDA'),
i,
"CLAVE MONEDA",
)
if err and (row_norm.get('TIPO MONEDA') or '').strip().upper() == 'MC':
skipped_missing_fk += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
else:
err = validate_public_code(
validator,
CurrencyType,
row_norm.get('CLAVE MONEDA'),
i,
"CLAVE MONEDA",
)
if err:
skipped_missing_fk += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
err = validate_public_code(
validator,
Incoterm,
row_norm.get('CLAVE INCOTERM'),
i,
"CLAVE INCOTERM",
)
if err:
skipped_missing_fk += 1
reason = f"{err['col']}: {err['msg']}"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
transport_type_val = row_norm.get('TIPO TRANSPORTE')
transport_str_normalized = (row_to_transport_type_clarion(transport_type_val) or str(transport_type_val or "").strip().lower() or "none")
if transport_type_val and transport_str_normalized not in TRANSPORT_TYPE_VALUES:
skipped_invalid += 1
reason = "TIPO TRANSPORTE: Tipo de transporte invalido"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
currency_val = row_norm.get('TIPO MONEDA')
if currency_val and str(currency_val).strip():
parsed_currency = parse_currency(currency_val, None)
val_norm = normalize_header(currency_val)
if parsed_currency.value == "manual" and "MANUAL" not in val_norm:
skipped_invalid += 1
reason = "TIPO MONEDA: Moneda invalida"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
# 2. Client/Provider and broker checks are handled above
# --- 4. Check for Existing Invoice (Upsert Logic) ---
existing_header = None
if invoice_number:
existing_header = (
session.query(InvoiceHeader)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.invoice_number == invoice_number,
InvoiceHeader.invoice_type == inv_type_value,
InvoiceHeader.operation_type == op_type_value,
)
.first()
)
# --- Resolve PEDIMENTO (Col A) to pedimento_id and REMESA (Col B) ---
pedimento_id = None
remesa_val = parse_int(row_norm.get('REMESA')) if inv_type_value != "MEX" else None
ped_str = (row_norm.get('PEDIMENTO') or '').strip() if inv_type_value != "MEX" else ''
if ped_str:
if template_id == "exp_def_header":
parsed = parse_pedimento_col_a_impo_def(ped_str)
else:
parsed = parse_pedimento_col_a(ped_str)
if parsed:
customs_office_p, license_p, num_p = (x.strip() if x else "" for x in parsed)
key_p = _pedimento_key_from_parsed(customs_office_p, license_p, num_p)
if key_p not in pedimento_id_cache:
co_prefix = (customs_office_p or "").strip()[:2].zfill(2)
ped_query = (
session.query(Pedimentos.id)
.filter(
Pedimentos.tenant_id == tenant_id,
Pedimentos.company_id == company_id,
Pedimentos.customs_office.startswith(co_prefix),
Pedimentos.license == license_p,
Pedimentos.pedimento_number == num_p,
)
)
if template_id == "exp_def_header":
if _es_cambio_regimen == "S":
ped_query = ped_query.filter(
func.lower(Pedimentos.operation_type) == "imp",
func.upper(Pedimentos.regime) == "IMD",
)
else:
ped_query = ped_query.filter(
func.lower(Pedimentos.operation_type) == "exp",
func.upper(Pedimentos.regime).in_(["EXD", "ETE", "ETR"]),
)
ped_row = ped_query.first()
pedimento_id_cache[key_p] = ped_row[0] if ped_row else None
pedimento_id = pedimento_id_cache[key_p]
if pedimento_id is not None and remesa_val is None and autonumerar_remesas_insert:
max_rem = (
session.query(func.max(InvoiceComplianceMx.remesa))
.filter(InvoiceComplianceMx.pedimento_id == pedimento_id)
.scalar()
)
remesa_val = (max_rem or 0) + 1
if existing_header:
# UPDATE existing header (paridad con InvoiceService.update)
header = existing_header
header.invoice_date = invoice_date
header.operation_type = op_type_value
# CSV import only captures data; "processed" is set by the manual/import processing flow.
# Legacy CSV imports may have persisted booleans ('True'/'False') into status.
if header.status not in (InvoiceStatus.PENDING, InvoiceStatus.PROCESSED, InvoiceStatus.REVERSED):
header.status = InvoiceStatus.PENDING
header.updated_date = datetime.utcnow()
capture_user = meta.get("capture_user") or "CSV"
header.who_processed = capture_user
# Backfill capture_user if missing or generic (paridad con service)
if not header.capture_user or header.capture_user == "System":
if capture_user != "CSV":
header.capture_user = capture_user
header.document_type = (
None if inv_type_value == "MEX" else
resolve_public_code(
session,
RegimenPedimento,
RegimenPedimento.code,
(row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')),
regimen_cache,
)
)
header.project_number = (row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None)
header.purchase_order = (row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None)
header.alternate_invoice = (row_norm.get('FACTURA ALTERNA') or None)
header.invoice_ref = (row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None)
header.emission_date = parse_date(row_norm.get('FECHA EMISION'), date_format)
header.observation_es = (row_norm.get('OBSERVACIONES E') or None)
header.observation_en = (row_norm.get('OBSERVACIONES I') or None)
logger.info(f"Row {i}: Updating existing invoice {invoice_number}")
# Clean up related data that will be re-inserted/updated
# Note: compliance, financials, logistics are 1-to-1 relationships and will be updated by assignment below
# but we might want to be explicit if ORM doesn't handle replace well.
# SQLAlchemy relationship assignment usually handles 1-to-1 updates correctly.
else:
# CREATE new header (paridad con InvoiceService.create: capture_user, who_processed)
capture_user = meta.get("capture_user") or "CSV"
header = InvoiceHeader(
invoice_number=invoice_number,
invoice_date=invoice_date,
operation_type=op_type_value,
status=InvoiceStatus.PENDING,
system="CSV",
capture_date=datetime.utcnow(),
capture_user=capture_user,
who_processed=capture_user,
invoice_type=inv_type_value,
document_type=(
None if inv_type_value == "MEX" else
resolve_public_code(
session,
RegimenPedimento,
RegimenPedimento.code,
(row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')),
regimen_cache,
)
),
project_number=(row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None),
purchase_order=(row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None),
alternate_invoice=(row_norm.get('FACTURA ALTERNA') or None),
invoice_ref=(row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None),
emission_date=parse_date(row_norm.get('FECHA EMISION'), date_format),
observation_es=(row_norm.get('OBSERVACIONES E') or None),
observation_en=(row_norm.get('OBSERVACIONES I') or None),
tenant_id=tenant_id,
company_id=company_id,
)
compliance = InvoiceComplianceMx(
pedimento_id=pedimento_id if inv_type_value != "MEX" else None,
remesa=remesa_val if inv_type_value != "MEX" else None,
aduana=(
None if inv_type_value == "MEX" else
resolve_public_code(
session,
CustomsSection,
CustomsSection.customs_code,
row_norm.get('ADUANA DE CRUCE'),
customs_section_cache,
)
),
provider_id=resolve_client_provider_id(
session,
ClientProvider,
row_norm.get('CLAVE PROVEEDOR'),
tenant_id,
company_id,
provider_cache,
),
sold_to_id=resolve_client_provider_id(
session,
ClientProvider,
row_norm.get('CLAVE VENDIDO A'),
tenant_id,
company_id,
sold_to_cache,
),
shipped_to_id=resolve_client_provider_id(
session,
ClientProvider,
row_norm.get('CLAVE ENVIADO A'),
tenant_id,
company_id,
shipped_to_cache,
),
customs_broker_id=(
None if inv_type_value == "MEX" else
resolve_customs_broker_id(
session,
CustomsBroker,
row_norm.get('AGENTE ADUANAL'),
tenant_id,
company_id,
broker_cache,
)
),
edocument=(row_norm.get('E DOCUMENT') or None),
vucem_operation_num=(row_norm.get('NUM OPERACION') or None),
manifest_number=(row_norm.get('MANIFIESTO') or None),
shipped_by_id=resolve_client_provider_id(
session,
ClientProvider,
row_norm.get('ENVIADO POR'),
tenant_id,
company_id,
shipped_by_cache,
) if row_norm.get('ENVIADO POR') else None,
tenant_id=tenant_id,
company_id=company_id,
)
financials_currency_type = resolve_public_code(
session,
CurrencyType,
CurrencyType.code,
row_norm.get('CLAVE MONEDA'),
currency_type_cache,
)
financials = InvoiceFinancials(
currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type),
currency_type=financials_currency_type,
exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')),
freight=decimal_or_zero(row_norm.get('FLETES')),
insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')),
insurance=decimal_or_zero(row_norm.get('SEGUROS')),
packaging=decimal_or_zero(row_norm.get('EMBALAJES')),
other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')),
tenant_id=tenant_id,
company_id=company_id,
)
weight_type = parse_weight_unit(row_norm.get('TIPO PESO'))
logistics = None
if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'):
raw_transport = row_norm.get('TIPO TRANSPORTE')
transport_str = (row_to_transport_type_clarion(raw_transport) or str(raw_transport or "").strip().lower() or "none")
try:
transport_type = TransportType(transport_str)
except ValueError:
transport_type = TransportType.NONE
logistics = InvoiceLogistics(
carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None),
driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None),
transport_type=transport_type,
transport_num=(row_norm.get('NUMERO TRANSPORTE') or None),
weight_type=weight_type or WeightUnit.KGS,
seal_number=(row_norm.get('PRECINTO') or None),
incoterm=(row_norm.get('CLAVE INCOTERM') or None),
entry_exit_date=parse_date(row_norm.get('FECHA EMISION'), date_format),
tenant_id=tenant_id,
company_id=company_id,
)
header.compliance_mx = compliance
header.financials = financials
if logistics:
header.logistics = logistics
headers_to_insert.append(header)
elif model_target == 'invoice_details':
if _template_id_insert == "exp_def_partidas":
invoice_number = (
row_norm.get('NUMERO FACTURA EXPO') or row_norm.get('NUMERO FACTURA EXPO.') or row_norm.get('FACTURA EXPO') or ''
).strip()
else:
invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or '').strip()
if not invoice_number:
skipped_invalid += 1
continue
if _template_id_insert == "exp_def_partidas":
# Expo: lookup by invoice_number + operation_type only (no invoice_type filter, matching scan behavior)
cache_key = f"{invoice_number}|exp"
if cache_key in invoice_id_cache:
invoice_id = invoice_id_cache[cache_key]
else:
invoice_id = (
session.query(InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.invoice_number == invoice_number,
InvoiceHeader.operation_type == "exp",
)
.scalar()
)
invoice_id_cache[cache_key] = invoice_id
else:
cache_key = f"{invoice_number}|{inv_type_value}|{op_type_value.value}"
if cache_key in invoice_id_cache:
invoice_id = invoice_id_cache[cache_key]
else:
invoice_id = (
session.query(InvoiceHeader.id)
.filter(
InvoiceHeader.tenant_id == tenant_id,
InvoiceHeader.company_id == company_id,
InvoiceHeader.invoice_number == invoice_number,
InvoiceHeader.invoice_type == inv_type_value,
InvoiceHeader.operation_type == op_type_value,
)
.scalar()
)
invoice_id_cache[cache_key] = invoice_id
if not invoice_id:
logger.warning(
"Invoice not found for details row %s (invoice_number=%s)",
i,
invoice_number,
)
skipped_missing_invoice += 1
continue
# --- Partidas Exportación Definitiva: paridad con flujo normal (validators + ItemService) ---
if _template_id_insert == "exp_def_partidas":
part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip()
part_id = part_cache.get(part_num) if part_num else None
if part_id is None and part_num:
p = session.query(Part.id).filter(Part.part_number == part_num, Part.tenant_id == tenant_id, Part.company_id == company_id).first()
if p:
part_id = p.id
part_cache[part_num] = p.id
line_num_val = (row_norm.get('LINEA EXPO') or row_norm.get('LINEA EXPO.') or row_norm.get('RENGLON EXPO'))
line_num = parse_int(line_num_val) or (len(details_to_insert) + 1)
uom_code = (row_norm.get('UNIDAD DE MEDIDA') or row_norm.get('U.M.') or row_norm.get('UNIDAD MEDIDA') or '').strip().upper()
uom_id = uom_id_by_code.get(uom_code) if uom_code else None
bulk_key = (row_norm.get('CLAVE BULTOS') or row_norm.get('CLAVEBULTOS') or '').strip()
package_id = package_id_by_key.get(bulk_key) if bulk_key else None
descarga_val = (row_norm.get('GENERA DESCARGA') or row_norm.get('GENERA DESCARGA?') or row_norm.get('DESCARGA') or 'SI').strip().upper()
tipo_impo = (row_norm.get('TIPO DE IMPO') or row_norm.get('TIPO DE IMPO.') or row_norm.get('TIPO IMPO') or row_norm.get('PROCEDENCIA') or '').strip().upper()
factura_impo = (row_norm.get('FACTURA IMPO') or row_norm.get('FACTURA IMPO.') or row_norm.get('FACTURA IMPORTACION') or '').strip()
linea_impo_val = (row_norm.get('LINEA IMPO') or row_norm.get('LINEA IMPO.') or row_norm.get('LINEA IMPORTACION') or '').strip()
se_pago = (row_norm.get('SE PAGO IMPUESTO') or row_norm.get('SE PAGO IMPUESTO? (SI o NO)') or row_norm.get('SEPAGOIMPUESTO') or '').strip().upper()
forma_pago = (row_norm.get('FORMA DE PAGO') or row_norm.get('FORMADEPAGO') or row_norm.get('FORMA PAGO') or '').strip() or None
es_sub_raw = (row_norm.get('ES PARTIDA/SUBPARTIDA') or row_norm.get('ESSUBPARTIDA') or row_norm.get('ES PARTIDA O SUBPARTIDA') or '').strip().upper()
linea_principal_val = (row_norm.get('LINEA PRINCIPAL') or row_norm.get('LINEAPRINCIPAL') or row_norm.get('PARTIDA PRINCIPAL') or '').strip()
is_subitem = (es_sub_raw == 'S')
contains_subitems = (es_sub_raw == 'P')
# Clear existing line items once per invoice
if invoice_id not in cleared_invoices:
logger.info(f"Clearing existing details for Expo Invoice {invoice_number} (ID: {invoice_id})")
session.query(LineItem).filter(LineItem.invoice_id == invoice_id).delete(synchronize_session=False)
session.query(InvoiceSalesDetails).filter(InvoiceSalesDetails.invoice_id == invoice_id).delete(synchronize_session=False)
cleared_invoices.add(invoice_id)
price = parse_decimal(row_norm.get('COSTO UNITARIO') or row_norm.get('COSTOUNITARIO'))
qty = parse_decimal(row_norm.get('CANTIDAD EXPORTADA/DESCARGAR') or row_norm.get('CANTIDAD EXPORTADA') or row_norm.get('CANTIDAD'))
commercial_total = (price * qty) if price and qty else None
net_w = parse_decimal(row_norm.get('PESO NETO') or row_norm.get('PESONETO'))
gross_w = parse_decimal(row_norm.get('PESO BRUTO') or row_norm.get('PESOBRUTO'))
origin = (row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') or row_norm.get('PAIS') or '').strip()
fraction = (row_norm.get('FRACCION ARANCELARIA') or row_norm.get('FRACCIONARANCELARIA') or '').strip()
american_fraction = (row_norm.get('FRACCION AMERICANA') or row_norm.get('FRACCIONAMERICANA') or '').strip()
extra_desc = (row_norm.get('DESCRIPCION EXTRA') or row_norm.get('DESCRIPCIONEXTRA') or '').strip()
additional_info = (row_norm.get('INFORMACION ADICIONAL') or row_norm.get('INFORMACIONADICIONAL') or '').strip()
lot = (row_norm.get('LOTE') or '').strip()
entry_number = (row_norm.get('NUMERO ENTRADA') or row_norm.get('NUM ENTRADA') or '').strip()
order_compra = (row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or row_norm.get('ORDEN DE VENTA') or None)
line_data = LineItemCreate(
invoice_id=invoice_id,
line_number=line_num,
part_number_id=part_id,
class_id=None,
unit_of_measure=uom_id,
order=order_compra,
tax_payment=(se_pago == 'SI'),
payment_method=forma_pago,
financial=LineFinancialCreate(
unit_cost_capture=decimal_or_zero(price),
total_commercial_value=decimal_or_zero(commercial_total),
),
quantity=LineQuantityCreate(
quantity=decimal_or_zero(qty),
net_weight=decimal_or_zero(net_w),
gross_weight=decimal_or_zero(gross_w),
package_quantity=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')),
package_id=package_id,
),
customs=LineCustomCreate(
origin_country=origin or None,
fraction=fraction or None,
american_fraction=american_fraction or None,
),
description=LineDescriptionCreate(
extra_description=extra_desc or None,
additional_info_spanish=additional_info or None,
lot=lot or None,
entry_number=entry_number or None,
),
fa_data=FaLineItemCreateDTO(
search_invoice=factura_impo or None,
search_line=parse_int(linea_impo_val),
search_type=tipo_impo or None,
download=(descarga_val == 'SI'),
is_subitem=is_subitem,
contains_subitems=contains_subitems,
subitem_number=parse_int(linea_principal_val) if is_subitem else 0,
),
)
if not apply_export_defaults_and_calculations_for_csv(
session, line_data, tenant_id, company_id, line_num
):
skipped_invalid += 1
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": "Factura sin datos financieros para enriquecer partida de exportación."})
continue
item_dict = line_data.model_dump(
exclude={"financial", "quantity", "customs", "description", "reference", "fa_data", "series"}
)
item_dict["tenant_id"] = tenant_id
item_dict["company_id"] = company_id
item_dict["line_number"] = line_num
line = LineItem(**item_dict)
session.add(line)
session.flush()
ItemService._create_line_nested_data(session, line, line_data, tenant_id, company_id)
session.add(InvoiceSalesDetails(
invoice_id=invoice_id,
line_number=line_num,
sales_order=order_compra,
line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')),
tenant_id=tenant_id,
company_id=company_id,
))
details_to_insert.append(line)
continue
part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip()
if not part_num:
skipped_invalid += 1
reason = "NUMPARTE: Requerido"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
if not validator.check_exists(Part, part_num, field_name="part_number"):
skipped_missing_fk += 1
reason = f"NUMPARTE '{part_num}' no existe"
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason})
logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}")
continue
# --- Prevent Duplicates: Clear existing line items for this invoice (Once per job) ---
if invoice_id not in cleared_invoices:
logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates")
session.query(LineItem).filter(LineItem.invoice_id == invoice_id).delete(synchronize_session=False)
session.query(InvoiceSalesDetails).filter(InvoiceSalesDetails.invoice_id == invoice_id).delete(synchronize_session=False)
cleared_invoices.add(invoice_id)
# --- Partidas importación: paridad con flujo normal (validators + ItemService) ---
# LineReference: solo se crea si line_data.reference viene informado; no inventar datos sin fuente (plan paridad CSV).
# Build LineItemCreate from CSV, apply import defaults/calculations, then persist.
part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip()
part_id = part_cache.get(part_num) if part_num else None
if part_id is None and part_num:
p = session.query(Part.id).filter(Part.part_number == part_num, Part.tenant_id == tenant_id, Part.company_id == company_id).first()
if p:
part_id = p.id
part_cache[part_num] = part_id
line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA'))
line_num = parse_int(line_num_val) or (len(details_to_insert) + 1)
class_code = (row_norm.get('CLASE') or '').strip().upper()
class_id = class_id_by_code.get(class_code) if class_code else None
uom_code = (row_norm.get('UNIDAD DE MEDIDA') or row_norm.get('UNIDAD MEDIDA') or '').strip().upper()
if not uom_code and class_code:
uom_code = class_uom_by_code.get(class_code) or ''
uom_id = uom_id_by_code.get(uom_code) if uom_code else None
bulk_key = (row_norm.get('CLAVE BULTOS') or row_norm.get('CLAVEBULTOS') or '').strip()
package_id = package_id_by_key.get(bulk_key) if bulk_key else None
price = parse_decimal(row_norm.get('COSTO UNITARIO') or row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO'))
if price is None:
total_val = parse_decimal(row_norm.get('TOTAL'))
qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD'))
price = (total_val / qty) if (total_val and qty and qty != 0) else None
qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD'))
commercial_total = (price * qty) if price and qty else parse_decimal(row_norm.get('TOTAL'))
net_w = parse_decimal(row_norm.get('PESO NETO') or row_norm.get('PESONETO'))
gross_w = parse_decimal(row_norm.get('PESO BRUTO') or row_norm.get('PESOBRUTO'))
origin = (row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') or row_norm.get('PAIS') or '').strip()
fraction = (row_norm.get('FRACCION ARANCELARIA') or row_norm.get('FRACCION') or row_norm.get('FRACCIONARANCELARIA') or '').strip()
if not fraction and class_code:
fraction = class_fraction_by_code.get(class_code) or ''
fraction_type = (row_norm.get('PREFERENCIA ARANCELARIA') or row_norm.get('PREFERENCIA') or '').strip()
sector = (row_norm.get('SECTOR') or '').strip()
american_fraction = (row_norm.get('FRACCION AMERICANA') or row_norm.get('FRACCIONAMERICANA') or '').strip()
desc_es = (row_norm.get('DESCRIPCION ESPAÑOL') or row_norm.get('DESCRIPCIONE') or row_norm.get('DESCRIPCION') or '').strip()
if not desc_es and class_code:
desc_es = class_desc_es_by_code.get(class_code) or ''
desc_en = (row_norm.get('DESCRIPCION INGLES') or row_norm.get('DESCRIPCIONI') or '').strip()
if not desc_en and class_code:
desc_en = class_desc_en_by_code.get(class_code) or ''
brand = (row_norm.get('MARCA') or '').strip()
model = (row_norm.get('MODELO') or '').strip()
extra_desc = (row_norm.get('DESCRIPCION EXTRA') or row_norm.get('DESCRIPCIONEXTRA') or '').strip()
additional_info = (row_norm.get('INFORMACION ADICIONAL') or row_norm.get('INFORMACIONADICIONAL') or '').strip()
lot = (row_norm.get('LOTE') or '').strip()
entry_number = (row_norm.get('NUMERO ENTRADA') or row_norm.get('NUMEROENTRADA') or row_norm.get('NUM ENTRADA') or '').strip()
line_data = LineItemCreate(
invoice_id=invoice_id,
line_number=line_num,
part_number_id=part_id,
class_id=class_id,
unit_of_measure=uom_id,
order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None),
material_type=(row_norm.get('ID TYPE') or row_norm.get('IDTYPE') or None),
tax_payment=(str(row_norm.get('SE PAGO IMPUESTO') or row_norm.get('SEPAGOIMPUESTO') or '').strip().upper() == 'SI'),
payment_method=(row_norm.get('FORMA DE PAGO') or row_norm.get('FORMADEPAGO') or row_norm.get('FORMA PAGO') or None),
valuation_method=(row_norm.get('METODO DE VALORACION') or row_norm.get('METODODEVALORACION') or row_norm.get('METODO VALORACION') or None),
financial=LineFinancialCreate(
unit_cost_capture=decimal_or_zero(price),
total_commercial_value=decimal_or_zero(commercial_total),
),
quantity=LineQuantityCreate(
quantity=decimal_or_zero(qty),
net_weight=decimal_or_zero(net_w),
gross_weight=decimal_or_zero(gross_w),
package_quantity=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')),
package_id=package_id,
),
customs=LineCustomCreate(
origin_country=origin or None,
fraction=fraction or None,
fraction_type=fraction_type or None,
sector=sector or None,
american_fraction=american_fraction or None,
),
description=LineDescriptionCreate(
description_spanish=desc_es or None,
description_english=desc_en or None,
brand=brand or None,
model=model or None,
extra_description=extra_desc or None,
additional_info_spanish=additional_info or None,
lot=lot or None,
entry_number=entry_number or None,
),
fa_data=FaLineItemCreateDTO(is_subitem=False, contains_subitems=False),
)
if not apply_import_defaults_and_calculations_for_csv(
session, line_data, tenant_id, company_id, line_num
):
skipped_invalid += 1
skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": "Factura sin datos financieros/logísticos para enriquecer partida."})
continue
item_dict = line_data.model_dump(
exclude={"financial", "quantity", "customs", "description", "reference", "fa_data", "series"}
)
item_dict["tenant_id"] = tenant_id
item_dict["company_id"] = company_id
item_dict["line_number"] = line_num
line = LineItem(**item_dict)
session.add(line)
session.flush()
ItemService._create_line_nested_data(session, line, line_data, tenant_id, company_id)
session.add(InvoiceSalesDetails(
invoice_id=invoice_id,
line_number=line_num,
sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None),
line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')),
tenant_id=tenant_id,
company_id=company_id,
))
details_to_insert.append(line)
# 3. Bulk Insert (ORM Transaction)
try:
if model_target == 'invoice_header':
if headers_to_insert:
logger.info(f"Attempting to commit {len(headers_to_insert)} headers")
session.add_all(headers_to_insert)
session.commit()
inserted_count = len(headers_to_insert)
logger.info(f"Headers commit successful. Inserted: {inserted_count}")
else:
logger.warning(f"No headers to insert for job {job_id}")
else:
if details_to_insert:
logger.info(f"Attempting to commit {len(details_to_insert)} items and related data")
session.commit() # Everything was already added with session.add()
inserted_count = len(details_to_insert)
logger.info(f"Details commit successful. Inserted: {inserted_count}")
else:
logger.warning(f"No details to insert for job {job_id}")
except Exception as db_err:
session.rollback()
logger.error(f"DB Error during {model_target} commit: {db_err}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(db_err)}
# 4. Determine final status and prepare response (inside session block to access variables)
total_skipped = skipped_invalid + skipped_missing_fk + skipped_missing_invoice
# Log summary
logger.info(f"Job {job_id} completed. Inserted: {inserted_count}, Skipped: {total_skipped} "
f"(invalid: {skipped_invalid}, missing_fk: {skipped_missing_fk}, missing_invoice: {skipped_missing_invoice})")
# Prepare response based on results
if inserted_count == 0:
if total_skipped > 0:
logger.warning(f"No valid records to insert for job {job_id}. All {total_skipped} records were rejected.")
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_invoice": skipped_missing_invoice,
"skipped_missing_fk": skipped_missing_fk,
"skipped_details": skipped_fk_details,
"message": f"No se insertaron registros. {total_skipped} fueron rechazados. Revisa el detalle por línea a continuación.",
}
else:
logger.error(f"No valid records found in CSV for job {job_id}")
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_invoice": skipped_missing_invoice,
"skipped_missing_fk": skipped_missing_fk,
"skipped_details": skipped_fk_details
}
else:
# Success case - at least some records were inserted
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_invoice": skipped_missing_invoice,
"skipped_missing_fk": skipped_missing_fk,
"skipped_details": skipped_fk_details
}
except Exception as e:
logger.error(f"Task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
# 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely
try:
common_storage.cleanup_import_job(
effective_job_type, job_id,
file_path=file_path,
error_path=error_path,
meta_path=meta_path,
)
except Exception as cleanup_err:
logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err)
# Ensure response is defined (fallback in case of unexpected errors)
if response is None:
logger.error(f"Unexpected error: response not set for job {job_id}")
response = {
"status": "failed",
"error": "Error inesperado durante el procesamiento",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_invoice": skipped_missing_invoice,
"skipped_missing_fk": skipped_missing_fk,
"skipped_details": skipped_fk_details
}
return response
Reference in New Issue View Git Blame Copy Permalink