""" Tareas Celery para importación CSV de facturas (encabezados, partidas, series). Flujo: scan_file (validación) → insert_valid_rows (commit). Objetivo en BD (paridad con flujo normal): al terminar el commit, los datos deben quedar igual que por UI/API: encabezados con capture_user/who_processed; partidas con costos, pesos y descripciones calculados/heredados según items/imports/validators; series con campos no presentes en CSV en null. No se modifican plantillas CSV; no se inventan datos sin fuente (p. ej. LineReference solo si hay fuente explícita). """ import os from datetime import datetime from decimal import Decimal import csv import json import logging import re import unicodedata from typing import Dict, Any, Optional, List, Set, Tuple from core.celery_app import celery_app from core.database import CoreSessionLocal from core.paths import layout_path from sqlalchemy import func from ..common import storage as common_storage from ..common import meta as common_meta from ..common import responses as common_responses from .template_config import row_from_template # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process logger = logging.getLogger(__name__) # Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py) JOB_TYPE = "" # Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="") IMPORT_FILE_KEY_PREFIX = "import_file:" IMPORT_META_KEY_PREFIX = "import_meta:" IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: """Usa common storage con job_type vacío (prefijo import_).""" return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import") def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import") def _delete_import_from_redis(job_id: str) -> None: common_storage.delete_import_from_redis(JOB_TYPE, job_id) class ForeignKeyValidator: def __init__(self, session, tenant_id, company_id): self.session = session self.tenant_id = tenant_id self.company_id = company_id self.cache = {} # {(model_name, value): bool} def check_exists(self, model, value, field_name="id", is_public=False): if value is None: return True # Assume optional if None, or let DB handle not-null key = (model.__name__, value) if key in self.cache: return self.cache[key] col = getattr(model, field_name) if field_name == "short_name" and hasattr(model, "short_name"): query = self.session.query(col).filter(func.upper(col) == (value.upper() if isinstance(value, str) else value)) else: query = self.session.query(col).filter(col == value) if not is_public: query = query.filter(model.tenant_id == self.tenant_id, model.company_id == self.company_id) exists = query.first() is not None self.cache[key] = exists return exists TRANSPORT_TYPE_VALUES = { "none", "transport", "box", "licence plates", "truck", "vessel", "rail_barge", "container", "airplane", "gondola", "flatbed", } def normalize_public_code(value: Optional[str]) -> Optional[str]: if value is None: return None text = str(value).strip().upper() return text or None def validate_public_code( validator: ForeignKeyValidator, model, value: Optional[str], line_num: int, col_name: str, field_name: str = "code", required: bool = False, ) -> Optional[Dict[str, Any]]: code = normalize_public_code(value) if not code: if required: return {"line": line_num, "col": col_name, "msg": "Requerido"} return None if not validator.check_exists(model, code, field_name=field_name, is_public=True): return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} return None def validate_tenant_fk_id( validator: ForeignKeyValidator, model, value: Optional[int], line_num: int, col_name: str, required: bool = False, ) -> Optional[Dict[str, Any]]: if value is None: if required: return {"line": line_num, "col": col_name, "msg": "Requerido"} return None if not validator.check_exists(model, value): return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} return None def _validate_client_provider_ref( validator: ForeignKeyValidator, model, raw_value: Any, line_num: int, col_name: str, required: bool, ) -> Optional[Dict[str, Any]]: """Valida CLAVE PROVEEDOR / VENDIDO A / ENVIADO A: acepta ID (entero) o short_name (texto).""" if raw_value is None or not str(raw_value).strip(): if required: return {"line": line_num, "col": col_name, "msg": "Requerido"} return None pid = parse_int(raw_value) if pid is not None: return validate_tenant_fk_id(validator, model, pid, line_num, col_name, required=False) short_norm = str(raw_value).strip().upper() if not validator.check_exists(model, short_norm, field_name="short_name"): return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} return None def _validate_customs_broker_ref( validator: ForeignKeyValidator, model, raw_value: Any, line_num: int, col_name: str, required: bool = False, ) -> Optional[Dict[str, Any]]: """Valida AGENTE ADUANAL: acepta ID (entero) o clave broker_key (texto).""" if raw_value is None or not str(raw_value).strip(): if required: return {"line": line_num, "col": col_name, "msg": "Requerido"} return None pid = parse_int(raw_value) if pid is not None: return validate_tenant_fk_id(validator, model, pid, line_num, col_name, required=False) clave = str(raw_value).strip() if not validator.check_exists(model, clave, field_name="broker_key"): return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} return None @celery_app.task(bind=True) def scan_file(self, job_id: str, model_target: str, config: str = None, job_type_override: Optional[str] = None): """Pass 1: Read CSV, Validate types, Write Errors to JSONL. Delegates to _do_scan_file.""" return _do_scan_file(self, job_id, model_target, config, job_type_override) def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = None, job_type_override: Optional[str] = None) -> Dict[str, Any]: """Pass 1 body: load file/meta from storage, run validations, store error lines. Uses effective_job_type for storage.""" effective_job_type = job_type_override if job_type_override is not None else JOB_TYPE log_prefix = "Exportación import" if effective_job_type else "Invoices import" logger.info(f"Starting scan for job {job_id} target {model_target}") # 1. Get file from Redis and write to worker local disk file_path = common_storage.ensure_file_from_redis(effective_job_type, job_id, log_prefix) if not file_path: return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} common_storage.ensure_meta_from_redis(effective_job_type, job_id, file_path, log_prefix) error_path = common_storage.error_path_for_job(effective_job_type, job_id) total_rows = 0 error_count = 0 processed_rows = 0 # 3. Count Total (Quick Pass) or just estimate try: with open(file_path, 'r', encoding='utf-8-sig') as f: total_rows = sum(1 for _ in f) - 1 # Minus header except Exception as e: return {"status": "failed", "error": f"Cannot read file: {e}"} footer_config = parse_footer_config(config) date_format = footer_config.get("dateFormat") # Validate and set default date_format if not provided if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) template_id = meta.get("template_id") or ( "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" if model_target == "invoice_details" else "imp_temp_series" ) # Cuando el scan viene de Exportación (job_type_override "exp"), forzar exp_def_header o exp_def_partidas if job_type_override == "exp" and model_target == "invoice_header": template_id = "exp_def_header" if job_type_override == "exp" and model_target == "invoice_details": template_id = "exp_def_partidas" if job_type_override == "exp" and model_target == "invoice_series": template_id = "exp_def_series" inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM") if not inv_type_value: inv_type_value = "TEM" if model_target == "invoice_series" and inv_type_value in ("DEF", "MATDE", "EXDEF"): template_id = "imp_def_series" if meta.get("operation_type") == "exp" and model_target == "invoice_details": template_id = "exp_def_partidas" logger.info( "Scan job %s template_id=%s model_target=%s job_type_override=%s", job_id, template_id, model_target, job_type_override, ) # --- Series de Importación Definitiva: flujo específico (Clarion VALIDA_TODA_SERIES_IMPO_DEF / VALIDA_PARCIAL) --- DEF_SERIES_TEMPLATE_OR_TYPE = ( model_target == "invoice_series" and ( template_id == "imp_def_series" or (inv_type_value in ("DEF", "MATDE", "EXDEF")) ) ) if DEF_SERIES_TEMPLATE_OR_TYPE: try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from api.v1.modules.a76.items.line_quantities.models import LineQuantity from .validators.series_impo_def import ( validate_row_series_impo_def, ) DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") actualizar = meta.get("actualizar", False) autonumerar = meta.get("autonumerar", True) validar_series_exception = meta.get("validar_series", False) _fc = parse_footer_config(meta.get("footer_config")) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) elif _fc.get("mode") == "update": actualizar = True elif _fc.get("mode") == "replace": actualizar = False if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) else: as_val = _fc.get("autonumber_series", "true") autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") if "validar_series" in _fc: validar_series_exception = bool(_fc["validar_series"]) with CoreSessionLocal() as session: q = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) rows_inv = q.all() invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in rows_inv: if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) partida_max_series: Dict[Tuple[str, str], int] = {} q_qty = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, LineQuantity.quantity, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) for num, ln, qty in q_qty.all(): if num is not None and ln is not None: key = (str(num).strip(), str(ln).strip()) if qty is not None: partida_max_series[key] = int(qty) if qty else 0 else: partida_max_series[key] = 0 existing_series_keys: Set[Tuple[str, str, str]] = set() existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} if actualizar and not autonumerar: q_ser = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, Serie.row, Serie.serial_numbers, Serie.model, Serie.sub_model, Serie.number_id, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(Serie, Serie.line_item_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) for num, ln, rw, sn, md, sm, nid in q_ser.all(): if num is not None: k = (str(num).strip(), str(ln).strip(), str(rw).strip()) existing_series_keys.add(k) existing_series_data.setdefault(k, { "serial_numbers": sn or "", "model": md or "", "sub_model": sm or "", "number_id": nid or "", }) csv_series_count_so_far: Dict[Tuple[str, str], int] = {} with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) errors_detail = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "imp_def_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] err = validate_row_series_impo_def( row_norm, i, actualizar=actualizar, autonumerar=autonumerar, validar_series_exception=validar_series_exception, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, partida_max_series=partida_max_series, csv_series_count_so_far=csv_series_count_so_far, existing_series_keys=existing_series_keys, existing_series_data=existing_series_data, warnings=warnings_list, ) if err and not err.get("warning"): error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) else: inv_num = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip() line_fac = (row_norm.get("LINEA FACTURA") or row_norm.get("LINEA") or row_norm.get("PARTIDA") or "").strip() if inv_num and line_fac: key_csv = (inv_num, line_fac) csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1 for w in warnings_list: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Series importación definitiva scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Series de Exportación Definitiva: flujo exp_def_series (Clarion VALIDA_TODA_SERIES_EXPO / VALIDA_PARCIAL) --- if model_target == "invoice_series" and template_id == "exp_def_series": logger.info("Series expo scan: running validation for job %s", job_id) try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from api.v1.modules.a76.items.line_quantities.models import LineQuantity from api.v1.modules.a76.general_catalogs.company.models import Company from .validators.series_expo import validate_row_series_expo _fc = parse_footer_config(meta.get("footer_config")) autonumerar = meta.get("autonumerar", True) actualizar = meta.get("actualizar", False) validar_series_exception = meta.get("validar_series", False) if _fc: if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) elif _fc.get("autonumber_series", "true") is not None: autonumerar = str(_fc.get("autonumber_series", "true")).lower() in ("true", "1", "si", "sí", "yes") if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "validar_series" in _fc: validar_series_exception = bool(_fc["validar_series"]) RFC_EXCEPTION_EGM = {"EGM0303257J1"} with CoreSessionLocal() as session: company = session.query(Company).filter(Company.id == company_id).first() company_rfc = (company.rfc or "").strip().upper() if company else "" q_inv_expo = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in q_inv_expo.all(): if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) partida_max_series: Dict[Tuple[str, str], int] = {} q_qty = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, LineQuantity.quantity, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) for num, ln, qty in q_qty.all(): if num is not None and ln is not None: key = (str(num).strip(), str(ln).strip()) if qty is not None: partida_max_series[key] = int(qty) if qty else 0 else: partida_max_series[key] = 0 existing_series_keys: Set[Tuple[str, str, str]] = set() existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} if actualizar and not autonumerar: q_ser = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, Serie.row, Serie.serial_numbers, Serie.model, Serie.sub_model, Serie.number_id, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(Serie, Serie.line_item_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) for num, ln, rw, sn, md, sm, nid in q_ser.all(): if num is not None: k = (str(num).strip(), str(ln).strip(), str(rw).strip()) existing_series_keys.add(k) existing_series_data.setdefault(k, { "serial_numbers": sn or "", "model": md or "", "sub_model": sm or "", "number_id": nid or "", }) csv_series_count_so_far: Dict[Tuple[str, str], int] = {} invoice_numbers_from_csv: Set[str] = set() with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) for row in reader: row_norm = row_from_template(row, "exp_def_series", normalize_header) inv = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA EXPO") or "").strip() if inv: invoice_numbers_from_csv.add(inv) if company_rfc in RFC_EXCEPTION_EGM: rfc_exception_updated: Set[str] = invoice_numbers_from_csv else: rfc_exception_updated = set() with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err: f_in.seek(0) try: dialect = csv.Sniffer().sniff(f_in.read(2048), delimiters=",;\t") except Exception: dialect = "excel" f_in.seek(0) reader = csv.DictReader(f_in, dialect=dialect) errors_detail = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "exp_def_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] err = validate_row_series_expo( row_norm, i, actualizar=actualizar, autonumerar=autonumerar, validar_series_exception=validar_series_exception, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, rfc_exception_updated=rfc_exception_updated, partida_max_series=partida_max_series, csv_series_count_so_far=csv_series_count_so_far, existing_series_keys=existing_series_keys, existing_series_data=existing_series_data, warnings=warnings_list, ) if err and not err.get("warning"): error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) else: inv_num = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA EXPO") or "").strip() line_fac = (row_norm.get("LINEA FACTURA") or row_norm.get("LINEA") or row_norm.get("PARTIDA") or "").strip() if inv_num and line_fac: key_csv = (inv_num, line_fac) csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1 for w in warnings_list: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Series exportación definitiva scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Series Compras Mexicanas: misma lógica que Impo Def, facturas MEX --- if model_target == "invoice_series" and template_id == "cmex_series": try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from api.v1.modules.a76.items.line_quantities.models import LineQuantity from .validators.series_impo_def import validate_row_series_impo_def actualizar = meta.get("actualizar", False) autonumerar = meta.get("autonumerar", True) validar_series_exception = meta.get("validar_series", False) _fc = parse_footer_config(meta.get("footer_config")) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) elif _fc.get("mode") == "update": actualizar = True elif _fc.get("mode") == "replace": actualizar = False if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) else: as_val = _fc.get("autonumber_series", "true") autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") if "validar_series" in _fc: validar_series_exception = bool(_fc["validar_series"]) with CoreSessionLocal() as session: q = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) rows_inv = q.all() invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in rows_inv: if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) partida_max_series: Dict[Tuple[str, str], int] = {} q_qty = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, LineQuantity.quantity, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) for num, ln, qty in q_qty.all(): if num is not None and ln is not None: key = (str(num).strip(), str(ln).strip()) if qty is not None: partida_max_series[key] = int(qty) if qty else 0 else: partida_max_series[key] = 0 existing_series_keys: Set[Tuple[str, str, str]] = set() existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} if actualizar and not autonumerar: q_ser = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, Serie.row, Serie.serial_numbers, Serie.model, Serie.sub_model, Serie.number_id, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(Serie, Serie.line_item_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) for num, ln, rw, sn, md, sm, nid in q_ser.all(): if num is not None: k = (str(num).strip(), str(ln).strip(), str(rw).strip()) existing_series_keys.add(k) existing_series_data.setdefault(k, { "serial_numbers": sn or "", "model": md or "", "sub_model": sm or "", "number_id": nid or "", }) csv_series_count_so_far: Dict[Tuple[str, str], int] = {} with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) errors_detail = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "cmex_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] err = validate_row_series_impo_def( row_norm, i, actualizar=actualizar, autonumerar=autonumerar, validar_series_exception=validar_series_exception, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, partida_max_series=partida_max_series, csv_series_count_so_far=csv_series_count_so_far, existing_series_keys=existing_series_keys, existing_series_data=existing_series_data, warnings=warnings_list, catalog_label="Compras Mexicanas", ) if err and not err.get("warning"): error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) else: inv_num = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip() line_fac = (row_norm.get("LINEA FACTURA") or row_norm.get("LINEA") or row_norm.get("PARTIDA") or "").strip() if inv_num and line_fac: key_csv = (inv_num, line_fac) csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1 for w in warnings_list: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Series Compras Mexicanas scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Series de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- if model_target == "invoice_series": try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from .validators.series_impo_temp import ( validate_row_series_impo_temp, ) actualizar = meta.get("actualizar", False) autonumerar = meta.get("autonumerar", True) validar_series_exception = meta.get("validar_series", False) _fc = parse_footer_config(meta.get("footer_config")) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) elif _fc.get("mode") == "update": actualizar = True elif _fc.get("mode") == "replace": actualizar = False if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) else: as_val = _fc.get("autonumber_series", "true") autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") if "validar_series" in _fc: validar_series_exception = bool(_fc["validar_series"]) with CoreSessionLocal() as session: # Invoice lookup: imp + TEM q = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) rows_inv = q.all() invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in rows_inv: if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) # Existing series keys: (invoice_number, linea_factura, linea_serie) existing_series_keys: Set[Tuple[str, str, str]] = set() existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} if actualizar and not autonumerar: q_ser = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, Serie.row, Serie.serial_numbers, Serie.model, Serie.sub_model, Serie.number_id, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(Serie, Serie.line_item_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) for num, ln, rw, sn, md, sm, nid in q_ser.all(): if num is not None: k = (str(num).strip(), str(ln).strip(), str(rw).strip()) existing_series_keys.add(k) existing_series_data.setdefault(k, { "serial_numbers": sn or "", "model": md or "", "sub_model": sm or "", "number_id": nid or "", }) with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "imp_temp_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] err = validate_row_series_impo_temp( row_norm, i, actualizar=actualizar, autonumerar=autonumerar, validar_series_exception=validar_series_exception, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, existing_series_keys=existing_series_keys, existing_series_data=existing_series_data, warnings=warnings_list, ) if err and not err.get("warning"): error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) for w in warnings_list: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Series import scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Partidas de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- if model_target == "invoice_details" and template_id == "imp_temp_details": try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.classes.models import Class from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure from api.v1.modules.a76.general_catalogs.packages.models import Package from api.v1.modules.public.reference_data.countries.models import Country from api.v1.modules.a76.general_catalogs.sectors.models import Sector from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction from api.v1.modules.a76.general_catalogs.company.models import Company from api.v1.modules.a76.parts.models import Part from .validators.partidas_impo_temp import validate_row_partidas_impo_temp _fc = parse_footer_config(meta.get("footer_config")) autonumerar = meta.get("autonumerar", True) actualizar = meta.get("actualizar", False) levantar_subpartidas = meta.get("levantar_subpartidas", False) calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False) validar_decimales_pza = meta.get("validar_decimales_pza", False) if _fc: if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) elif _fc.get("autonumber_partidas", "true") is not None: autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes") if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "levantar_subpartidas" in _fc: levantar_subpartidas = bool(_fc["levantar_subpartidas"]) if "calcular_costo_unitario_en_base_a_valor_total" in _fc: calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"]) if "validar_decimales_pza" in _fc: validar_decimales_pza = bool(_fc["validar_decimales_pza"]) RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"} RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"} with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in q_inv.all(): if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) existing_line_keys_by_invoice: Dict[str, Set[str]] = {} q_li = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) for num, ln in q_li.all(): if num is not None: key = str(num).strip() if key not in existing_line_keys_by_invoice: existing_line_keys_by_invoice[key] = set() existing_line_keys_by_invoice[key].add(str(ln).strip()) partidas_principales_bd: Set[Tuple[str, str]] = set() try: from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem q_pp = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(FaLineItem, FaLineItem.id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, FaLineItem.is_subitem == False, FaLineItem.contains_subitems == True, ) ) for num, ln in q_pp.all(): if num is not None: partidas_principales_bd.add((str(num).strip(), str(ln).strip())) except Exception: pass valid_class_codes: Set[str] = set() class_um_by_code: Dict[str, str] = {} class_fraction_by_code: Dict[str, str] = {} class_desc_es_by_code: Dict[str, str] = {} class_desc_en_by_code: Dict[str, str] = {} for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): code = (c.class_code or "").strip().upper() if code: valid_class_codes.add(code) class_um_by_code[code] = (c.unit_of_measure or "").strip().upper() class_fraction_by_code[code] = (c.fraction or "").strip() class_desc_es_by_code[code] = (c.description_es or "").strip() class_desc_en_by_code[code] = (c.description_en or "").strip() valid_uom_codes: Set[str] = set() for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): if u[0]: valid_uom_codes.add((u[0] or "").strip().upper()) valid_bulks_codes: Set[str] = set() for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): if p[0]: valid_bulks_codes.add((p[0] or "").strip()) valid_country_keys: Set[str] = set() for row in session.query(Country.m3_key, Country.ame_key).all(): if row[0]: valid_country_keys.add((row[0] or "").strip().upper()) if row[1]: valid_country_keys.add((row[1] or "").strip().upper()) valid_fraction_ame: Set[str] = set() for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all(): if row[0]: valid_fraction_ame.add((row[0] or "").strip()) authorized_sectors: Set[str] = set() for row in session.query(Sector.key).filter( Sector.authorized == True, Sector.tenant_id == tenant_id, Sector.company_id == company_id, ).all(): if row[0]: authorized_sectors.add((row[0] or "").strip().upper()) valid_payment_methods: Set[str] = set() for row in session.query(PaymentMethod.key).all(): if row[0] is not None: valid_payment_methods.add(str(row[0]).strip()) valid_valuation_methods: Set[str] = set() for row in session.query(ValuationMethod.key).all(): if row[0]: valid_valuation_methods.add((row[0] or "").strip()) company = session.query(Company).filter(Company.id == company_id).first() company_has_prosec = bool(company.prosec) if company else False company_rfc = (company.rfc or "").strip().upper() if company else "" valid_part_numbers: Set[str] = set() for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all(): if row[0]: valid_part_numbers.add((row[0] or "").strip().upper()) rfc_exception_updated: Set[str] = set() rfc_exception_num_parte: Set[str] = set() with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) invoice_numbers_from_csv = set() for row in rows_list: inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() if inv: invoice_numbers_from_csv.add(inv) if company_rfc in RFC_EXCEPTION_UPDATED: rfc_exception_updated = invoice_numbers_from_csv if company_rfc in RFC_EXCEPTION_NUM_PARTE: rfc_exception_num_parte = invoice_numbers_from_csv line_counts_csv: Dict[Tuple[str, str], int] = {} partidas_principales_csv: Set[Tuple[str, str]] = set() def _get_row(row_norm: Dict[str, Any], *keys: str) -> str: for k in keys: v = row_norm.get(k) if v is not None and str(v).strip(): return str(v).strip() return "" for row in rows_list: row_norm = row_from_template(row, "imp_temp_details", normalize_header) inv = _get_row(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") linea = _get_row(row_norm, "LINEA", "RENGLON", "PARTIDA") if inv and linea: key = (inv, linea) line_counts_csv[key] = line_counts_csv.get(key, 0) + 1 u = _get_row(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper() if u == "P" and inv and linea: partidas_principales_csv.add((inv, linea)) error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_temp_details", normalize_header) err = validate_row_partidas_impo_temp( row_norm, i, autonumerar=autonumerar, actualizar=actualizar, levantar_subpartidas=levantar_subpartidas, calcular_costo_en_base_a_total=calcular_costo_en_base_a_total, validar_decimales_pza=validar_decimales_pza, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, rfc_exception_updated=rfc_exception_updated, existing_line_keys_by_invoice=existing_line_keys_by_invoice, line_counts_csv=line_counts_csv, partidas_principales_csv=partidas_principales_csv, partidas_principales_bd=partidas_principales_bd, valid_class_codes=valid_class_codes, class_um_by_code=class_um_by_code, class_fraction_by_code=class_fraction_by_code, class_desc_es_by_code=class_desc_es_by_code, class_desc_en_by_code=class_desc_en_by_code, valid_uom_codes=valid_uom_codes, valid_bulks_codes=valid_bulks_codes, valid_country_keys=valid_country_keys, valid_fraction_ame=valid_fraction_ame, valid_payment_methods=valid_payment_methods, valid_valuation_methods=valid_valuation_methods, authorized_sectors=authorized_sectors, company_has_prosec=company_has_prosec, rfc_exception_num_parte=rfc_exception_num_parte or None, valid_part_numbers=valid_part_numbers, warnings=None, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) except Exception as e: logger.exception("Partidas import scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Partidas de Importación Definitiva: flujo específico (Clarion VALIDA_TODA_PARIMPO_DEF / VALIDA_PARCIAL) --- # Estructura de columnas: misma que partidas TEM (imp_def_details resuelve a imp_temp_details). Facturas DEF/MATDE/EXDEF. if model_target == "invoice_details" and template_id == "imp_def_details": try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.classes.models import Class from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure from api.v1.modules.a76.general_catalogs.packages.models import Package from api.v1.modules.public.reference_data.countries.models import Country from api.v1.modules.a76.general_catalogs.sectors.models import Sector from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction from api.v1.modules.a76.general_catalogs.company.models import Company from api.v1.modules.a76.parts.models import Part from .validators.partidas_impo_def import validate_row_partidas_impo_def DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") _fc = parse_footer_config(meta.get("footer_config")) autonumerar = meta.get("autonumerar", True) actualizar = meta.get("actualizar", False) levantar_subpartidas = meta.get("levantar_subpartidas", False) calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False) validar_decimales_pza = meta.get("validar_decimales_pza", False) if _fc: if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) elif _fc.get("autonumber_partidas", "true") is not None: autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes") if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "levantar_subpartidas" in _fc: levantar_subpartidas = bool(_fc["levantar_subpartidas"]) if "calcular_costo_unitario_en_base_a_valor_total" in _fc: calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"]) if "validar_decimales_pza" in _fc: validar_decimales_pza = bool(_fc["validar_decimales_pza"]) RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"} RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"} with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in q_inv.all(): if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) existing_line_keys_by_invoice: Dict[str, Set[str]] = {} q_li = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) for num, ln in q_li.all(): if num is not None: key = str(num).strip() if key not in existing_line_keys_by_invoice: existing_line_keys_by_invoice[key] = set() existing_line_keys_by_invoice[key].add(str(ln).strip()) partidas_principales_bd: Set[Tuple[str, str]] = set() try: from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem q_pp = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(FaLineItem, FaLineItem.id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), FaLineItem.is_subitem == False, FaLineItem.contains_subitems == True, ) ) for num, ln in q_pp.all(): if num is not None: partidas_principales_bd.add((str(num).strip(), str(ln).strip())) except Exception: pass valid_class_codes: Set[str] = set() class_um_by_code: Dict[str, str] = {} class_fraction_by_code: Dict[str, str] = {} class_desc_es_by_code: Dict[str, str] = {} class_desc_en_by_code: Dict[str, str] = {} for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): code = (c.class_code or "").strip().upper() if code: valid_class_codes.add(code) class_um_by_code[code] = (c.unit_of_measure or "").strip().upper() class_fraction_by_code[code] = (c.fraction or "").strip() class_desc_es_by_code[code] = (c.description_es or "").strip() class_desc_en_by_code[code] = (c.description_en or "").strip() valid_uom_codes: Set[str] = set() for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): if u[0]: valid_uom_codes.add((u[0] or "").strip().upper()) valid_bulks_codes: Set[str] = set() for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): if p[0]: valid_bulks_codes.add((p[0] or "").strip()) valid_country_keys: Set[str] = set() for row in session.query(Country.m3_key, Country.ame_key).all(): if row[0]: valid_country_keys.add((row[0] or "").strip().upper()) if row[1]: valid_country_keys.add((row[1] or "").strip().upper()) valid_fraction_ame: Set[str] = set() for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all(): if row[0]: valid_fraction_ame.add((row[0] or "").strip()) authorized_sectors: Set[str] = set() for row in session.query(Sector.key).filter( Sector.authorized == True, Sector.tenant_id == tenant_id, Sector.company_id == company_id, ).all(): if row[0]: authorized_sectors.add((row[0] or "").strip().upper()) valid_payment_methods: Set[str] = set() for row in session.query(PaymentMethod.key).all(): if row[0] is not None: valid_payment_methods.add(str(row[0]).strip()) valid_valuation_methods: Set[str] = set() for row in session.query(ValuationMethod.key).all(): if row[0]: valid_valuation_methods.add((row[0] or "").strip()) company = session.query(Company).filter(Company.id == company_id).first() company_has_prosec = bool(company.prosec) if company else False company_rfc = (company.rfc or "").strip().upper() if company else "" valid_part_numbers: Set[str] = set() for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all(): if row[0]: valid_part_numbers.add((row[0] or "").strip().upper()) rfc_exception_updated: Set[str] = set() rfc_exception_num_parte: Set[str] = set() with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) invoice_numbers_from_csv = set() for row in rows_list: inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() if inv: invoice_numbers_from_csv.add(inv) if company_rfc in RFC_EXCEPTION_UPDATED: rfc_exception_updated = invoice_numbers_from_csv if company_rfc in RFC_EXCEPTION_NUM_PARTE: rfc_exception_num_parte = invoice_numbers_from_csv line_counts_csv: Dict[Tuple[str, str], int] = {} partidas_principales_csv: Set[Tuple[str, str]] = set() def _get_row_def(row_norm: Dict[str, Any], *keys: str) -> str: for k in keys: v = row_norm.get(k) if v is not None and str(v).strip(): return str(v).strip() return "" for row in rows_list: row_norm = row_from_template(row, "imp_def_details", normalize_header) inv = _get_row_def(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") linea = _get_row_def(row_norm, "LINEA", "RENGLON", "PARTIDA") if inv and linea: key = (inv, linea) line_counts_csv[key] = line_counts_csv.get(key, 0) + 1 u = _get_row_def(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper() if u == "P" and inv and linea: partidas_principales_csv.add((inv, linea)) error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_def_details", normalize_header) err = validate_row_partidas_impo_def( row_norm, i, autonumerar=autonumerar, actualizar=actualizar, levantar_subpartidas=levantar_subpartidas, calcular_costo_en_base_a_total=calcular_costo_en_base_a_total, validar_decimales_pza=validar_decimales_pza, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, rfc_exception_updated=rfc_exception_updated, existing_line_keys_by_invoice=existing_line_keys_by_invoice, line_counts_csv=line_counts_csv, partidas_principales_csv=partidas_principales_csv, partidas_principales_bd=partidas_principales_bd, valid_class_codes=valid_class_codes, class_um_by_code=class_um_by_code, class_fraction_by_code=class_fraction_by_code, class_desc_es_by_code=class_desc_es_by_code, class_desc_en_by_code=class_desc_en_by_code, valid_uom_codes=valid_uom_codes, valid_bulks_codes=valid_bulks_codes, valid_country_keys=valid_country_keys, valid_fraction_ame=valid_fraction_ame, valid_payment_methods=valid_payment_methods, valid_valuation_methods=valid_valuation_methods, authorized_sectors=authorized_sectors, company_has_prosec=company_has_prosec, rfc_exception_num_parte=rfc_exception_num_parte or None, valid_part_numbers=valid_part_numbers, warnings=None, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) except Exception as e: logger.exception("Partidas importación definitiva scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Partidas Exportación Definitiva: Clarion VALIDA_TODA_PAR_EXPO / VALIDA_PARCIAL_PAR_EXPO / VALIDACIONES_PAR_EXPO --- if model_target == "invoice_details" and template_id == "exp_def_partidas": logger.info("Partidas expo scan: running validation for job %s", job_id) try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure from api.v1.modules.a76.general_catalogs.packages.models import Package from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction from api.v1.modules.a76.general_catalogs.company.models import Company from api.v1.modules.a76.parts.models import Part from .validators.partidas_expo import validate_row_partidas_expo _fc = parse_footer_config(meta.get("footer_config")) autonumerar = meta.get("autonumerar", True) actualizar = meta.get("actualizar", False) levantar_subpartidas = meta.get("levantar_subpartidas", False) validar_decimales_pza = meta.get("validar_decimales_pza", False) if _fc: if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) elif _fc.get("autonumber_partidas", "true") is not None: autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes") if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "levantar_subpartidas" in _fc: levantar_subpartidas = bool(_fc["levantar_subpartidas"]) if "validar_decimales_pza" in _fc: validar_decimales_pza = bool(_fc["validar_decimales_pza"]) RFC_EXCEPTION_UPDATED = set() RFC_EXCEPTION_EGM = {"EGM0303257J1"} with CoreSessionLocal() as session: company = session.query(Company).filter(Company.id == company_id).first() company_rfc = (company.rfc or "").strip().upper() if company else "" if company_rfc in RFC_EXCEPTION_EGM: rfc_exception_egm = True else: rfc_exception_egm = False q_inv_expo = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in q_inv_expo.all(): if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) existing_line_keys_by_invoice: Dict[str, Set[str]] = {} q_li_expo = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) for num, ln in q_li_expo.all(): if num is not None: key = str(num).strip() if key not in existing_line_keys_by_invoice: existing_line_keys_by_invoice[key] = set() existing_line_keys_by_invoice[key].add(str(ln).strip()) partidas_principales_bd: Set[Tuple[str, str]] = set() try: from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem q_pp = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(FaLineItem, FaLineItem.id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", FaLineItem.is_subitem == False, FaLineItem.contains_subitems == True, ) ) for num, ln in q_pp.all(): if num is not None: partidas_principales_bd.add((str(num).strip(), str(ln).strip())) except Exception: pass factura_impo_tem_by_number: Dict[str, int] = {} q_tem = session.query(InvoiceHeader.invoice_number, InvoiceHeader.id).filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) for num, iid in q_tem.all(): if num: factura_impo_tem_by_number[str(num).strip()] = iid factura_impo_def_by_number: Dict[str, int] = {} DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") q_def = session.query(InvoiceHeader.invoice_number, InvoiceHeader.id).filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) for num, iid in q_def.all(): if num: factura_impo_def_by_number[str(num).strip()] = iid line_exists_tem: Set[Tuple[int, str]] = set() q_li_tem = ( session.query(LineItem.invoice_id, LineItem.line_number) .join(InvoiceHeader, InvoiceHeader.id == LineItem.invoice_id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) for inv_id, ln in q_li_tem.all(): if inv_id is not None and ln is not None: line_exists_tem.add((inv_id, str(ln).strip())) line_exists_def: Set[Tuple[int, str]] = set() q_li_def = ( session.query(LineItem.invoice_id, LineItem.line_number) .join(InvoiceHeader, InvoiceHeader.id == LineItem.invoice_id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) for inv_id, ln in q_li_def.all(): if inv_id is not None and ln is not None: line_exists_def.add((inv_id, str(ln).strip())) valid_uom_codes: Set[str] = set() for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): if u[0]: valid_uom_codes.add((u[0] or "").strip().upper()) valid_bulks_codes: Set[str] = set() for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): if p[0]: valid_bulks_codes.add((p[0] or "").strip()) valid_payment_methods: Set[str] = set() for row in session.query(PaymentMethod.key).all(): if row[0] is not None: valid_payment_methods.add(str(row[0]).strip()) valid_fraction_ame: Set[str] = set() for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all(): if row[0]: valid_fraction_ame.add((row[0] or "").strip()) valid_part_numbers: Set[str] = set() for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all(): if row[0]: valid_part_numbers.add((row[0] or "").strip().upper()) invoice_numbers_from_csv = set() with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) for row in rows_list: row_norm = row_from_template(row, "exp_def_partidas", normalize_header) inv = (row_norm.get("NUMERO FACTURA EXPO") or row_norm.get("NUMERO FACTURA EXPO.") or row_norm.get("FACTURA EXPO") or "").strip() if inv: invoice_numbers_from_csv.add(inv) if company_rfc in RFC_EXCEPTION_EGM: rfc_exception_updated = invoice_numbers_from_csv else: rfc_exception_updated = set() line_counts_csv: Dict[Tuple[str, str], int] = {} partidas_principales_csv: Set[Tuple[str, str]] = set() def _get_row_expo(row_norm: Dict[str, Any], *keys: str) -> str: for k in keys: v = row_norm.get(k) if v is not None and str(v).strip(): return str(v).strip() return "" for row in rows_list: row_norm = row_from_template(row, "exp_def_partidas", normalize_header) inv = _get_row_expo(row_norm, "NUMERO FACTURA EXPO", "NUMERO FACTURA EXPO.", "FACTURA EXPO") linea = _get_row_expo(row_norm, "LINEA EXPO", "LINEA EXPO.", "RENGLON EXPO") if inv and linea: key = (inv, linea) line_counts_csv[key] = line_counts_csv.get(key, 0) + 1 u = _get_row_expo(row_norm, "ES PARTIDA/SUBPARTIDA", "ESSUBPARTIDA", "ES PARTIDA O SUBPARTIDA").upper() if u == "P" and inv and linea: partidas_principales_csv.add((inv, linea)) error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): row_norm = row_from_template(row, "exp_def_partidas", normalize_header) err = validate_row_partidas_expo( row_norm, i, autonumerar=autonumerar, actualizar=actualizar, levantar_subpartidas=levantar_subpartidas, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, rfc_exception_updated=rfc_exception_updated, existing_line_keys_by_invoice=existing_line_keys_by_invoice, line_counts_csv=line_counts_csv, partidas_principales_csv=partidas_principales_csv, partidas_principales_bd=partidas_principales_bd, valid_uom_codes=valid_uom_codes, valid_bulks_codes=valid_bulks_codes, valid_payment_methods=valid_payment_methods, valid_fraction_ame=valid_fraction_ame, valid_part_numbers=valid_part_numbers, factura_impo_tem_by_number=factura_impo_tem_by_number, factura_impo_def_by_number=factura_impo_def_by_number, line_exists_tem=line_exists_tem, line_exists_def=line_exists_def, rfc_exception_egm=rfc_exception_egm, validar_decimales_pza=validar_decimales_pza, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") logger.info( "Partidas expo scan rechazo línea %s (col %s): %s", err["line"], err.get("col", ""), err.get("msg", ""), ) if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Partidas exportación definitiva scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Partidas Compras Mexicanas: misma lógica que Impo Def, facturas MEX --- if model_target == "invoice_details" and template_id == "cmex_details": try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.classes.models import Class from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure from api.v1.modules.a76.general_catalogs.packages.models import Package from api.v1.modules.public.reference_data.countries.models import Country from api.v1.modules.a76.general_catalogs.sectors.models import Sector from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction from api.v1.modules.a76.general_catalogs.company.models import Company from api.v1.modules.a76.parts.models import Part from .validators.partidas_impo_def import validate_row_partidas_impo_def _fc = parse_footer_config(meta.get("footer_config")) autonumerar = meta.get("autonumerar", True) actualizar = meta.get("actualizar", False) levantar_subpartidas = meta.get("levantar_subpartidas", False) calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False) validar_decimales_pza = meta.get("validar_decimales_pza", False) if _fc: if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) elif _fc.get("autonumber_partidas", "true") is not None: autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes") if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "levantar_subpartidas" in _fc: levantar_subpartidas = bool(_fc["levantar_subpartidas"]) if "calcular_costo_unitario_en_base_a_valor_total" in _fc: calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"]) if "validar_decimales_pza" in _fc: validar_decimales_pza = bool(_fc["validar_decimales_pza"]) RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"} RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"} with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in q_inv.all(): if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) existing_line_keys_by_invoice: Dict[str, Set[str]] = {} q_li = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) for num, ln in q_li.all(): if num is not None: key = str(num).strip() if key not in existing_line_keys_by_invoice: existing_line_keys_by_invoice[key] = set() existing_line_keys_by_invoice[key].add(str(ln).strip()) partidas_principales_bd: Set[Tuple[str, str]] = set() try: from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem q_pp = ( session.query(InvoiceHeader.invoice_number, LineItem.line_number) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(FaLineItem, FaLineItem.id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", FaLineItem.is_subitem == False, FaLineItem.contains_subitems == True, ) ) for num, ln in q_pp.all(): if num is not None: partidas_principales_bd.add((str(num).strip(), str(ln).strip())) except Exception: pass valid_class_codes: Set[str] = set() class_um_by_code: Dict[str, str] = {} class_fraction_by_code: Dict[str, str] = {} class_desc_es_by_code: Dict[str, str] = {} class_desc_en_by_code: Dict[str, str] = {} for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): code = (c.class_code or "").strip().upper() if code: valid_class_codes.add(code) class_um_by_code[code] = (c.unit_of_measure or "").strip().upper() class_fraction_by_code[code] = (c.fraction or "").strip() class_desc_es_by_code[code] = (c.description_es or "").strip() class_desc_en_by_code[code] = (c.description_en or "").strip() valid_uom_codes: Set[str] = set() for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): if u[0]: valid_uom_codes.add((u[0] or "").strip().upper()) valid_bulks_codes: Set[str] = set() for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): if p[0]: valid_bulks_codes.add((p[0] or "").strip()) valid_country_keys: Set[str] = set() for row in session.query(Country.m3_key, Country.ame_key).all(): if row[0]: valid_country_keys.add((row[0] or "").strip().upper()) if row[1]: valid_country_keys.add((row[1] or "").strip().upper()) valid_fraction_ame: Set[str] = set() for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all(): if row[0]: valid_fraction_ame.add((row[0] or "").strip()) authorized_sectors: Set[str] = set() for row in session.query(Sector.key).filter( Sector.authorized == True, Sector.tenant_id == tenant_id, Sector.company_id == company_id, ).all(): if row[0]: authorized_sectors.add((row[0] or "").strip().upper()) valid_payment_methods: Set[str] = set() for row in session.query(PaymentMethod.key).all(): if row[0] is not None: valid_payment_methods.add(str(row[0]).strip()) valid_valuation_methods: Set[str] = set() for row in session.query(ValuationMethod.key).all(): if row[0]: valid_valuation_methods.add((row[0] or "").strip()) company = session.query(Company).filter(Company.id == company_id).first() company_has_prosec = bool(company.prosec) if company else False company_rfc = (company.rfc or "").strip().upper() if company else "" valid_part_numbers: Set[str] = set() for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all(): if row[0]: valid_part_numbers.add((row[0] or "").strip().upper()) rfc_exception_updated: Set[str] = set() rfc_exception_num_parte: Set[str] = set() with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) invoice_numbers_from_csv = set() for row in rows_list: inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() if inv: invoice_numbers_from_csv.add(inv) if company_rfc in RFC_EXCEPTION_UPDATED: rfc_exception_updated = invoice_numbers_from_csv if company_rfc in RFC_EXCEPTION_NUM_PARTE: rfc_exception_num_parte = invoice_numbers_from_csv line_counts_csv: Dict[Tuple[str, str], int] = {} partidas_principales_csv: Set[Tuple[str, str]] = set() def _get_row_cmex(row_norm: Dict[str, Any], *keys: str) -> str: for k in keys: v = row_norm.get(k) if v is not None and str(v).strip(): return str(v).strip() return "" for row in rows_list: row_norm = row_from_template(row, "cmex_details", normalize_header) inv = _get_row_cmex(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") linea = _get_row_cmex(row_norm, "LINEA", "RENGLON", "PARTIDA") if inv and linea: key = (inv, linea) line_counts_csv[key] = line_counts_csv.get(key, 0) + 1 u = _get_row_cmex(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper() if u == "P" and inv and linea: partidas_principales_csv.add((inv, linea)) error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "cmex_details", normalize_header) err = validate_row_partidas_impo_def( row_norm, i, autonumerar=autonumerar, actualizar=actualizar, levantar_subpartidas=levantar_subpartidas, calcular_costo_en_base_a_total=calcular_costo_en_base_a_total, validar_decimales_pza=validar_decimales_pza, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, rfc_exception_updated=rfc_exception_updated, existing_line_keys_by_invoice=existing_line_keys_by_invoice, line_counts_csv=line_counts_csv, partidas_principales_csv=partidas_principales_csv, partidas_principales_bd=partidas_principales_bd, valid_class_codes=valid_class_codes, class_um_by_code=class_um_by_code, class_fraction_by_code=class_fraction_by_code, class_desc_es_by_code=class_desc_es_by_code, class_desc_en_by_code=class_desc_en_by_code, valid_uom_codes=valid_uom_codes, valid_bulks_codes=valid_bulks_codes, valid_country_keys=valid_country_keys, valid_fraction_ame=valid_fraction_ame, valid_payment_methods=valid_payment_methods, valid_valuation_methods=valid_valuation_methods, authorized_sectors=authorized_sectors, company_has_prosec=company_has_prosec, rfc_exception_num_parte=rfc_exception_num_parte or None, valid_part_numbers=valid_part_numbers, warnings=None, catalog_label="Compras Mexicanas", ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) except Exception as e: logger.exception("Partidas Compras Mexicanas scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Encabezados de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- if model_target == "invoice_header" and template_id == "imp_temp_header": try: from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate from api.v1.modules.a76.transportation.transporters.models import Transporter from .validators.encabezados_impo_temp import ( validate_row_encabezados_impo_temp, parse_pedimento_col_a, _pedimento_key_from_parsed, ) def _ped_key_from_row(ped_str: str) -> Optional[str]: parsed = parse_pedimento_col_a(ped_str) if not parsed: return None return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2]) _fc = parse_footer_config(meta.get("footer_config")) actualizar = meta.get("actualizar", False) autonumerar_remesas = meta.get("autonumerar_remesas", False) control_remesa = bool(_fc.get("control_remesa", False)) remesa_inicio = _fc.get("remesa_inicio") remesa_fin = _fc.get("remesa_fin") if remesa_inicio is not None: try: remesa_inicio = int(remesa_inicio) except (TypeError, ValueError): remesa_inicio = None if remesa_fin is not None: try: remesa_fin = int(remesa_fin) except (TypeError, ValueError): remesa_fin = None if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "autonumerar_remesas" in _fc: autonumerar_remesas = bool(_fc["autonumerar_remesas"]) with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) invoice_exists_by_number: Dict[str, bool] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in q_inv.all(): if num: n = str(num).strip() invoice_exists_by_number[n] = True invoice_processed_by_number[n] = bool(is_upd) pedimento_data_by_key: Dict[str, List[Dict[str, Any]]] = {} for p in ( session.query( Pedimentos.id, Pedimentos.year, Pedimentos.customs_office, Pedimentos.license, Pedimentos.pedimento_number, Pedimentos.operation_type, Pedimentos.regime, Pedimentos.pedimento_type, ) .filter( Pedimentos.tenant_id == tenant_id, Pedimentos.company_id == company_id, ) .all() ): co = (p.customs_office or "").strip() lic = (p.license or "").strip() num = (p.pedimento_number or "").strip() if not co or not lic or not num: continue key = _pedimento_key_from_parsed(co, lic, num) entry_date = None end_date = None pd = ( session.query(PedimentoDates.entry_date, PedimentoDates.end_date) .filter(PedimentoDates.pedimento_id == p.id).first() ) if pd: entry_date = pd[0] end_date = pd[1] info = { "id": p.id, "regime": (p.regime or "").strip(), "operation_type": (p.operation_type or "").strip(), "pedimento_type": (p.pedimento_type or "").strip(), "entry_date": entry_date, "end_date": end_date, } if key not in pedimento_data_by_key: pedimento_data_by_key[key] = [] pedimento_data_by_key[key].append(info) remesa_por_pedimento_bd: Dict[str, Set[int]] = {} q_rem = ( session.query( InvoiceComplianceMx.remesa, Pedimentos.year, Pedimentos.customs_office, Pedimentos.license, Pedimentos.pedimento_number, ) .join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id) .filter( InvoiceComplianceMx.tenant_id == tenant_id, InvoiceComplianceMx.company_id == company_id, InvoiceComplianceMx.pedimento_id.isnot(None), InvoiceComplianceMx.remesa.isnot(None), ) ) for rem, y, co, lic, num in q_rem.all(): if co and lic and num and rem is not None: key = _pedimento_key_from_parsed( (co or "").strip(), (lic or "").strip(), (num or "").strip(), ) if key not in remesa_por_pedimento_bd: remesa_por_pedimento_bd[key] = set() remesa_por_pedimento_bd[key].add(int(rem)) valid_provider_ids: Set[int] = set() valid_sold_to_ids: Set[int] = set() valid_shipped_to_ids: Set[int] = set() valid_provider_short_names: Set[str] = set() valid_sold_to_short_names: Set[str] = set() valid_shipped_to_short_names: Set[str] = set() for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter( ClientProvider.tenant_id == tenant_id, ClientProvider.company_id == company_id, ).all(): valid_provider_ids.add(cp[0]) valid_sold_to_ids.add(cp[0]) valid_shipped_to_ids.add(cp[0]) if cp[1] and str(cp[1]).strip(): sn_upper = str(cp[1]).strip().upper() valid_provider_short_names.add(sn_upper) valid_sold_to_short_names.add(sn_upper) valid_shipped_to_short_names.add(sn_upper) valid_broker_ids: Set[int] = set() valid_broker_claves: Set[str] = set() for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter( CustomsBroker.tenant_id == tenant_id, CustomsBroker.company_id == company_id, ).all(): valid_broker_ids.add(cb[0]) if cb[1] and str(cb[1]).strip(): valid_broker_claves.add(str(cb[1]).strip()) valid_transporter_keys: Set[str] = set() for t in session.query(Transporter.transporter_key).filter( Transporter.tenant_id == tenant_id, Transporter.company_id == company_id, ).all(): if t[0]: valid_transporter_keys.add((t[0] or "").strip().upper()) valid_incoterms: Set[str] = set() for inc in session.query(Incoterm.code).all(): if inc[0]: valid_incoterms.add((inc[0] or "").strip().upper()) valid_aduana_codes: Set[str] = set() for cs in session.query(CustomsSection.customs_code).all(): if cs[0]: valid_aduana_codes.add((cs[0] or "").strip()) valid_currency_codes: Set[str] = set() for ct in session.query(CurrencyType.code).all(): if ct[0]: valid_currency_codes.add((ct[0] or "").strip().upper()) exchange_rate_by_date: Dict[str, Any] = {} for er in session.query(ExchangeRate.date, ExchangeRate.value).filter( ExchangeRate.tenant_id == tenant_id, ExchangeRate.company_id == company_id, ).all(): if er[0] and er[1] is not None: dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10] exchange_rate_by_date[dk] = er[1] invoice_has_partidas_by_number: Dict[str, bool] = {} existing_tipo_moneda_by_number: Dict[str, str] = {} q_li_count = ( session.query(InvoiceHeader.invoice_number, func.count(LineItem.id)) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) .group_by(InvoiceHeader.invoice_number) ) for num, cnt in q_li_count.all(): if num: invoice_has_partidas_by_number[str(num).strip()] = cnt > 0 q_fin = ( session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency) .join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) for num, cur in q_fin.all(): if num and cur: cur_str = (cur or "").strip().lower() if cur_str == "foreign": existing_tipo_moneda_by_number[str(num).strip()] = "ME" elif cur_str == "local": existing_tipo_moneda_by_number[str(num).strip()] = "MN" else: existing_tipo_moneda_by_number[str(num).strip()] = cur_str.upper()[:2] with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) remesa_por_pedimento_csv: Dict[str, Dict[int, str]] = {} for row in rows_list: row_norm = row_from_template(row, "imp_temp_header", normalize_header) ped = (row_norm.get("PEDIMENTO") or "").strip() rem = row_norm.get("REMESA") factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip() if not ped or not factura: continue key = _ped_key_from_row(ped) if not key: continue try: rem_int = int(rem) if rem is not None and str(rem).strip() else None except (TypeError, ValueError): rem_int = None if rem_int is not None: if key not in remesa_por_pedimento_csv: remesa_por_pedimento_csv[key] = {} if rem_int not in remesa_por_pedimento_csv[key]: remesa_por_pedimento_csv[key][rem_int] = factura error_count = 0 processed_rows = 0 error_lines_list: List[int] = [] errors_detail: List[Dict[str, Any]] = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_temp_header", normalize_header) warnings_row: List[Dict[str, Any]] = [] err = validate_row_encabezados_impo_temp( row_norm, i, actualizar=actualizar, invoice_exists_by_number=invoice_exists_by_number, invoice_processed_by_number=invoice_processed_by_number, pedimento_data_by_key=pedimento_data_by_key, remesa_por_pedimento_bd=remesa_por_pedimento_bd, remesa_por_pedimento_csv=remesa_por_pedimento_csv, valid_provider_ids=valid_provider_ids, valid_sold_to_ids=valid_sold_to_ids, valid_shipped_to_ids=valid_shipped_to_ids, valid_provider_short_names=valid_provider_short_names, valid_sold_to_short_names=valid_sold_to_short_names, valid_shipped_to_short_names=valid_shipped_to_short_names, valid_broker_ids=valid_broker_ids, valid_broker_claves=valid_broker_claves, valid_transporter_keys=valid_transporter_keys, valid_incoterms=valid_incoterms, valid_aduana_codes=valid_aduana_codes, valid_currency_codes=valid_currency_codes, exchange_rate_by_date=exchange_rate_by_date, invoice_has_partidas_by_number=invoice_has_partidas_by_number, existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, autonumerar_remesas=autonumerar_remesas, control_remesa=control_remesa, remesa_inicio=remesa_inicio, remesa_fin=remesa_fin, date_format=date_format, parse_date_fn=parse_date, warnings=warnings_row, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) for w in warnings_row: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Encabezados importación temporal scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Encabezados de Importación Definitiva: flujo específico (Clarion VALIDA_TODA_FACIMPO_DEF / VALIDA_PARCIAL) --- if model_target == "invoice_header" and template_id == "imp_def_header": try: from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate from api.v1.modules.a76.transportation.transporters.models import Transporter from .validators.encabezados_impo_def import ( validate_row_encabezados_impo_def, parse_pedimento_col_a_impo_def, ) from .validators.encabezados_impo_temp import _pedimento_key_from_parsed DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") def _ped_key_from_row_def(ped_str: str) -> Optional[str]: parsed = parse_pedimento_col_a_impo_def(ped_str) if not parsed: return None return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2]) _fc = parse_footer_config(meta.get("footer_config")) actualizar = meta.get("actualizar", False) autonumerar_remesas = meta.get("autonumerar_remesas", False) control_remesa = bool(_fc.get("control_remesa", False)) remesa_inicio = _fc.get("remesa_inicio") remesa_fin = _fc.get("remesa_fin") if remesa_inicio is not None: try: remesa_inicio = int(remesa_inicio) except (TypeError, ValueError): remesa_inicio = None if remesa_fin is not None: try: remesa_fin = int(remesa_fin) except (TypeError, ValueError): remesa_fin = None if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "autonumerar_remesas" in _fc: autonumerar_remesas = bool(_fc["autonumerar_remesas"]) with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) invoice_exists_by_number = {} invoice_processed_by_number = {} for num, iid, is_upd in q_inv.all(): if num: n = str(num).strip() invoice_exists_by_number[n] = True invoice_processed_by_number[n] = bool(is_upd) pedimento_data_by_key = {} for p in ( session.query( Pedimentos.id, Pedimentos.year, Pedimentos.customs_office, Pedimentos.license, Pedimentos.pedimento_number, Pedimentos.operation_type, Pedimentos.regime, Pedimentos.pedimento_type, ) .filter( Pedimentos.tenant_id == tenant_id, Pedimentos.company_id == company_id, Pedimentos.operation_type == "imp", Pedimentos.regime == "IMD", ) .all() ): co = (p.customs_office or "").strip() lic = (p.license or "").strip() num = (p.pedimento_number or "").strip() if not co or not lic or not num: continue key = _pedimento_key_from_parsed(co, lic, num) entry_date = None end_date = None pd = ( session.query(PedimentoDates.entry_date, PedimentoDates.end_date) .filter(PedimentoDates.pedimento_id == p.id).first() ) if pd: entry_date = pd[0] end_date = pd[1] info = { "id": p.id, "regime": (p.regime or "").strip(), "operation_type": (p.operation_type or "").strip(), "pedimento_type": (p.pedimento_type or "").strip(), "entry_date": entry_date, "end_date": end_date, } if key not in pedimento_data_by_key: pedimento_data_by_key[key] = [] pedimento_data_by_key[key].append(info) remesa_por_pedimento_bd = {} q_rem = ( session.query( InvoiceComplianceMx.remesa, Pedimentos.customs_office, Pedimentos.license, Pedimentos.pedimento_number, ) .join(InvoiceHeader, InvoiceHeader.id == InvoiceComplianceMx.invoice_id) .join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id) .filter( InvoiceComplianceMx.tenant_id == tenant_id, InvoiceComplianceMx.company_id == company_id, InvoiceComplianceMx.pedimento_id.isnot(None), InvoiceComplianceMx.remesa.isnot(None), InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) for rem, co, lic, num in q_rem.all(): if co and lic and num and rem is not None: key = _pedimento_key_from_parsed( (co or "").strip(), (lic or "").strip(), (num or "").strip(), ) if key not in remesa_por_pedimento_bd: remesa_por_pedimento_bd[key] = set() remesa_por_pedimento_bd[key].add(int(rem)) valid_provider_ids = set() valid_sold_to_ids = set() valid_shipped_to_ids = set() valid_provider_short_names = set() valid_sold_to_short_names = set() valid_shipped_to_short_names = set() for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter( ClientProvider.tenant_id == tenant_id, ClientProvider.company_id == company_id, ).all(): valid_provider_ids.add(cp[0]) valid_sold_to_ids.add(cp[0]) valid_shipped_to_ids.add(cp[0]) if cp[1] and str(cp[1]).strip(): sn_upper = str(cp[1]).strip().upper() valid_provider_short_names.add(sn_upper) valid_sold_to_short_names.add(sn_upper) valid_shipped_to_short_names.add(sn_upper) valid_broker_ids = set() valid_broker_claves = set() for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter( CustomsBroker.tenant_id == tenant_id, CustomsBroker.company_id == company_id, ).all(): valid_broker_ids.add(cb[0]) if cb[1] and str(cb[1]).strip(): valid_broker_claves.add(str(cb[1]).strip()) valid_transporter_keys = set() for t in session.query(Transporter.transporter_key).filter( Transporter.tenant_id == tenant_id, Transporter.company_id == company_id, ).all(): if t[0]: valid_transporter_keys.add((t[0] or "").strip().upper()) valid_incoterms = set() for inc in session.query(Incoterm.code).all(): if inc[0]: valid_incoterms.add((inc[0] or "").strip().upper()) valid_aduana_codes = set() for cs in session.query(CustomsSection.customs_code).all(): if cs[0]: valid_aduana_codes.add((cs[0] or "").strip()) valid_currency_codes = set() for ct in session.query(CurrencyType.code).all(): if ct[0]: valid_currency_codes.add((ct[0] or "").strip().upper()) exchange_rate_by_date = {} for er in session.query(ExchangeRate.date, ExchangeRate.value).filter( ExchangeRate.tenant_id == tenant_id, ExchangeRate.company_id == company_id, ).all(): if er[0] and er[1] is not None: dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10] exchange_rate_by_date[dk] = er[1] invoice_has_partidas_by_number = {} existing_tipo_moneda_by_number = {} q_li_count = ( session.query(InvoiceHeader.invoice_number, func.count(LineItem.id)) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) .group_by(InvoiceHeader.invoice_number) ) for num, cnt in q_li_count.all(): if num: invoice_has_partidas_by_number[str(num).strip()] = cnt > 0 q_fin = ( session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency) .join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(DEF_INVOICE_TYPES), ) ) for num, cur in q_fin.all(): if num and cur: cur_str = (cur or "").strip().lower() if cur_str == "foreign": existing_tipo_moneda_by_number[str(num).strip()] = "ME" elif cur_str == "local": existing_tipo_moneda_by_number[str(num).strip()] = "MN" else: existing_tipo_moneda_by_number[str(num).strip()] = (cur_str or "").upper()[:2] with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) remesa_por_pedimento_csv = {} for row in rows_list: row_norm = row_from_template(row, "imp_def_header", normalize_header) ped = (row_norm.get("PEDIMENTO") or "").strip() rem = row_norm.get("REMESA") factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip() if not ped or not factura: continue key = _ped_key_from_row_def(ped) if not key: continue try: rem_int = int(rem) if rem is not None and str(rem).strip() else None except (TypeError, ValueError): rem_int = None if rem_int is not None: if key not in remesa_por_pedimento_csv: remesa_por_pedimento_csv[key] = {} if rem_int not in remesa_por_pedimento_csv[key]: remesa_por_pedimento_csv[key][rem_int] = factura error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_def_header", normalize_header) warnings_row = [] err = validate_row_encabezados_impo_def( row_norm, i, actualizar=actualizar, invoice_exists_by_number=invoice_exists_by_number, invoice_processed_by_number=invoice_processed_by_number, pedimento_data_by_key=pedimento_data_by_key, remesa_por_pedimento_bd=remesa_por_pedimento_bd, remesa_por_pedimento_csv=remesa_por_pedimento_csv, valid_provider_ids=valid_provider_ids, valid_sold_to_ids=valid_sold_to_ids, valid_shipped_to_ids=valid_shipped_to_ids, valid_provider_short_names=valid_provider_short_names, valid_sold_to_short_names=valid_sold_to_short_names, valid_shipped_to_short_names=valid_shipped_to_short_names, valid_broker_ids=valid_broker_ids, valid_broker_claves=valid_broker_claves, valid_transporter_keys=valid_transporter_keys, valid_incoterms=valid_incoterms, valid_aduana_codes=valid_aduana_codes, valid_currency_codes=valid_currency_codes, exchange_rate_by_date=exchange_rate_by_date, invoice_has_partidas_by_number=invoice_has_partidas_by_number, existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, autonumerar_remesas=autonumerar_remesas, control_remesa=control_remesa, remesa_inicio=remesa_inicio, remesa_fin=remesa_fin, date_format=date_format, parse_date_fn=parse_date, warnings=warnings_row, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) for w in warnings_row: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Encabezados importación definitiva scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Encabezados Exportación (Expo Def) y Cambio de Régimen: flujo específico (Clarion VALIDA_TODA_FAC_EXPO / VALIDA_PARCIAL) --- if model_target == "invoice_header" and template_id == "exp_def_header": try: from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate from api.v1.modules.a76.transportation.transporters.models import Transporter from api.v1.modules.a76.manifests.manifest.models import Manifest from .validators.encabezados_expo import validate_row_encabezados_expo from .validators.encabezados_impo_temp import _pedimento_key_from_parsed from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def _fc = parse_footer_config(meta.get("footer_config")) actualizar = meta.get("actualizar", False) autonumerar_remesas = meta.get("autonumerar_remesas", False) recalcular_fecha_pedimentos = meta.get("recalcular_fecha_pedimentos", False) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "autonumerar_remesas" in _fc: autonumerar_remesas = bool(_fc["autonumerar_remesas"]) if "recalcular_fecha_pedimentos" in _fc: recalcular_fecha_pedimentos = bool(_fc["recalcular_fecha_pedimentos"]) cambio_regimen_raw = (meta.get("cambio_regimen") or _fc.get("cambio_regimen") or "NO").strip().upper() cambio_regimen = cambio_regimen_raw == "SI" tipo_factura = (meta.get("tipo_factura") or _fc.get("tipo_factura") or "AFIJO").strip().upper() TIPOS_FACTURA_EXPO_VALIDOS = frozenset({"NODES", "AFIJO", "DONAC", "SCRAP"}) if tipo_factura not in TIPOS_FACTURA_EXPO_VALIDOS: return { "status": "failed", "error": f"Tipo de factura '{tipo_factura}' no válido para Exportación. Debe ser uno de: NODES, AFIJO, DONAC, SCRAP.", } if cambio_regimen and tipo_factura != "AFIJO": return { "status": "failed", "error": "Este tipo de factura no es compatible para Cambio de Régimen, seleccionar AFIJO.", } def _ped_key_from_row_expo(ped_str: str): parsed = parse_pedimento_col_a_impo_def(ped_str) if not parsed: return None return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2]) with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status, InvoiceHeader.status_rep) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) invoice_exists_by_number = {} invoice_processed_by_number = {} invoice_in_report_by_number = {} for num, iid, is_upd, is_rep in q_inv.all(): if num: n = str(num).strip() invoice_exists_by_number[n] = True invoice_processed_by_number[n] = bool(is_upd) invoice_in_report_by_number[n] = bool(is_rep) if is_rep is not None else False if cambio_regimen: ped_filter_op = "imp" ped_filter_regimes = ["IMD"] else: ped_filter_op = "exp" ped_filter_regimes = ["EXD", "ETE", "ETR"] pedimento_data_by_key = {} for p in ( session.query( Pedimentos.id, Pedimentos.customs_office, Pedimentos.license, Pedimentos.pedimento_number, Pedimentos.operation_type, Pedimentos.regime, Pedimentos.pedimento_type, Pedimentos.pedimento_code, ) .filter( Pedimentos.tenant_id == tenant_id, Pedimentos.company_id == company_id, Pedimentos.operation_type == ped_filter_op, Pedimentos.regime.in_(ped_filter_regimes), ) .all() ): co = (p.customs_office or "").strip() lic = (p.license or "").strip() num = (p.pedimento_number or "").strip() if not co or not lic or not num: continue key = _pedimento_key_from_parsed(co, lic, num) entry_date = None end_date = None pd = ( session.query(PedimentoDates.entry_date, PedimentoDates.end_date) .filter(PedimentoDates.pedimento_id == p.id).first() ) if pd: entry_date = pd[0] end_date = pd[1] info = { "id": p.id, "regime": (p.regime or "").strip(), "operation_type": (p.operation_type or "").strip().upper()[:3], "pedimento_type": (p.pedimento_type or "").strip(), "pedimento_code": (p.pedimento_code or "").strip().upper(), "entry_date": entry_date, "end_date": end_date, } if key not in pedimento_data_by_key: pedimento_data_by_key[key] = [] pedimento_data_by_key[key].append(info) remesa_por_pedimento_bd = {} q_rem = ( session.query( InvoiceComplianceMx.remesa, Pedimentos.customs_office, Pedimentos.license, Pedimentos.pedimento_number, ) .join(InvoiceHeader, InvoiceHeader.id == InvoiceComplianceMx.invoice_id) .join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id) .filter( InvoiceComplianceMx.tenant_id == tenant_id, InvoiceComplianceMx.company_id == company_id, InvoiceComplianceMx.pedimento_id.isnot(None), InvoiceComplianceMx.remesa.isnot(None), InvoiceHeader.operation_type == "exp", ) ) for rem, co, lic, num in q_rem.all(): if co and lic and num and rem is not None: key = _pedimento_key_from_parsed( (co or "").strip()[:2], (lic or "").strip(), (num or "").strip(), ) if key not in remesa_por_pedimento_bd: remesa_por_pedimento_bd[key] = set() remesa_por_pedimento_bd[key].add(int(rem)) valid_provider_ids = set() valid_sold_to_ids = set() valid_shipped_to_ids = set() valid_provider_short_names = set() valid_sold_to_short_names = set() valid_shipped_to_short_names = set() for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter( ClientProvider.tenant_id == tenant_id, ClientProvider.company_id == company_id, ).all(): valid_provider_ids.add(cp[0]) valid_sold_to_ids.add(cp[0]) valid_shipped_to_ids.add(cp[0]) if cp[1] and str(cp[1]).strip(): sn_upper = str(cp[1]).strip().upper() valid_provider_short_names.add(sn_upper) valid_sold_to_short_names.add(sn_upper) valid_shipped_to_short_names.add(sn_upper) valid_broker_ids = set() valid_broker_claves = set() for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter( CustomsBroker.tenant_id == tenant_id, CustomsBroker.company_id == company_id, ).all(): valid_broker_ids.add(cb[0]) if cb[1] and str(cb[1]).strip(): valid_broker_claves.add(str(cb[1]).strip()) valid_transporter_keys = set() for t in session.query(Transporter.transporter_key).filter( Transporter.tenant_id == tenant_id, Transporter.company_id == company_id, ).all(): if t[0]: valid_transporter_keys.add((t[0] or "").strip().upper()) valid_incoterms = set() for inc in session.query(Incoterm.code).all(): if inc[0]: valid_incoterms.add((inc[0] or "").strip().upper()) valid_aduana_codes = set() for cs in session.query(CustomsSection.customs_code).all(): if cs[0]: valid_aduana_codes.add((cs[0] or "").strip()) valid_currency_codes = set() for ct in session.query(CurrencyType.code).all(): if ct[0]: valid_currency_codes.add((ct[0] or "").strip().upper()) valid_manifiesto_codes = set() for m in session.query(Manifest.manifest_number).filter( Manifest.tenant_id == tenant_id, Manifest.company_id == company_id, ).all(): if m[0] and str(m[0]).strip(): valid_manifiesto_codes.add(str(m[0]).strip()) exchange_rate_by_date = {} for er in session.query(ExchangeRate.date, ExchangeRate.value).filter( ExchangeRate.tenant_id == tenant_id, ExchangeRate.company_id == company_id, ).all(): if er[0] and er[1] is not None: dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10] exchange_rate_by_date[dk] = er[1] invoice_has_partidas_by_number = {} existing_tipo_moneda_by_number = {} q_li_count = ( session.query(InvoiceHeader.invoice_number, func.count(LineItem.id)) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) .group_by(InvoiceHeader.invoice_number) ) for num, cnt in q_li_count.all(): if num: invoice_has_partidas_by_number[str(num).strip()] = cnt > 0 q_fin = ( session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency) .join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) for num, cur in q_fin.all(): if num and cur: cur_str = (cur or "").strip().lower() if cur_str == "foreign": existing_tipo_moneda_by_number[str(num).strip()] = "ME" elif cur_str == "local": existing_tipo_moneda_by_number[str(num).strip()] = "MN" else: existing_tipo_moneda_by_number[str(num).strip()] = (cur_str or "").upper()[:2] date_format = _fc.get("dateFormat") or meta.get("date_format") with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) remesa_por_pedimento_csv = {} for row in rows_list: row_norm = row_from_template(row, "exp_def_header", normalize_header) ped = (row_norm.get("PEDIMENTO") or "").strip() rem = row_norm.get("REMESA") factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip() if not ped or not factura: continue key = _ped_key_from_row_expo(ped) if not key: continue try: rem_int = int(rem) if rem is not None and str(rem).strip() else None except (TypeError, ValueError): rem_int = None if rem_int is not None: if key not in remesa_por_pedimento_csv: remesa_por_pedimento_csv[key] = {} if rem_int not in remesa_por_pedimento_csv[key]: remesa_por_pedimento_csv[key][rem_int] = factura error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "exp_def_header", normalize_header) warnings_row = [] err = validate_row_encabezados_expo( row_norm, i, actualizar=actualizar, cambio_regimen=cambio_regimen, tipo_factura=tipo_factura, invoice_exists_by_number=invoice_exists_by_number, invoice_processed_by_number=invoice_processed_by_number, invoice_in_report_by_number=invoice_in_report_by_number, pedimento_data_by_key=pedimento_data_by_key, remesa_por_pedimento_bd=remesa_por_pedimento_bd, remesa_por_pedimento_csv=remesa_por_pedimento_csv, valid_provider_ids=valid_provider_ids, valid_sold_to_ids=valid_sold_to_ids, valid_shipped_to_ids=valid_shipped_to_ids, valid_broker_ids=valid_broker_ids, valid_broker_claves=valid_broker_claves, valid_transporter_keys=valid_transporter_keys, valid_incoterms=valid_incoterms, valid_aduana_codes=valid_aduana_codes, valid_currency_codes=valid_currency_codes, valid_provider_short_names=valid_provider_short_names, valid_sold_to_short_names=valid_sold_to_short_names, valid_shipped_to_short_names=valid_shipped_to_short_names, valid_manifiesto_codes=valid_manifiesto_codes, valid_enviado_por_ids=valid_provider_ids, valid_enviado_por_short_names=valid_provider_short_names, exchange_rate_by_date=exchange_rate_by_date, invoice_has_partidas_by_number=invoice_has_partidas_by_number, existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, autonumerar_remesas=autonumerar_remesas, recalcular_fecha_pedimentos=recalcular_fecha_pedimentos, date_format=date_format, parse_date_fn=parse_date, warnings=warnings_row, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) for w in warnings_row: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Encabezados exportación scan failed: %s", e) return {"status": "failed", "error": str(e)} # --- Encabezados Compras Mexicanas: flujo específico (Clarion VALIDA_TODA_FAC_COM_MEX / VALIDA_PARCIAL) --- if model_target == "invoice_header" and template_id == "cmex_header": try: from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceFinancials from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate from api.v1.modules.a76.transportation.transporters.models import Transporter from .validators.encabezados_cmex import validate_row_encabezados_cmex _fc = parse_footer_config(meta.get("footer_config")) actualizar = meta.get("actualizar", False) if _fc and "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) invoice_exists_by_number = {} invoice_processed_by_number = {} for num, iid, is_upd in q_inv.all(): if num: n = str(num).strip() invoice_exists_by_number[n] = True invoice_processed_by_number[n] = bool(is_upd) valid_provider_ids = set() valid_sold_to_ids = set() valid_shipped_to_ids = set() valid_provider_short_names = set() valid_sold_to_short_names = set() valid_shipped_to_short_names = set() for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter( ClientProvider.tenant_id == tenant_id, ClientProvider.company_id == company_id, ).all(): valid_provider_ids.add(cp[0]) valid_sold_to_ids.add(cp[0]) valid_shipped_to_ids.add(cp[0]) if cp[1] and str(cp[1]).strip(): sn_upper = str(cp[1]).strip().upper() valid_provider_short_names.add(sn_upper) valid_sold_to_short_names.add(sn_upper) valid_shipped_to_short_names.add(sn_upper) valid_transporter_keys = set() for t in session.query(Transporter.transporter_key).filter( Transporter.tenant_id == tenant_id, Transporter.company_id == company_id, ).all(): if t[0]: valid_transporter_keys.add((t[0] or "").strip().upper()) valid_incoterms = set() for inc in session.query(Incoterm.code).all(): if inc[0]: valid_incoterms.add((inc[0] or "").strip().upper()) valid_currency_codes = set() for ct in session.query(CurrencyType.code).all(): if ct[0]: valid_currency_codes.add((ct[0] or "").strip().upper()) exchange_rate_by_date = {} for er in session.query(ExchangeRate.date, ExchangeRate.value).filter( ExchangeRate.tenant_id == tenant_id, ExchangeRate.company_id == company_id, ).all(): if er[0] and er[1] is not None: dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10] exchange_rate_by_date[dk] = er[1] invoice_has_partidas_by_number = {} existing_tipo_moneda_by_number = {} q_li_count = ( session.query(InvoiceHeader.invoice_number, func.count(LineItem.id)) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) .group_by(InvoiceHeader.invoice_number) ) for num, cnt in q_li_count.all(): if num: invoice_has_partidas_by_number[str(num).strip()] = cnt > 0 q_fin = ( session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency) .join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "MEX", ) ) for num, cur in q_fin.all(): if num and cur: cur_str = (cur or "").strip().lower() if cur_str == "foreign": existing_tipo_moneda_by_number[str(num).strip()] = "ME" elif cur_str == "local": existing_tipo_moneda_by_number[str(num).strip()] = "MN" else: existing_tipo_moneda_by_number[str(num).strip()] = (cur_str or "").upper()[:2] with open(file_path, "r", encoding="utf-8-sig") as f_in: sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f_in, dialect=dialect) rows_list = list(reader) error_count = 0 processed_rows = 0 error_lines_list = [] errors_detail = [] with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): if i % 1000 == 0: self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "cmex_header", normalize_header) warnings_row = [] err = validate_row_encabezados_cmex( row_norm, i, actualizar=actualizar, invoice_exists_by_number=invoice_exists_by_number, invoice_processed_by_number=invoice_processed_by_number, valid_provider_ids=valid_provider_ids, valid_sold_to_ids=valid_sold_to_ids, valid_shipped_to_ids=valid_shipped_to_ids, valid_transporter_keys=valid_transporter_keys, valid_incoterms=valid_incoterms, valid_currency_codes=valid_currency_codes, exchange_rate_by_date=exchange_rate_by_date, invoice_has_partidas_by_number=invoice_has_partidas_by_number, existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, valid_provider_short_names=valid_provider_short_names, valid_sold_to_short_names=valid_sold_to_short_names, valid_shipped_to_short_names=valid_shipped_to_short_names, date_format=date_format, parse_date_fn=parse_date, warnings=warnings_row, ) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") if len(errors_detail) < 500: errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) for w in warnings_row: if len(errors_detail) < 500: errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows ) except Exception as e: logger.exception("Encabezados Compras Mexicanas scan failed: %s", e) return {"status": "failed", "error": str(e)} try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( CodePedimentoRegimen, ) from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.a76.parts.models import Part models = { "InvoiceHeader": InvoiceHeader, "InvoiceType": InvoiceType, "ClientProvider": ClientProvider, "CustomsBroker": CustomsBroker, "RegimenPedimento": RegimenPedimento, "CodePedimentoRegimen": CodePedimentoRegimen, "PedimentoCode": PedimentoCode, "CurrencyType": CurrencyType, "CustomsSection": CustomsSection, "Incoterm": Incoterm, "Part": Part, } with CoreSessionLocal() as session, \ open(file_path, 'r', encoding='utf-8-sig') as f_in, \ open(error_path, 'w', encoding='utf-8') as f_err: validator = ForeignKeyValidator(session, tenant_id, company_id) invoice_id_cache: Dict[str, Optional[int]] = {} # Detect Delimiter sample = f_in.read(2048) f_in.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except: dialect = 'excel' reader = csv.DictReader(f_in, dialect=dialect) for i, row in enumerate(reader, start=1): # Check for Progress Update if i % 1000 == 0: self.update_state(state='PROGRESS', meta={ 'current': i, 'total': total_rows, 'errors': error_count }) # Solo columnas de la plantilla (respetar plantilla tal cual) row_norm = row_from_template(row, template_id, normalize_header) errors = validate_row_strict( row_norm, model_target, i, date_format, validator, inv_type_value, invoice_id_cache, models, ) if errors: error_count += 1 # Write simple JSON error f_err.write(json.dumps(errors) + "\n") processed_rows += 1 except Exception as e: logger.error(f"Scan failed: {e}") return {"status": "failed", "error": str(e)} # 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker) error_lines_list = [] errors_detail: List[Dict[str, Any]] = [] try: if os.path.exists(error_path): with open(error_path, "r", encoding="utf-8") as f: for line in f: try: err = json.loads(line) if "line" in err: error_lines_list.append(err["line"]) if len(errors_detail) < 500: errors_detail.append( { "line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", ""), } ) except Exception: pass common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) except Exception as e: logger.warning(f"Failed to store error lines in Redis: {e}") return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def validate_row_phase_1( row: Dict[str, Any], target: str, line_num: int, date_format: Optional[str], ) -> Optional[Dict[str, Any]]: """ Validation: Unique IDs, Dates, and Numeric constraint checks. Target: 'invoice_header' or 'invoice_details' """ def check_decimal(col_name): val = row.get(col_name) if val and str(val).strip(): if parse_decimal(val) is None: return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"} return None def check_int(col_name): val = row.get(col_name) if val and str(val).strip(): if parse_int(val) is None: return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"} return None def check_date(col_name): date_str = row.get(col_name) if date_str and str(date_str).strip(): if not is_valid_date(date_str, date_format): expected = display_date_format(date_format) return { "line": line_num, "col": col_name, "msg": f"Formato de fecha inválido ({expected})", } return None def check_weight(col_name): val = row.get(col_name) if val and str(val).strip(): if parse_weight_unit(val) is None: return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"} return None def check_currency(col_name): val = row.get(col_name) if val and str(val).strip(): parsed_currency = parse_currency(val, None) val_norm = normalize_header(val) # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid if parsed_currency.value == "manual" and "MANUAL" not in val_norm: return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"} return None def check_transport_type(col_name): val = row.get(col_name) if val and str(val).strip(): if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES: return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"} return None # A. Invoice Header if target == 'invoice_header': # 1. Unique ID if not row.get('NUMERO FACTURA') and not row.get('NUM FACTURA') and not row.get('ID'): return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} # 2. Date Format date_str = row.get('FECHA FACTURA') or row.get('FECHA') if not date_str or not str(date_str).strip(): return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"} err = check_date('FECHA FACTURA') or check_date('FECHA') if err: return err err = check_date('FECHA EMISION') if err: return err # 3. Numeric Fields for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']: err = check_decimal(col) if err: return err # 4. Integer FKs (CLAVE PROVEEDOR/VENDIDO/ENVIADO aceptan short_name; AGENTE ADUANAL acepta clave; solo REMESA exige entero) int_fk_cols = ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA'] if target == 'invoice_header': int_fk_cols = ['REMESA'] # proveedor/vendido/enviado por short_name; agente aduanal por clave for col in int_fk_cols: err = check_int(col) if err: return err # 5. Enums for col in ['TIPO PESO']: err = check_weight(col) if err: return err for col in ['TIPO MONEDA']: err = check_currency(col) if err: return err for col in ['TIPO TRANSPORTE']: err = check_transport_type(col) if err: return err # B. Invoice Details (Parts) elif target == 'invoice_details': # 1. Line Number if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'): return {"line": line_num, "col": "LINEA", "msg": "Requerido"} # 2. Parent Link (Invoice Number) if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')): return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} # 3. Numeric Fields for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']: err = check_decimal(col) if err: return err for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']: err = check_int(col) if err: return err return None def validate_row_strict( row: Dict[str, Any], target: str, line_num: int, date_format: Optional[str], validator: ForeignKeyValidator, inv_type_value: str, invoice_id_cache: Dict[str, Optional[int]], models: Dict[str, Any], ) -> Optional[Dict[str, Any]]: err = validate_row_phase_1(row, target, line_num, date_format) if err: return err InvoiceHeader = models["InvoiceHeader"] InvoiceType = models["InvoiceType"] ClientProvider = models["ClientProvider"] CustomsBroker = models["CustomsBroker"] RegimenPedimento = models["RegimenPedimento"] CurrencyType = models["CurrencyType"] CustomsSection = models["CustomsSection"] Incoterm = models["Incoterm"] Part = models["Part"] if target == "invoice_header": if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"} err = _validate_client_provider_ref( validator, ClientProvider, row.get("CLAVE PROVEEDOR"), line_num, "CLAVE PROVEEDOR", required=True ) if err: return err err = _validate_client_provider_ref( validator, ClientProvider, row.get("CLAVE VENDIDO A"), line_num, "CLAVE VENDIDO A", required=True ) if err: return err err = _validate_client_provider_ref( validator, ClientProvider, row.get("CLAVE ENVIADO A"), line_num, "CLAVE ENVIADO A", required=True ) if err: return err err = _validate_customs_broker_ref( validator, CustomsBroker, row.get("AGENTE ADUANAL"), line_num, "AGENTE ADUANAL", required=False ) if err: return err err = validate_public_code( validator, RegimenPedimento, row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"), line_num, "CLAVEDOCUMENTO", ) if err: return err err = validate_public_code( validator, CustomsSection, row.get("ADUANA DE CRUCE"), line_num, "ADUANA DE CRUCE", field_name="customs_code", ) if err: return err err = validate_public_code( validator, CurrencyType, row.get("CLAVE MONEDA"), line_num, "CLAVE MONEDA", ) if err: return err err = validate_public_code( validator, Incoterm, row.get("CLAVE INCOTERM"), line_num, "CLAVE INCOTERM", ) if err: return err elif target == "invoice_details": invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() if not invoice_number: return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} cache_key = f"{invoice_number}|{inv_type_value}" if cache_key in invoice_id_cache: invoice_id = invoice_id_cache[cache_key] else: invoice_id = ( validator.session.query(InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == validator.tenant_id, InvoiceHeader.company_id == validator.company_id, InvoiceHeader.invoice_number == invoice_number, InvoiceHeader.invoice_type == inv_type_value, ) .scalar() ) invoice_id_cache[cache_key] = invoice_id if not invoice_id: return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"} part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip() if not part_num: return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} if not validator.check_exists(Part, part_num, field_name="part_number"): return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"} return None def parse_footer_config(config: Optional[str]) -> Dict[str, Any]: if not config: return {} try: if isinstance(config, str): return json.loads(config) if isinstance(config, dict): return config except Exception: return {} return {} def display_date_format(date_format: Optional[str]) -> str: if not date_format: return "YYYY-MM-DD" return date_format.upper() def parse_date(date_text: Optional[str], date_format: Optional[str]) -> Optional[datetime.date]: if not date_text: return None candidates = [] fmt_map = { "dd/mm/yyyy": "%d/%m/%Y", "mm/dd/yyyy": "%m/%d/%Y", "yyyy-mm-dd": "%Y-%m-%d", } if date_format and date_format in fmt_map: candidates.append(fmt_map[date_format]) candidates.extend(["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y"]) for fmt in candidates: try: return datetime.strptime(str(date_text).strip(), fmt).date() except ValueError: continue return None def is_valid_date(date_text: Optional[str], date_format: Optional[str]) -> bool: return parse_date(date_text, date_format) is not None def normalize_header(name: Optional[str]) -> str: if not name: return "" name = unicodedata.normalize("NFKD", str(name)).upper() name = "".join(ch for ch in name if not unicodedata.combining(ch)) name = re.sub(r"[^A-Z0-9]+", " ", name) return re.sub(r"\s+", " ", name).strip() def normalize_row(row: Dict[str, Any]) -> Dict[str, Any]: return {normalize_header(k): v for k, v in row.items()} def parse_int(value: Any) -> Optional[int]: if value is None: return None text = str(value).strip() if not text: return None try: return int(text) except ValueError: pass try: f = float(text.replace(",", "")) if f == int(f): return int(f) return None except ValueError: return None def parse_decimal(value: Any) -> Optional[Decimal]: if value is None: return None text = str(value).strip() if not text: return None text = text.replace(",", "") try: return Decimal(text) except Exception: return None def decimal_or_zero(value: Any) -> Decimal: """Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default).""" return parse_decimal(value) or Decimal("0") def int_or_zero(value: Any) -> int: """Return parsed int or 0 for CSV nulls/empty (vanilla default).""" return parse_int(value) if parse_int(value) is not None else 0 def parse_currency(value: Optional[str], currency_type: Optional[str]): from api.v1.modules.a76.invoices.models import Currency if value: normalized = normalize_header(value) if normalized in {"MN", "M N", "NACIONAL", "LOCAL", "PESOS", "PESO"}: return Currency.LOCAL if normalized in {"ME", "M E", "EXTRANJERA", "EXTRANJERO", "FOREIGN", "USD", "DOLAR", "DOLARES"}: return Currency.FOREIGN if "MANUAL" in normalized: return Currency.MANUAL if currency_type and str(currency_type).strip().upper() == "MXN": return Currency.LOCAL if currency_type: return Currency.FOREIGN return Currency.MANUAL def parse_weight_unit(value: Optional[str]): from api.v1.modules.a76.invoices.models import WeightUnit if not value: return None normalized = normalize_header(value) if normalized in {"KG", "KGS", "KILOS", "KILOGRAMOS"}: return WeightUnit.KGS if normalized in {"LB", "LBS", "LIBRAS"}: return WeightUnit.LBS return None def resolve_tenant_fk_id( session: CoreSessionLocal, model, value: Optional[int], tenant_id: int, company_id: int, cache: Dict[int, Optional[int]], ) -> Optional[int]: if value is None: return None if value in cache: return cache[value] exists = ( session.query(model.id) .filter( model.id == value, model.tenant_id == tenant_id, model.company_id == company_id, ) .scalar() ) cache[value] = value if exists is not None else None return cache[value] def resolve_client_provider_id( session: CoreSessionLocal, model, value: Any, tenant_id: int, company_id: int, cache: Dict[Any, Optional[int]], ) -> Optional[int]: """Resuelve ID de ClientProvider por id (entero) o por short_name (texto). value puede ser int o str.""" if value is None or (isinstance(value, str) and not value.strip()): return None if value in cache: return cache[value] pid = parse_int(value) if pid is not None: found = ( session.query(model.id) .filter( model.id == pid, model.tenant_id == tenant_id, model.company_id == company_id, ) .scalar() ) cache[value] = found return found short_norm = str(value).strip().upper() if short_norm in cache: return cache[short_norm] found = ( session.query(model.id) .filter( func.upper(model.short_name) == short_norm, model.tenant_id == tenant_id, model.company_id == company_id, ) .scalar() ) cache[value] = found cache[short_norm] = found return found def resolve_customs_broker_id( session: CoreSessionLocal, model, value: Any, tenant_id: int, company_id: int, cache: Dict[Any, Optional[int]], ) -> Optional[int]: """Resuelve ID de CustomsBroker por id (entero) o por broker_key (clave). value puede ser int o str.""" if value is None or (isinstance(value, str) and not value.strip()): return None if value in cache: return cache[value] pid = parse_int(value) if pid is not None: found = ( session.query(model.id) .filter( model.id == pid, model.tenant_id == tenant_id, model.company_id == company_id, ) .scalar() ) cache[value] = found return found clave = str(value).strip() if clave in cache: return cache[clave] found = ( session.query(model.id) .filter( model.broker_key == clave, model.tenant_id == tenant_id, model.company_id == company_id, ) .scalar() ) cache[value] = found cache[clave] = found return found def resolve_public_code( session: CoreSessionLocal, model, column, value: Optional[str], cache: Dict[str, Optional[str]], ) -> Optional[str]: if not value: return None normalized = str(value).strip().upper() if not normalized: return None if normalized in cache: return cache[normalized] exists = session.query(column).filter(column == normalized).scalar() cache[normalized] = normalized if exists is not None else None return cache[normalized] @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str, model_target: str, job_type_override: Optional[str] = None): """Pass 2: Re-read CSV, Skip Errors, Bulk Insert. Delegates to _do_insert_valid_rows.""" return _do_insert_valid_rows(job_id, model_target, job_type_override) def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Optional[str] = None) -> Dict[str, Any]: """ Pass 2: Re-read CSV, Skip Errors, Bulk Insert. File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem. When job_type_override is set (e.g. "exp" for Exportación), storage keys use that prefix. """ effective_job_type = job_type_override if job_type_override is not None else JOB_TYPE log_prefix = "Exportación import" if effective_job_type else "Invoices import" logger.info(f"Starting Commit for {job_id} target {model_target}") # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) file_path = common_storage.ensure_file_from_redis(effective_job_type, job_id, log_prefix) if not file_path: alt_path = common_storage.file_path_for_job(effective_job_type, job_id) if not os.path.exists(alt_path): return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} file_path = alt_path else: common_storage.ensure_meta_from_redis(effective_job_type, job_id, file_path, log_prefix) try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) meta_path = common_meta.get_meta_path(file_path) error_path = common_storage.error_path_for_job(effective_job_type, job_id) error_lines = common_storage.get_error_lines(effective_job_type, job_id, error_path) # Si el upload fue de series (template_id imp_temp_series o imp_def_series), usar flujo series aunque model_target venga mal use_series_flow = ( model_target == "invoice_series" or meta.get("template_id") in ("imp_temp_series", "imp_def_series", "cmex_series", "exp_def_series") ) _footer_for_series = parse_footer_config(meta.get("footer_config")) or {} _inv_type_series = normalize_public_code(_footer_for_series.get("invoice_type") or meta.get("invoice_type") or "") use_def_series_commit = ( use_series_flow and ( meta.get("template_id") == "imp_def_series" or meta.get("template_id") == "cmex_series" or _inv_type_series in ("DEF", "MATDE", "EXDEF") ) ) use_expo_series_commit = use_series_flow and meta.get("template_id") == "exp_def_series" # --- Series de Exportación Definitiva: commit (INSERT/UPDATE item_line_series para facturas exp) --- if use_expo_series_commit: try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from .validators.series_expo import row_to_series_normalized_expo actualizar = meta.get("actualizar", False) autonumerar = meta.get("autonumerar", True) _fc = parse_footer_config(meta.get("footer_config")) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) else: as_val = _fc.get("autonumber_series", "true") autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") with CoreSessionLocal() as session: q_inv = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "exp", ) ) invoice_id_by_number: Dict[str, int] = {} for num, iid in q_inv.all(): if num: invoice_id_by_number[str(num).strip()] = iid inserted_count = 0 updated_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] with open(file_path, "r", encoding="utf-8-sig") as f: sample = f.read(2048) f.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f, dialect=dialect) for i, row in enumerate(reader, start=1): if i in error_lines: skipped_invalid += 1 continue row_norm = row_from_template(row, "exp_def_series", normalize_header) data = row_to_series_normalized_expo(row_norm) invoice_number = data["NUMERO FACTURA"] linea_factura = data["LINEA FACTURA"] linea_serie = data["LINEA SERIE"] if not invoice_number or invoice_number not in invoice_id_by_number: skipped_invalid += 1 skipped_details.append({ "line": i, "invoice": invoice_number or "(vacío)", "reason": "Factura de exportación no encontrada.", }) continue invoice_id = invoice_id_by_number[invoice_number] line_number_val = parse_int(linea_factura) if line_number_val is None: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."}) continue line_item = ( session.query(LineItem) .filter( LineItem.invoice_id == invoice_id, LineItem.line_number == line_number_val, LineItem.tenant_id == tenant_id, LineItem.company_id == company_id, ) .first() ) if not line_item: line_numbers = [ r[0] for r in session.query(LineItem.line_number) .filter( LineItem.invoice_id == invoice_id, LineItem.tenant_id == tenant_id, LineItem.company_id == company_id, ) .order_by(LineItem.line_number) .all() ] existing_str = ", ".join(str(n) for n in line_numbers) if line_numbers else "ninguna" skipped_invalid += 1 skipped_details.append({ "line": i, "invoice": invoice_number, "reason": f"Partida línea {linea_factura} no existe en la factura. Partidas existentes: {existing_str}.", }) continue if autonumerar: max_row = ( session.query(Serie.row) .filter(Serie.line_item_id == line_item.id) .order_by(Serie.row.desc()) .limit(1) .scalar() ) row_num = (max_row or 0) + 1 else: row_num = parse_int(linea_serie) if row_num is None: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."}) continue existing_serie = ( session.query(Serie) .filter( Serie.line_item_id == line_item.id, Serie.row == row_num, ) .first() ) if existing_serie: if actualizar: existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers existing_serie.model = data["MODELO"] or existing_serie.model existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id session.add(existing_serie) updated_count += 1 else: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."}) else: new_serie = Serie( tenant_id=tenant_id, company_id=company_id, line_item_id=line_item.id, row=row_num, serial_numbers=data["SERIE"] or None, model=data["MODELO"] or None, sub_model=data["SUB MODELO"] or None, number_id=data["NUMERO ID"] or None, ) session.add(new_serie) inserted_count += 1 session.commit() common_storage.cleanup_import_job(effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path) status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed") out = { "status": status, "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_missing_fk": 0, "skipped_duplicate": 0, "skipped_details": skipped_details, } if status == "failed": out["error"] = "No hay registros válidos en el archivo CSV." elif status == "warning" and skipped_invalid: out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados." return out except Exception as e: logger.exception("Series exportación definitiva commit failed: %s", e) return {"status": "failed", "error": str(e)} # --- Series de Importación Definitiva: commit (INSERT/UPDATE item_line_series para facturas DEF o MEX) --- if use_def_series_commit: try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from api.v1.modules.a76.items.line_quantities.models import LineQuantity from .validators.series_impo_def import ( validate_row_series_impo_def, row_to_series_normalized_def, ) _series_template_id = meta.get("template_id") is_cmex_series = _series_template_id == "cmex_series" SERIES_INV_TYPES = ("MEX",) if is_cmex_series else ("DEF", "MATDE", "EXDEF") series_row_template_id = "cmex_series" if is_cmex_series else "imp_def_series" DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") # keep for any legacy reference actualizar = meta.get("actualizar", False) autonumerar = meta.get("autonumerar", True) validar_series_exception = meta.get("validar_series", False) _fc = parse_footer_config(meta.get("footer_config")) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) elif _fc.get("mode") == "update": actualizar = True elif _fc.get("mode") == "replace": actualizar = False if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) else: as_val = _fc.get("autonumber_series", "true") autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") if "validar_series" in _fc: validar_series_exception = bool(_fc["validar_series"]) with CoreSessionLocal() as session: q = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(SERIES_INV_TYPES), ) ) rows_inv = q.all() invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in rows_inv: if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) partida_max_series = {} q_qty = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, LineQuantity.quantity, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .outerjoin(LineQuantity, LineQuantity.item_line_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(SERIES_INV_TYPES), ) ) for num, ln, qty in q_qty.all(): if num is not None and ln is not None: key = (str(num).strip(), str(ln).strip()) partida_max_series[key] = int(qty) if qty else 0 existing_series_keys = set() existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} if actualizar and not autonumerar: q_ser = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, Serie.row, Serie.serial_numbers, Serie.model, Serie.sub_model, Serie.number_id, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(Serie, Serie.line_item_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type.in_(SERIES_INV_TYPES), ) ) for num, ln, rw, sn, md, sm, nid in q_ser.all(): if num is not None: k = (str(num).strip(), str(ln).strip(), str(rw).strip()) existing_series_keys.add(k) existing_series_data.setdefault(k, { "serial_numbers": sn or "", "model": md or "", "sub_model": sm or "", "number_id": nid or "", }) csv_series_count_so_far: Dict[Tuple[str, str], int] = {} inserted_count = 0 updated_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] with open(file_path, "r", encoding="utf-8-sig") as f: sample = f.read(2048) f.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f, dialect=dialect) for i, row in enumerate(reader, start=1): if i in error_lines: continue row_norm = row_from_template(row, series_row_template_id, normalize_header) err = validate_row_series_impo_def( row_norm, i, actualizar=actualizar, autonumerar=autonumerar, validar_series_exception=validar_series_exception, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, partida_max_series=partida_max_series, csv_series_count_so_far=csv_series_count_so_far, existing_series_keys=existing_series_keys, existing_series_data=existing_series_data, warnings=None, catalog_label="Compras Mexicanas" if is_cmex_series else "Importación Definitiva", ) if err: skipped_invalid += 1 skipped_details.append({ "line": i, "invoice": (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip(), "reason": err.get("msg", ""), }) continue data = row_to_series_normalized_def(row_norm) invoice_number = data["NUMERO FACTURA"] linea_factura = data["LINEA FACTURA"] linea_serie = data["LINEA SERIE"] if not invoice_number or invoice_number not in invoice_id_by_number: skipped_invalid += 1 continue invoice_id = invoice_id_by_number[invoice_number] line_number_val = parse_int(linea_factura) if line_number_val is None: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."}) continue line_item = ( session.query(LineItem) .filter( LineItem.invoice_id == invoice_id, LineItem.line_number == line_number_val, LineItem.tenant_id == tenant_id, LineItem.company_id == company_id, ) .first() ) if not line_item: line_numbers = [ r[0] for r in session.query(LineItem.line_number) .filter( LineItem.invoice_id == invoice_id, LineItem.tenant_id == tenant_id, LineItem.company_id == company_id, ) .order_by(LineItem.line_number) .all() ] existing_str = ", ".join(str(n) for n in line_numbers) if line_numbers else "ninguna" skipped_invalid += 1 skipped_details.append({ "line": i, "invoice": invoice_number, "reason": f"Partida línea {linea_factura} no existe en la factura. Partidas existentes en la factura: {existing_str}.", }) continue if autonumerar: max_row = ( session.query(Serie.row) .filter(Serie.line_item_id == line_item.id) .order_by(Serie.row.desc()) .limit(1) .scalar() ) row_num = (max_row or 0) + 1 else: row_num = parse_int(linea_serie) if row_num is None: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."}) continue existing_serie = ( session.query(Serie) .filter( Serie.line_item_id == line_item.id, Serie.row == row_num, ) .first() ) if existing_serie: if actualizar: existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers existing_serie.model = data["MODELO"] or existing_serie.model existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id session.add(existing_serie) updated_count += 1 else: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."}) else: new_serie = Serie( tenant_id=tenant_id, company_id=company_id, line_item_id=line_item.id, row=row_num, serial_numbers=data["SERIE"] or None, model=data["MODELO"] or None, sub_model=data["SUB MODELO"] or None, number_id=data["NUMERO ID"] or None, ) session.add(new_serie) inserted_count += 1 key_csv = (invoice_number.strip(), (linea_factura or "").strip()) if key_csv[0] and key_csv[1]: csv_series_count_so_far[key_csv] = csv_series_count_so_far.get(key_csv, 0) + 1 session.commit() common_storage.cleanup_import_job(effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path) status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed") out = { "status": status, "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_missing_fk": 0, "skipped_duplicate": 0, "skipped_details": skipped_details, } if status == "failed": out["error"] = "No hay registros válidos en el archivo CSV." elif status == "warning" and skipped_invalid: out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados." return out except Exception as e: logger.exception("Series importación definitiva commit failed: %s", e) return {"status": "failed", "error": str(e)} # --- Series de Importación Temporal: commit (INSERT/UPDATE item_line_series) --- # Paridad CSV: campos no presentes en CSV se persisten como null; no se exigen campos que no están en la plantilla. if use_series_flow: try: from api.v1.modules.a76.invoices.models import InvoiceHeader from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.series.models import Serie from .validators.series_impo_temp import ( validate_row_series_impo_temp, row_to_series_normalized, ) actualizar = meta.get("actualizar", False) autonumerar = meta.get("autonumerar", True) validar_series_exception = meta.get("validar_series", False) _fc = parse_footer_config(meta.get("footer_config")) if _fc: if "actualizar" in _fc: actualizar = bool(_fc["actualizar"]) elif _fc.get("mode") == "update": actualizar = True elif _fc.get("mode") == "replace": actualizar = False if "autonumerar" in _fc: autonumerar = bool(_fc["autonumerar"]) else: as_val = _fc.get("autonumber_series", "true") autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") if "validar_series" in _fc: validar_series_exception = bool(_fc["validar_series"]) with CoreSessionLocal() as session: q = ( session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.status) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) rows_inv = q.all() invoice_id_by_number: Dict[str, int] = {} invoice_processed_by_number: Dict[str, bool] = {} for num, iid, is_upd in rows_inv: if num: invoice_id_by_number[str(num).strip()] = iid invoice_processed_by_number[str(num).strip()] = bool(is_upd) existing_series_keys: Set[Tuple[str, str, str]] = set() existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} if actualizar and not autonumerar: q_ser = ( session.query( InvoiceHeader.invoice_number, LineItem.line_number, Serie.row, Serie.serial_numbers, Serie.model, Serie.sub_model, Serie.number_id, ) .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) .join(Serie, Serie.line_item_id == LineItem.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.operation_type == "imp", InvoiceHeader.invoice_type == "TEM", ) ) for num, ln, rw, sn, md, sm, nid in q_ser.all(): if num is not None: k = (str(num).strip(), str(ln).strip(), str(rw).strip()) existing_series_keys.add(k) existing_series_data.setdefault(k, { "serial_numbers": sn or "", "model": md or "", "sub_model": sm or "", "number_id": nid or "", }) inserted_count = 0 updated_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] with open(file_path, "r", encoding="utf-8-sig") as f: sample = f.read(2048) f.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except Exception: dialect = "excel" reader = csv.DictReader(f, dialect=dialect) for i, row in enumerate(reader, start=1): if i in error_lines: continue row_norm = row_from_template(row, "imp_temp_series", normalize_header) err = validate_row_series_impo_temp( row_norm, i, actualizar=actualizar, autonumerar=autonumerar, validar_series_exception=validar_series_exception, invoice_id_by_number=invoice_id_by_number, invoice_processed_by_number=invoice_processed_by_number, existing_series_keys=existing_series_keys, existing_series_data=existing_series_data, warnings=None, ) if err: skipped_invalid += 1 skipped_details.append({ "line": i, "invoice": (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip(), "reason": err.get("msg", ""), }) continue data = row_to_series_normalized(row_norm) invoice_number = data["NUMERO FACTURA"] linea_factura = data["LINEA FACTURA"] linea_serie = data["LINEA SERIE"] if not invoice_number or invoice_number not in invoice_id_by_number: skipped_invalid += 1 continue invoice_id = invoice_id_by_number[invoice_number] line_number_val = parse_int(linea_factura) if line_number_val is None: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."}) continue line_item = ( session.query(LineItem) .filter( LineItem.invoice_id == invoice_id, LineItem.line_number == line_number_val, LineItem.tenant_id == tenant_id, LineItem.company_id == company_id, ) .first() ) if not line_item: skipped_invalid += 1 skipped_details.append({ "line": i, "invoice": invoice_number, "reason": f"Partida línea {linea_factura} no existe en la factura.", }) continue if autonumerar: max_row = ( session.query(Serie.row) .filter(Serie.line_item_id == line_item.id) .order_by(Serie.row.desc()) .limit(1) .scalar() ) row_num = (max_row or 0) + 1 else: row_num = parse_int(linea_serie) if row_num is None: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."}) continue existing_serie = ( session.query(Serie) .filter( Serie.line_item_id == line_item.id, Serie.row == row_num, ) .first() ) if existing_serie: if actualizar: existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers existing_serie.model = data["MODELO"] or existing_serie.model existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id session.add(existing_serie) updated_count += 1 else: skipped_invalid += 1 skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."}) else: new_serie = Serie( tenant_id=tenant_id, company_id=company_id, line_item_id=line_item.id, row=row_num, serial_numbers=data["SERIE"] or None, model=data["MODELO"] or None, sub_model=data["SUB MODELO"] or None, number_id=data["NUMERO ID"] or None, ) session.add(new_serie) inserted_count += 1 session.commit() common_storage.cleanup_import_job(effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path) status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed") out = { "status": status, "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_missing_fk": 0, "skipped_duplicate": 0, "skipped_details": skipped_details, } if status == "failed": out["error"] = "No hay registros válidos en el archivo CSV." elif status == "warning" and skipped_invalid: out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados." return out except Exception as e: logger.exception("Series import commit failed: %s", e) return {"status": "failed", "error": str(e)} try: from api.v1.modules.a76.invoices.models import ( InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials, InvoiceLogistics, InvoiceSalesDetails, InvoiceStatus, OperationType, TransportType, WeightUnit, ) from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection from api.v1.modules.public.reference_data.incoterms.models import Incoterm from .validators.encabezados_impo_temp import ( parse_pedimento_col_a, _pedimento_key_from_parsed, row_to_transport_type_clarion, ) from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def from api.v1.modules.a76.items.models import LineItem from api.v1.modules.a76.items.line_financials.models import LineFinancial from api.v1.modules.a76.items.line_quantities.models import LineQuantity from api.v1.modules.a76.items.line_customs.models import LineCustom from api.v1.modules.a76.items.line_descriptions.models import LineDescription from api.v1.modules.a76.items.schemas import LineItemCreate from api.v1.modules.a76.items.line_financials.schemas import LineFinancialCreate from api.v1.modules.a76.items.line_quantities.schemas import LineQuantityCreate from api.v1.modules.a76.items.line_customs.schemas import LineCustomCreate from api.v1.modules.a76.items.line_descriptions.schemas import LineDescriptionCreate from api.v1.modules.a24.fa.fa_item_lines.dto import FaLineItemCreateDTO from api.v1.modules.a76.layouts_csv.facturas.line_item_enrichment import ( apply_import_defaults_and_calculations_for_csv, apply_export_defaults_and_calculations_for_csv, ) from api.v1.modules.a76.items.service import ItemService from api.v1.modules.a76.parts.models import Part from api.v1.modules.a76.classes.models import Class from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure from api.v1.modules.a76.general_catalogs.packages.models import Package footer_config = parse_footer_config(meta.get("footer_config")) date_format = footer_config.get("dateFormat") # Validate and set default date_format if not provided if not date_format: date_format = "yyyy-mm-dd" # Default to ISO format logger.info(f"No date_format specified in config, using default: {date_format}") else: logger.info(f"Using date_format from config: {date_format}") # Default types from config or fallback op_type_value = OperationType(meta.get('operation_type', 'imp').lower()) inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM' _template_id_insert = meta.get("template_id") or ( "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" ) if job_type_override == "exp" and model_target == "invoice_details": _template_id_insert = "exp_def_partidas" if job_type_override == "exp" and model_target == "invoice_series": _template_id_insert = "exp_def_series" if model_target == "invoice_header" and _template_id_insert == "imp_def_header": inv_type_value = "DEF" if model_target == "invoice_header" and _template_id_insert == "cmex_header": inv_type_value = "MEX" if model_target == "invoice_header" and _template_id_insert == "exp_def_header": op_type_value = OperationType("exp") inv_type_value = normalize_public_code( meta.get("tipo_factura") or footer_config.get("tipo_factura") or "AFIJO" ) or "AFIJO" _es_cambio_regimen = None if model_target == "invoice_header" and _template_id_insert == "exp_def_header": cambio_regimen_raw = ( str(meta.get("cambio_regimen") or footer_config.get("cambio_regimen") or "NO").strip().upper() ) _es_cambio_regimen = "S" if cambio_regimen_raw == "SI" else "N" if model_target == "invoice_details" and _template_id_insert == "imp_def_details": inv_type_value = "DEF" if model_target == "invoice_details" and _template_id_insert == "cmex_details": inv_type_value = "MEX" if model_target == "invoice_details" and _template_id_insert == "exp_def_partidas": op_type_value = OperationType("exp") inv_type_value = normalize_public_code( meta.get("tipo_factura") or footer_config.get("tipo_factura") or "AFIJO" ) or "AFIJO" if _template_id_insert == "cmex_series": inv_type_value = "MEX" logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") headers_to_insert = [] details_to_insert = [] skipped_invalid = 0 skipped_missing_invoice = 0 skipped_missing_fk = 0 skipped_fk_details = [] inserted_count = 0 response = None with CoreSessionLocal() as session: invoice_id_cache = {} cleared_invoices = set() # Track invoices where we've already cleared items in this job provider_cache: Dict[Any, Optional[int]] = {} sold_to_cache: Dict[Any, Optional[int]] = {} shipped_to_cache: Dict[Any, Optional[int]] = {} broker_cache: Dict[Any, Optional[int]] = {} regimen_cache: Dict[str, Optional[str]] = {} currency_type_cache: Dict[str, Optional[str]] = {} customs_section_cache: Dict[str, Optional[str]] = {} part_cache: Dict[str, Optional[int]] = {} pedimento_id_cache: Dict[str, Optional[int]] = {} shipped_by_cache: Dict[Any, Optional[int]] = {} _fc_insert = parse_footer_config(meta.get("footer_config")) autonumerar_remesas_insert = _fc_insert.get("autonumerar_remesas", False) class_id_by_code: Dict[str, int] = {} class_uom_by_code: Dict[str, Optional[str]] = {} class_fraction_by_code: Dict[str, Optional[str]] = {} class_desc_es_by_code: Dict[str, Optional[str]] = {} class_desc_en_by_code: Dict[str, Optional[str]] = {} uom_id_by_code: Dict[str, int] = {} package_id_by_key: Dict[str, int] = {} if model_target == 'invoice_details': for c in session.query( Class.id, Class.class_code, Class.unit_of_measure, Class.fraction, Class.description_es, Class.description_en, ).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): if c[1]: class_code_key = (c[1] or "").strip().upper() class_id_by_code[class_code_key] = c[0] class_uom_by_code[class_code_key] = (c[2] or "").strip().upper() or None class_fraction_by_code[class_code_key] = (c[3] or "").strip() or None class_desc_es_by_code[class_code_key] = (c[4] or "").strip() or None class_desc_en_by_code[class_code_key] = (c[5] or "").strip() or None for u in session.query(UnitOfMeasure.id, UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): if u[1]: uom_id_by_code[(u[1] or "").strip().upper()] = u[0] for p in session.query(Package.id, Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): if p[1]: package_id_by_key[(p[1] or "").strip()] = p[0] validator = ForeignKeyValidator(session, tenant_id, company_id) error_msg_by_line: Dict[int, str] = {} if error_path and os.path.exists(error_path): try: with open(error_path, "r", encoding="utf-8") as f_err: for line in f_err: line = line.strip() if not line: continue try: rec = json.loads(line) if "line" in rec and "msg" in rec: error_msg_by_line[int(rec["line"])] = str(rec["msg"]).strip() except (json.JSONDecodeError, ValueError, TypeError): pass except Exception: pass with open(file_path, 'r', encoding='utf-8-sig') as f: # Detect Delimiter sample = f.read(2048) f.seek(0) try: dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") except: dialect = 'excel' reader = csv.DictReader(f, dialect=dialect) template_id = _template_id_insert for i, row in enumerate(reader, start=1): row_norm = row_from_template(row, template_id, normalize_header) if i in error_lines: skipped_invalid += 1 if model_target == 'invoice_header': inv_for_detail = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() elif _template_id_insert == "exp_def_partidas": inv_for_detail = (row_norm.get('NUMERO FACTURA EXPO') or row_norm.get('NUMERO FACTURA EXPO.') or row_norm.get('FACTURA EXPO') or '').strip() else: inv_for_detail = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() reason = error_msg_by_line.get(i, "Línea marcada con error en el escaneo previo (revisar reporte de validación).") skipped_fk_details.append({ "line": i, "invoice": inv_for_detail or "(vacío)", "reason": reason, }) continue # Mapping Logic (solo campos que acepta el modelo de facturas) if model_target == 'invoice_header': invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format) if not invoice_number or not invoice_date: skipped_invalid += 1 reason = "Número de factura o fecha faltante/inválida" skipped_fk_details.append({"line": i, "invoice": invoice_number or "(vacío)", "reason": reason}) logger.debug(f"Row {i}: Skipped - missing invoice_number or invalid invoice_date. " f"Invoice: {invoice_number}, Date: {row_norm.get('FECHA FACTURA') or row_norm.get('FECHA')}") continue # --- NEW: Foreign Key Validations --- # 1. Invoice Type (Public) if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): skipped_missing_fk += 1 reason = f"Tipo de factura '{inv_type_value}' no existe" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue err = _validate_client_provider_ref( validator, ClientProvider, row_norm.get('CLAVE PROVEEDOR'), i, "CLAVE PROVEEDOR", required=True, ) if err: skipped_invalid += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue err = _validate_client_provider_ref( validator, ClientProvider, row_norm.get('CLAVE VENDIDO A'), i, "CLAVE VENDIDO A", required=True, ) if err: skipped_invalid += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue err = _validate_client_provider_ref( validator, ClientProvider, row_norm.get('CLAVE ENVIADO A'), i, "CLAVE ENVIADO A", required=True, ) if err: skipped_invalid += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue if inv_type_value != "MEX": err = _validate_customs_broker_ref( validator, CustomsBroker, row_norm.get('AGENTE ADUANAL'), i, "AGENTE ADUANAL", required=False, ) if err: skipped_missing_fk += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue err = validate_public_code( validator, RegimenPedimento, row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'), i, "CLAVEDOCUMENTO", ) if err: skipped_missing_fk += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue err = validate_public_code( validator, CustomsSection, row_norm.get('ADUANA DE CRUCE'), i, "ADUANA DE CRUCE", field_name="customs_code", ) if err: skipped_missing_fk += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue if inv_type_value == "MEX": err = validate_public_code( validator, CurrencyType, row_norm.get('CLAVE MONEDA'), i, "CLAVE MONEDA", ) if err and (row_norm.get('TIPO MONEDA') or '').strip().upper() == 'MC': skipped_missing_fk += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue else: err = validate_public_code( validator, CurrencyType, row_norm.get('CLAVE MONEDA'), i, "CLAVE MONEDA", ) if err: skipped_missing_fk += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue err = validate_public_code( validator, Incoterm, row_norm.get('CLAVE INCOTERM'), i, "CLAVE INCOTERM", ) if err: skipped_missing_fk += 1 reason = f"{err['col']}: {err['msg']}" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue transport_type_val = row_norm.get('TIPO TRANSPORTE') transport_str_normalized = (row_to_transport_type_clarion(transport_type_val) or str(transport_type_val or "").strip().lower() or "none") if transport_type_val and transport_str_normalized not in TRANSPORT_TYPE_VALUES: skipped_invalid += 1 reason = "TIPO TRANSPORTE: Tipo de transporte invalido" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue currency_val = row_norm.get('TIPO MONEDA') if currency_val and str(currency_val).strip(): parsed_currency = parse_currency(currency_val, None) val_norm = normalize_header(currency_val) if parsed_currency.value == "manual" and "MANUAL" not in val_norm: skipped_invalid += 1 reason = "TIPO MONEDA: Moneda invalida" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue # 2. Client/Provider and broker checks are handled above # --- 4. Check for Existing Invoice (Upsert Logic) --- existing_header = None if invoice_number: existing_header = ( session.query(InvoiceHeader) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.invoice_number == invoice_number, InvoiceHeader.invoice_type == inv_type_value, InvoiceHeader.operation_type == op_type_value, ) .first() ) # --- Resolve PEDIMENTO (Col A) to pedimento_id and REMESA (Col B) --- pedimento_id = None remesa_val = parse_int(row_norm.get('REMESA')) if inv_type_value != "MEX" else None ped_str = (row_norm.get('PEDIMENTO') or '').strip() if inv_type_value != "MEX" else '' if ped_str: if template_id == "exp_def_header": parsed = parse_pedimento_col_a_impo_def(ped_str) else: parsed = parse_pedimento_col_a(ped_str) if parsed: customs_office_p, license_p, num_p = (x.strip() if x else "" for x in parsed) key_p = _pedimento_key_from_parsed(customs_office_p, license_p, num_p) if key_p not in pedimento_id_cache: co_prefix = (customs_office_p or "").strip()[:2].zfill(2) ped_query = ( session.query(Pedimentos.id) .filter( Pedimentos.tenant_id == tenant_id, Pedimentos.company_id == company_id, Pedimentos.customs_office.startswith(co_prefix), Pedimentos.license == license_p, Pedimentos.pedimento_number == num_p, ) ) if template_id == "exp_def_header": if _es_cambio_regimen == "S": ped_query = ped_query.filter( func.lower(Pedimentos.operation_type) == "imp", func.upper(Pedimentos.regime) == "IMD", ) else: ped_query = ped_query.filter( func.lower(Pedimentos.operation_type) == "exp", func.upper(Pedimentos.regime).in_(["EXD", "ETE", "ETR"]), ) ped_row = ped_query.first() pedimento_id_cache[key_p] = ped_row[0] if ped_row else None pedimento_id = pedimento_id_cache[key_p] if pedimento_id is not None and remesa_val is None and autonumerar_remesas_insert: max_rem = ( session.query(func.max(InvoiceComplianceMx.remesa)) .filter(InvoiceComplianceMx.pedimento_id == pedimento_id) .scalar() ) remesa_val = (max_rem or 0) + 1 if existing_header: # UPDATE existing header (paridad con InvoiceService.update) header = existing_header header.invoice_date = invoice_date header.operation_type = op_type_value # CSV import only captures data; "processed" is set by the manual/import processing flow. # Legacy CSV imports may have persisted booleans ('True'/'False') into status. if header.status not in (InvoiceStatus.PENDING, InvoiceStatus.PROCESSED, InvoiceStatus.REVERSED): header.status = InvoiceStatus.PENDING header.updated_date = datetime.utcnow() capture_user = meta.get("capture_user") or "CSV" header.who_processed = capture_user # Backfill capture_user if missing or generic (paridad con service) if not header.capture_user or header.capture_user == "System": if capture_user != "CSV": header.capture_user = capture_user header.document_type = ( None if inv_type_value == "MEX" else resolve_public_code( session, RegimenPedimento, RegimenPedimento.code, (row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')), regimen_cache, ) ) header.project_number = (row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None) header.purchase_order = (row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None) header.alternate_invoice = (row_norm.get('FACTURA ALTERNA') or None) header.invoice_ref = (row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None) header.emission_date = parse_date(row_norm.get('FECHA EMISION'), date_format) header.observation_es = (row_norm.get('OBSERVACIONES E') or None) header.observation_en = (row_norm.get('OBSERVACIONES I') or None) logger.info(f"Row {i}: Updating existing invoice {invoice_number}") # Clean up related data that will be re-inserted/updated # Note: compliance, financials, logistics are 1-to-1 relationships and will be updated by assignment below # but we might want to be explicit if ORM doesn't handle replace well. # SQLAlchemy relationship assignment usually handles 1-to-1 updates correctly. else: # CREATE new header (paridad con InvoiceService.create: capture_user, who_processed) capture_user = meta.get("capture_user") or "CSV" header = InvoiceHeader( invoice_number=invoice_number, invoice_date=invoice_date, operation_type=op_type_value, status=InvoiceStatus.PENDING, system="CSV", capture_date=datetime.utcnow(), capture_user=capture_user, who_processed=capture_user, invoice_type=inv_type_value, document_type=( None if inv_type_value == "MEX" else resolve_public_code( session, RegimenPedimento, RegimenPedimento.code, (row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')), regimen_cache, ) ), project_number=(row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None), purchase_order=(row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None), alternate_invoice=(row_norm.get('FACTURA ALTERNA') or None), invoice_ref=(row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None), emission_date=parse_date(row_norm.get('FECHA EMISION'), date_format), observation_es=(row_norm.get('OBSERVACIONES E') or None), observation_en=(row_norm.get('OBSERVACIONES I') or None), tenant_id=tenant_id, company_id=company_id, ) compliance = InvoiceComplianceMx( pedimento_id=pedimento_id if inv_type_value != "MEX" else None, remesa=remesa_val if inv_type_value != "MEX" else None, aduana=( None if inv_type_value == "MEX" else resolve_public_code( session, CustomsSection, CustomsSection.customs_code, row_norm.get('ADUANA DE CRUCE'), customs_section_cache, ) ), provider_id=resolve_client_provider_id( session, ClientProvider, row_norm.get('CLAVE PROVEEDOR'), tenant_id, company_id, provider_cache, ), sold_to_id=resolve_client_provider_id( session, ClientProvider, row_norm.get('CLAVE VENDIDO A'), tenant_id, company_id, sold_to_cache, ), shipped_to_id=resolve_client_provider_id( session, ClientProvider, row_norm.get('CLAVE ENVIADO A'), tenant_id, company_id, shipped_to_cache, ), customs_broker_id=( None if inv_type_value == "MEX" else resolve_customs_broker_id( session, CustomsBroker, row_norm.get('AGENTE ADUANAL'), tenant_id, company_id, broker_cache, ) ), edocument=(row_norm.get('E DOCUMENT') or None), vucem_operation_num=(row_norm.get('NUM OPERACION') or None), manifest_number=(row_norm.get('MANIFIESTO') or None), shipped_by_id=resolve_client_provider_id( session, ClientProvider, row_norm.get('ENVIADO POR'), tenant_id, company_id, shipped_by_cache, ) if row_norm.get('ENVIADO POR') else None, tenant_id=tenant_id, company_id=company_id, ) financials_currency_type = resolve_public_code( session, CurrencyType, CurrencyType.code, row_norm.get('CLAVE MONEDA'), currency_type_cache, ) financials = InvoiceFinancials( currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type), currency_type=financials_currency_type, exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')), freight=decimal_or_zero(row_norm.get('FLETES')), insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')), insurance=decimal_or_zero(row_norm.get('SEGUROS')), packaging=decimal_or_zero(row_norm.get('EMBALAJES')), other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')), tenant_id=tenant_id, company_id=company_id, ) weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) logistics = None if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): raw_transport = row_norm.get('TIPO TRANSPORTE') transport_str = (row_to_transport_type_clarion(raw_transport) or str(raw_transport or "").strip().lower() or "none") try: transport_type = TransportType(transport_str) except ValueError: transport_type = TransportType.NONE logistics = InvoiceLogistics( carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), transport_type=transport_type, transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), weight_type=weight_type or WeightUnit.KGS, seal_number=(row_norm.get('PRECINTO') or None), incoterm=(row_norm.get('CLAVE INCOTERM') or None), entry_exit_date=parse_date(row_norm.get('FECHA EMISION'), date_format), tenant_id=tenant_id, company_id=company_id, ) header.compliance_mx = compliance header.financials = financials if logistics: header.logistics = logistics headers_to_insert.append(header) elif model_target == 'invoice_details': if _template_id_insert == "exp_def_partidas": invoice_number = ( row_norm.get('NUMERO FACTURA EXPO') or row_norm.get('NUMERO FACTURA EXPO.') or row_norm.get('FACTURA EXPO') or '' ).strip() else: invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or '').strip() if not invoice_number: skipped_invalid += 1 continue if _template_id_insert == "exp_def_partidas": # Expo: lookup by invoice_number + operation_type only (no invoice_type filter, matching scan behavior) cache_key = f"{invoice_number}|exp" if cache_key in invoice_id_cache: invoice_id = invoice_id_cache[cache_key] else: invoice_id = ( session.query(InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.invoice_number == invoice_number, InvoiceHeader.operation_type == "exp", ) .scalar() ) invoice_id_cache[cache_key] = invoice_id else: cache_key = f"{invoice_number}|{inv_type_value}|{op_type_value.value}" if cache_key in invoice_id_cache: invoice_id = invoice_id_cache[cache_key] else: invoice_id = ( session.query(InvoiceHeader.id) .filter( InvoiceHeader.tenant_id == tenant_id, InvoiceHeader.company_id == company_id, InvoiceHeader.invoice_number == invoice_number, InvoiceHeader.invoice_type == inv_type_value, InvoiceHeader.operation_type == op_type_value, ) .scalar() ) invoice_id_cache[cache_key] = invoice_id if not invoice_id: logger.warning( "Invoice not found for details row %s (invoice_number=%s)", i, invoice_number, ) skipped_missing_invoice += 1 continue # --- Partidas Exportación Definitiva: paridad con flujo normal (validators + ItemService) --- if _template_id_insert == "exp_def_partidas": part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() part_id = part_cache.get(part_num) if part_num else None if part_id is None and part_num: p = session.query(Part.id).filter(Part.part_number == part_num, Part.tenant_id == tenant_id, Part.company_id == company_id).first() if p: part_id = p.id part_cache[part_num] = p.id line_num_val = (row_norm.get('LINEA EXPO') or row_norm.get('LINEA EXPO.') or row_norm.get('RENGLON EXPO')) line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) uom_code = (row_norm.get('UNIDAD DE MEDIDA') or row_norm.get('U.M.') or row_norm.get('UNIDAD MEDIDA') or '').strip().upper() uom_id = uom_id_by_code.get(uom_code) if uom_code else None bulk_key = (row_norm.get('CLAVE BULTOS') or row_norm.get('CLAVEBULTOS') or '').strip() package_id = package_id_by_key.get(bulk_key) if bulk_key else None descarga_val = (row_norm.get('GENERA DESCARGA') or row_norm.get('GENERA DESCARGA?') or row_norm.get('DESCARGA') or 'SI').strip().upper() tipo_impo = (row_norm.get('TIPO DE IMPO') or row_norm.get('TIPO DE IMPO.') or row_norm.get('TIPO IMPO') or row_norm.get('PROCEDENCIA') or '').strip().upper() factura_impo = (row_norm.get('FACTURA IMPO') or row_norm.get('FACTURA IMPO.') or row_norm.get('FACTURA IMPORTACION') or '').strip() linea_impo_val = (row_norm.get('LINEA IMPO') or row_norm.get('LINEA IMPO.') or row_norm.get('LINEA IMPORTACION') or '').strip() se_pago = (row_norm.get('SE PAGO IMPUESTO') or row_norm.get('SE PAGO IMPUESTO? (SI o NO)') or row_norm.get('SEPAGOIMPUESTO') or '').strip().upper() forma_pago = (row_norm.get('FORMA DE PAGO') or row_norm.get('FORMADEPAGO') or row_norm.get('FORMA PAGO') or '').strip() or None es_sub_raw = (row_norm.get('ES PARTIDA/SUBPARTIDA') or row_norm.get('ESSUBPARTIDA') or row_norm.get('ES PARTIDA O SUBPARTIDA') or '').strip().upper() linea_principal_val = (row_norm.get('LINEA PRINCIPAL') or row_norm.get('LINEAPRINCIPAL') or row_norm.get('PARTIDA PRINCIPAL') or '').strip() is_subitem = (es_sub_raw == 'S') contains_subitems = (es_sub_raw == 'P') # Clear existing line items once per invoice if invoice_id not in cleared_invoices: logger.info(f"Clearing existing details for Expo Invoice {invoice_number} (ID: {invoice_id})") session.query(LineItem).filter(LineItem.invoice_id == invoice_id).delete(synchronize_session=False) session.query(InvoiceSalesDetails).filter(InvoiceSalesDetails.invoice_id == invoice_id).delete(synchronize_session=False) cleared_invoices.add(invoice_id) price = parse_decimal(row_norm.get('COSTO UNITARIO') or row_norm.get('COSTOUNITARIO')) qty = parse_decimal(row_norm.get('CANTIDAD EXPORTADA/DESCARGAR') or row_norm.get('CANTIDAD EXPORTADA') or row_norm.get('CANTIDAD')) commercial_total = (price * qty) if price and qty else None net_w = parse_decimal(row_norm.get('PESO NETO') or row_norm.get('PESONETO')) gross_w = parse_decimal(row_norm.get('PESO BRUTO') or row_norm.get('PESOBRUTO')) origin = (row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') or row_norm.get('PAIS') or '').strip() fraction = (row_norm.get('FRACCION ARANCELARIA') or row_norm.get('FRACCIONARANCELARIA') or '').strip() american_fraction = (row_norm.get('FRACCION AMERICANA') or row_norm.get('FRACCIONAMERICANA') or '').strip() extra_desc = (row_norm.get('DESCRIPCION EXTRA') or row_norm.get('DESCRIPCIONEXTRA') or '').strip() additional_info = (row_norm.get('INFORMACION ADICIONAL') or row_norm.get('INFORMACIONADICIONAL') or '').strip() lot = (row_norm.get('LOTE') or '').strip() entry_number = (row_norm.get('NUMERO ENTRADA') or row_norm.get('NUM ENTRADA') or '').strip() order_compra = (row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or row_norm.get('ORDEN DE VENTA') or None) line_data = LineItemCreate( invoice_id=invoice_id, line_number=line_num, part_number_id=part_id, class_id=None, unit_of_measure=uom_id, order=order_compra, tax_payment=(se_pago == 'SI'), payment_method=forma_pago, financial=LineFinancialCreate( unit_cost_capture=decimal_or_zero(price), total_commercial_value=decimal_or_zero(commercial_total), ), quantity=LineQuantityCreate( quantity=decimal_or_zero(qty), net_weight=decimal_or_zero(net_w), gross_weight=decimal_or_zero(gross_w), package_quantity=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), package_id=package_id, ), customs=LineCustomCreate( origin_country=origin or None, fraction=fraction or None, american_fraction=american_fraction or None, ), description=LineDescriptionCreate( extra_description=extra_desc or None, additional_info_spanish=additional_info or None, lot=lot or None, entry_number=entry_number or None, ), fa_data=FaLineItemCreateDTO( search_invoice=factura_impo or None, search_line=parse_int(linea_impo_val), search_type=tipo_impo or None, download=(descarga_val == 'SI'), is_subitem=is_subitem, contains_subitems=contains_subitems, subitem_number=parse_int(linea_principal_val) if is_subitem else 0, ), ) if not apply_export_defaults_and_calculations_for_csv( session, line_data, tenant_id, company_id, line_num ): skipped_invalid += 1 skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": "Factura sin datos financieros para enriquecer partida de exportación."}) continue item_dict = line_data.model_dump( exclude={"financial", "quantity", "customs", "description", "reference", "fa_data", "series"} ) item_dict["tenant_id"] = tenant_id item_dict["company_id"] = company_id item_dict["line_number"] = line_num line = LineItem(**item_dict) session.add(line) session.flush() ItemService._create_line_nested_data(session, line, line_data, tenant_id, company_id) session.add(InvoiceSalesDetails( invoice_id=invoice_id, line_number=line_num, sales_order=order_compra, line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), tenant_id=tenant_id, company_id=company_id, )) details_to_insert.append(line) continue part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() if not part_num: skipped_invalid += 1 reason = "NUMPARTE: Requerido" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue if not validator.check_exists(Part, part_num, field_name="part_number"): skipped_missing_fk += 1 reason = f"NUMPARTE '{part_num}' no existe" skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") continue # --- Prevent Duplicates: Clear existing line items for this invoice (Once per job) --- if invoice_id not in cleared_invoices: logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates") session.query(LineItem).filter(LineItem.invoice_id == invoice_id).delete(synchronize_session=False) session.query(InvoiceSalesDetails).filter(InvoiceSalesDetails.invoice_id == invoice_id).delete(synchronize_session=False) cleared_invoices.add(invoice_id) # --- Partidas importación: paridad con flujo normal (validators + ItemService) --- # LineReference: solo se crea si line_data.reference viene informado; no inventar datos sin fuente (plan paridad CSV). # Build LineItemCreate from CSV, apply import defaults/calculations, then persist. part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() part_id = part_cache.get(part_num) if part_num else None if part_id is None and part_num: p = session.query(Part.id).filter(Part.part_number == part_num, Part.tenant_id == tenant_id, Part.company_id == company_id).first() if p: part_id = p.id part_cache[part_num] = part_id line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA')) line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) class_code = (row_norm.get('CLASE') or '').strip().upper() class_id = class_id_by_code.get(class_code) if class_code else None uom_code = (row_norm.get('UNIDAD DE MEDIDA') or row_norm.get('UNIDAD MEDIDA') or '').strip().upper() if not uom_code and class_code: uom_code = class_uom_by_code.get(class_code) or '' uom_id = uom_id_by_code.get(uom_code) if uom_code else None bulk_key = (row_norm.get('CLAVE BULTOS') or row_norm.get('CLAVEBULTOS') or '').strip() package_id = package_id_by_key.get(bulk_key) if bulk_key else None price = parse_decimal(row_norm.get('COSTO UNITARIO') or row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO')) if price is None: total_val = parse_decimal(row_norm.get('TOTAL')) qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD')) price = (total_val / qty) if (total_val and qty and qty != 0) else None qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD')) commercial_total = (price * qty) if price and qty else parse_decimal(row_norm.get('TOTAL')) net_w = parse_decimal(row_norm.get('PESO NETO') or row_norm.get('PESONETO')) gross_w = parse_decimal(row_norm.get('PESO BRUTO') or row_norm.get('PESOBRUTO')) origin = (row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') or row_norm.get('PAIS') or '').strip() fraction = (row_norm.get('FRACCION ARANCELARIA') or row_norm.get('FRACCION') or row_norm.get('FRACCIONARANCELARIA') or '').strip() if not fraction and class_code: fraction = class_fraction_by_code.get(class_code) or '' fraction_type = (row_norm.get('PREFERENCIA ARANCELARIA') or row_norm.get('PREFERENCIA') or '').strip() sector = (row_norm.get('SECTOR') or '').strip() american_fraction = (row_norm.get('FRACCION AMERICANA') or row_norm.get('FRACCIONAMERICANA') or '').strip() desc_es = (row_norm.get('DESCRIPCION ESPAÑOL') or row_norm.get('DESCRIPCIONE') or row_norm.get('DESCRIPCION') or '').strip() if not desc_es and class_code: desc_es = class_desc_es_by_code.get(class_code) or '' desc_en = (row_norm.get('DESCRIPCION INGLES') or row_norm.get('DESCRIPCIONI') or '').strip() if not desc_en and class_code: desc_en = class_desc_en_by_code.get(class_code) or '' brand = (row_norm.get('MARCA') or '').strip() model = (row_norm.get('MODELO') or '').strip() extra_desc = (row_norm.get('DESCRIPCION EXTRA') or row_norm.get('DESCRIPCIONEXTRA') or '').strip() additional_info = (row_norm.get('INFORMACION ADICIONAL') or row_norm.get('INFORMACIONADICIONAL') or '').strip() lot = (row_norm.get('LOTE') or '').strip() entry_number = (row_norm.get('NUMERO ENTRADA') or row_norm.get('NUMEROENTRADA') or row_norm.get('NUM ENTRADA') or '').strip() line_data = LineItemCreate( invoice_id=invoice_id, line_number=line_num, part_number_id=part_id, class_id=class_id, unit_of_measure=uom_id, order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), material_type=(row_norm.get('ID TYPE') or row_norm.get('IDTYPE') or None), tax_payment=(str(row_norm.get('SE PAGO IMPUESTO') or row_norm.get('SEPAGOIMPUESTO') or '').strip().upper() == 'SI'), payment_method=(row_norm.get('FORMA DE PAGO') or row_norm.get('FORMADEPAGO') or row_norm.get('FORMA PAGO') or None), valuation_method=(row_norm.get('METODO DE VALORACION') or row_norm.get('METODODEVALORACION') or row_norm.get('METODO VALORACION') or None), financial=LineFinancialCreate( unit_cost_capture=decimal_or_zero(price), total_commercial_value=decimal_or_zero(commercial_total), ), quantity=LineQuantityCreate( quantity=decimal_or_zero(qty), net_weight=decimal_or_zero(net_w), gross_weight=decimal_or_zero(gross_w), package_quantity=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), package_id=package_id, ), customs=LineCustomCreate( origin_country=origin or None, fraction=fraction or None, fraction_type=fraction_type or None, sector=sector or None, american_fraction=american_fraction or None, ), description=LineDescriptionCreate( description_spanish=desc_es or None, description_english=desc_en or None, brand=brand or None, model=model or None, extra_description=extra_desc or None, additional_info_spanish=additional_info or None, lot=lot or None, entry_number=entry_number or None, ), fa_data=FaLineItemCreateDTO(is_subitem=False, contains_subitems=False), ) if not apply_import_defaults_and_calculations_for_csv( session, line_data, tenant_id, company_id, line_num ): skipped_invalid += 1 skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": "Factura sin datos financieros/logísticos para enriquecer partida."}) continue item_dict = line_data.model_dump( exclude={"financial", "quantity", "customs", "description", "reference", "fa_data", "series"} ) item_dict["tenant_id"] = tenant_id item_dict["company_id"] = company_id item_dict["line_number"] = line_num line = LineItem(**item_dict) session.add(line) session.flush() ItemService._create_line_nested_data(session, line, line_data, tenant_id, company_id) session.add(InvoiceSalesDetails( invoice_id=invoice_id, line_number=line_num, sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), tenant_id=tenant_id, company_id=company_id, )) details_to_insert.append(line) # 3. Bulk Insert (ORM Transaction) try: if model_target == 'invoice_header': if headers_to_insert: logger.info(f"Attempting to commit {len(headers_to_insert)} headers") session.add_all(headers_to_insert) session.commit() inserted_count = len(headers_to_insert) logger.info(f"Headers commit successful. Inserted: {inserted_count}") else: logger.warning(f"No headers to insert for job {job_id}") else: if details_to_insert: logger.info(f"Attempting to commit {len(details_to_insert)} items and related data") session.commit() # Everything was already added with session.add() inserted_count = len(details_to_insert) logger.info(f"Details commit successful. Inserted: {inserted_count}") else: logger.warning(f"No details to insert for job {job_id}") except Exception as db_err: session.rollback() logger.error(f"DB Error during {model_target} commit: {db_err}") import traceback logger.error(traceback.format_exc()) return {"status": "failed", "error": str(db_err)} # 4. Determine final status and prepare response (inside session block to access variables) total_skipped = skipped_invalid + skipped_missing_fk + skipped_missing_invoice # Log summary logger.info(f"Job {job_id} completed. Inserted: {inserted_count}, Skipped: {total_skipped} " f"(invalid: {skipped_invalid}, missing_fk: {skipped_missing_fk}, missing_invoice: {skipped_missing_invoice})") # Prepare response based on results if inserted_count == 0: if total_skipped > 0: logger.warning(f"No valid records to insert for job {job_id}. All {total_skipped} records were rejected.") response = { "status": "warning", "inserted": 0, "skipped_invalid": skipped_invalid, "skipped_missing_invoice": skipped_missing_invoice, "skipped_missing_fk": skipped_missing_fk, "skipped_details": skipped_fk_details, "message": f"No se insertaron registros. {total_skipped} fueron rechazados. Revisa el detalle por línea a continuación.", } else: logger.error(f"No valid records found in CSV for job {job_id}") response = { "status": "failed", "error": "No hay registros válidos en el archivo CSV", "inserted": 0, "skipped_invalid": skipped_invalid, "skipped_missing_invoice": skipped_missing_invoice, "skipped_missing_fk": skipped_missing_fk, "skipped_details": skipped_fk_details } else: # Success case - at least some records were inserted response = { "status": "finished", "inserted": inserted_count, "skipped_invalid": skipped_invalid, "skipped_missing_invoice": skipped_missing_invoice, "skipped_missing_fk": skipped_missing_fk, "skipped_details": skipped_fk_details } except Exception as e: logger.error(f"Task failed: {e}") import traceback logger.error(traceback.format_exc()) return {"status": "failed", "error": str(e)} # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely try: common_storage.cleanup_import_job( effective_job_type, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path, ) except Exception as cleanup_err: logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) # Ensure response is defined (fallback in case of unexpected errors) if response is None: logger.error(f"Unexpected error: response not set for job {job_id}") response = { "status": "failed", "error": "Error inesperado durante el procesamiento", "inserted": 0, "skipped_invalid": skipped_invalid, "skipped_missing_invoice": skipped_missing_invoice, "skipped_missing_fk": skipped_missing_fk, "skipped_details": skipped_fk_details } return response