diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index 14d11aa4..fd489b6e 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -1,2019 +1,2537 @@ -import os -from datetime import datetime -from decimal import Decimal -import csv -import json -import logging -import re -import unicodedata -from typing import Dict, Any, Optional, List, Set, Tuple - -from core.celery_app import celery_app -from core.database import CoreSessionLocal -from core.paths import layout_path - -from ..common import storage as common_storage -from ..common import meta as common_meta -from ..common import responses as common_responses -from .template_config import row_from_template -# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process - -logger = logging.getLogger(__name__) - -# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py) -JOB_TYPE = "" - -# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="") -IMPORT_FILE_KEY_PREFIX = "import_file:" -IMPORT_META_KEY_PREFIX = "import_meta:" -IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" -IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL - - -def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: - """Usa common storage con job_type vacío (prefijo import_).""" - return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import") - - -def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: - return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import") - - -def _delete_import_from_redis(job_id: str) -> None: - common_storage.delete_import_from_redis(JOB_TYPE, job_id) - -class ForeignKeyValidator: - def __init__(self, session, tenant_id, company_id): - self.session = session - self.tenant_id = tenant_id - self.company_id = company_id - self.cache = {} # {(model_name, value): bool} - - def check_exists(self, model, value, field_name="id", is_public=False): - if value is None: - return True # Assume optional if None, or let DB handle not-null - - key = (model.__name__, value) - if key in self.cache: - return self.cache[key] - - query = self.session.query(getattr(model, field_name)).filter(getattr(model, field_name) == value) - if not is_public: - query = query.filter(model.tenant_id == self.tenant_id, model.company_id == self.company_id) - - exists = query.first() is not None - self.cache[key] = exists - return exists - - -TRANSPORT_TYPE_VALUES = { - "none", - "transport", - "box", - "licence plates", - "truck", - "vessel", - "rail_barge", - "container", - "airplane", - "gondola", - "flatbed", -} - - -def normalize_public_code(value: Optional[str]) -> Optional[str]: - if value is None: - return None - text = str(value).strip().upper() - return text or None - - -def validate_public_code( - validator: ForeignKeyValidator, - model, - value: Optional[str], - line_num: int, - col_name: str, - field_name: str = "code", - required: bool = False, -) -> Optional[Dict[str, Any]]: - code = normalize_public_code(value) - if not code: - if required: - return {"line": line_num, "col": col_name, "msg": "Requerido"} - return None - if not validator.check_exists(model, code, field_name=field_name, is_public=True): - return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} - return None - - -def validate_tenant_fk_id( - validator: ForeignKeyValidator, - model, - value: Optional[int], - line_num: int, - col_name: str, - required: bool = False, -) -> Optional[Dict[str, Any]]: - if value is None: - if required: - return {"line": line_num, "col": col_name, "msg": "Requerido"} - return None - if not validator.check_exists(model, value): - return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} - return None - -@celery_app.task(bind=True) -def scan_file(self, job_id: str, model_target: str, config: str = None): - """ - Pass 1: Read CSV, Validate types, Write Errors to JSONL. - File content is loaded from Redis (written by API on upload) so worker does not need shared filesystem. - """ - logger.info(f"Starting scan for job {job_id} target {model_target}") - - # 1. Get file from Redis and write to worker local disk - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} - _ensure_worker_has_meta_from_redis(job_id, file_path) - - error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - - total_rows = 0 - error_count = 0 - processed_rows = 0 - - # 3. Count Total (Quick Pass) or just estimate - try: - with open(file_path, 'r', encoding='utf-8-sig') as f: - total_rows = sum(1 for _ in f) - 1 # Minus header - except Exception as e: - return {"status": "failed", "error": f"Cannot read file: {e}"} - - footer_config = parse_footer_config(config) - date_format = footer_config.get("dateFormat") - - # Validate and set default date_format if not provided - if not date_format: - date_format = "yyyy-mm-dd" # Default to ISO format - logger.info(f"No date_format specified in config, using default: {date_format}") - - try: - tenant_id, company_id = common_meta.require_tenant_context(file_path) - except ValueError as e: - return {"status": "failed", "error": str(e)} - - meta = common_meta.load_meta(file_path) - template_id = meta.get("template_id") or ( - "imp_temp_header" if model_target == "invoice_header" else - "imp_temp_details" if model_target == "invoice_details" else "imp_temp_series" - ) - - inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM") - if not inv_type_value: - inv_type_value = "TEM" - - # --- Series de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- - if model_target == "invoice_series": - try: - from api.v1.modules.a76.invoices.models import InvoiceHeader - from api.v1.modules.a76.items.models import LineItem - from api.v1.modules.a76.items.series.models import Serie - from .validators.series_impo_temp import ( - validate_row_series_impo_temp, - ) - - actualizar = meta.get("actualizar", False) - autonumerar = meta.get("autonumerar", True) - validar_series_exception = meta.get("validar_series", False) - _fc = parse_footer_config(meta.get("footer_config")) - if _fc: - if "actualizar" in _fc: - actualizar = bool(_fc["actualizar"]) - elif _fc.get("mode") == "update": - actualizar = True - elif _fc.get("mode") == "replace": - actualizar = False - if "autonumerar" in _fc: - autonumerar = bool(_fc["autonumerar"]) - else: - as_val = _fc.get("autonumber_series", "true") - autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") - if "validar_series" in _fc: - validar_series_exception = bool(_fc["validar_series"]) - - with CoreSessionLocal() as session: - # Invoice lookup: imp + TEM - q = ( - session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.operation_type == "imp", - InvoiceHeader.invoice_type == "TEM", - ) - ) - rows_inv = q.all() - invoice_id_by_number: Dict[str, int] = {} - invoice_updated_by_number: Dict[str, bool] = {} - for num, iid, is_upd in rows_inv: - if num: - invoice_id_by_number[str(num).strip()] = iid - invoice_updated_by_number[str(num).strip()] = bool(is_upd) - - # Existing series keys: (invoice_number, linea_factura, linea_serie) - existing_series_keys: Set[Tuple[str, str, str]] = set() - existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} - if actualizar and not autonumerar: - q_ser = ( - session.query( - InvoiceHeader.invoice_number, - LineItem.line_number, - Serie.row, - Serie.serial_numbers, - Serie.model, - Serie.sub_model, - Serie.number_id, - ) - .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) - .join(Serie, Serie.line_item_id == LineItem.id) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.operation_type == "imp", - InvoiceHeader.invoice_type == "TEM", - ) - ) - for num, ln, rw, sn, md, sm, nid in q_ser.all(): - if num is not None: - k = (str(num).strip(), str(ln).strip(), str(rw).strip()) - existing_series_keys.add(k) - existing_series_data.setdefault(k, { - "serial_numbers": sn or "", - "model": md or "", - "sub_model": sm or "", - "number_id": nid or "", - }) - - with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - errors_detail: List[Dict[str, Any]] = [] - error_lines_list: List[int] = [] - for i, row in enumerate(reader, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) - row_norm = row_from_template(row, "imp_temp_series", normalize_header) - warnings_list: List[Dict[str, Any]] = [] - err = validate_row_series_impo_temp( - row_norm, - i, - actualizar=actualizar, - autonumerar=autonumerar, - validar_series_exception=validar_series_exception, - invoice_id_by_number=invoice_id_by_number, - invoice_updated_by_number=invoice_updated_by_number, - existing_series_keys=existing_series_keys, - existing_series_data=existing_series_data, - warnings=warnings_list, - ) - if err and not err.get("warning"): - error_count += 1 - error_lines_list.append(err["line"]) - f_err.write(json.dumps(err) + "\n") - if len(errors_detail) < 500: - errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) - for w in warnings_list: - if len(errors_detail) < 500: - errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) - processed_rows += 1 - - if error_lines_list: - common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) - return common_responses.scan_result( - job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows - ) - except Exception as e: - logger.exception("Series import scan failed: %s", e) - return {"status": "failed", "error": str(e)} - - # --- Partidas de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- - if model_target == "invoice_details" and template_id == "imp_temp_details": - try: - from api.v1.modules.a76.invoices.models import InvoiceHeader - from api.v1.modules.a76.items.models import LineItem - from api.v1.modules.a76.classes.models import Class - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - from api.v1.modules.a76.general_catalogs.packages.models import Package - from api.v1.modules.public.reference_data.countries.models import Country - from api.v1.modules.public.reference_data.sectors.models import Sector - from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod - from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod - from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction - from api.v1.modules.a76.general_catalogs.company.models import Company - from api.v1.modules.a76.parts.models import Part - from .validators.partidas_impo_temp import validate_row_partidas_impo_temp - - _fc = parse_footer_config(meta.get("footer_config")) - autonumerar = meta.get("autonumerar", True) - actualizar = meta.get("actualizar", False) - levantar_subpartidas = meta.get("levantar_subpartidas", False) - calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False) - validar_decimales_pza = meta.get("validar_decimales_pza", False) - if _fc: - if "autonumerar" in _fc: - autonumerar = bool(_fc["autonumerar"]) - elif _fc.get("autonumber_partidas", "true") is not None: - autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes") - if "actualizar" in _fc: - actualizar = bool(_fc["actualizar"]) - if "levantar_subpartidas" in _fc: - levantar_subpartidas = bool(_fc["levantar_subpartidas"]) - if "calcular_costo_unitario_en_base_a_valor_total" in _fc: - calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"]) - if "validar_decimales_pza" in _fc: - validar_decimales_pza = bool(_fc["validar_decimales_pza"]) - - RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"} - RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"} - - with CoreSessionLocal() as session: - q_inv = ( - session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.operation_type == "imp", - InvoiceHeader.invoice_type == "TEM", - ) - ) - invoice_id_by_number: Dict[str, int] = {} - invoice_updated_by_number: Dict[str, bool] = {} - for num, iid, is_upd in q_inv.all(): - if num: - invoice_id_by_number[str(num).strip()] = iid - invoice_updated_by_number[str(num).strip()] = bool(is_upd) - - existing_line_keys_by_invoice: Dict[str, Set[str]] = {} - q_li = ( - session.query(InvoiceHeader.invoice_number, LineItem.line_number) - .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.operation_type == "imp", - InvoiceHeader.invoice_type == "TEM", - ) - ) - for num, ln in q_li.all(): - if num is not None: - key = str(num).strip() - if key not in existing_line_keys_by_invoice: - existing_line_keys_by_invoice[key] = set() - existing_line_keys_by_invoice[key].add(str(ln).strip()) - - partidas_principales_bd: Set[Tuple[str, str]] = set() - try: - from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem - q_pp = ( - session.query(InvoiceHeader.invoice_number, LineItem.line_number) - .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) - .join(FaLineItem, FaLineItem.id == LineItem.id) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - FaLineItem.is_subitem == False, - FaLineItem.contains_subitems == True, - ) - ) - for num, ln in q_pp.all(): - if num is not None: - partidas_principales_bd.add((str(num).strip(), str(ln).strip())) - except Exception: - pass - - valid_class_codes: Set[str] = set() - class_um_by_code: Dict[str, str] = {} - class_fraction_by_code: Dict[str, str] = {} - class_desc_es_by_code: Dict[str, str] = {} - class_desc_en_by_code: Dict[str, str] = {} - for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): - code = (c.class_code or "").strip().upper() - if code: - valid_class_codes.add(code) - class_um_by_code[code] = (c.unit_of_measure or "").strip().upper() - class_fraction_by_code[code] = (c.fraction or "").strip() - class_desc_es_by_code[code] = (c.description_es or "").strip() - class_desc_en_by_code[code] = (c.description_en or "").strip() - - valid_uom_codes: Set[str] = set() - for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): - if u[0]: - valid_uom_codes.add((u[0] or "").strip().upper()) - - valid_bulks_codes: Set[str] = set() - for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): - if p[0]: - valid_bulks_codes.add((p[0] or "").strip()) - - valid_country_keys: Set[str] = set() - for row in session.query(Country.m3_key, Country.ame_key).all(): - if row[0]: - valid_country_keys.add((row[0] or "").strip().upper()) - if row[1]: - valid_country_keys.add((row[1] or "").strip().upper()) - - valid_fraction_ame: Set[str] = set() - for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all(): - if row[0]: - valid_fraction_ame.add((row[0] or "").strip()) - - authorized_sectors: Set[str] = set() - for row in session.query(Sector.key).filter(Sector.authorized == True).all(): - if row[0]: - authorized_sectors.add((row[0] or "").strip().upper()) - - valid_payment_methods: Set[str] = set() - for row in session.query(PaymentMethod.key).all(): - if row[0] is not None: - valid_payment_methods.add(str(row[0]).strip()) - - valid_valuation_methods: Set[str] = set() - for row in session.query(ValuationMethod.key).all(): - if row[0]: - valid_valuation_methods.add((row[0] or "").strip()) - - company = session.query(Company).filter(Company.id == company_id).first() - company_has_prosec = bool(company.prosec) if company else False - company_rfc = (company.rfc or "").strip().upper() if company else "" - - valid_part_numbers: Set[str] = set() - for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all(): - if row[0]: - valid_part_numbers.add((row[0] or "").strip().upper()) - - rfc_exception_updated: Set[str] = set() - rfc_exception_num_parte: Set[str] = set() - - with open(file_path, "r", encoding="utf-8-sig") as f_in: - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f_in, dialect=dialect) - rows_list = list(reader) - - invoice_numbers_from_csv = set() - for row in rows_list: - inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() - if inv: - invoice_numbers_from_csv.add(inv) - if company_rfc in RFC_EXCEPTION_UPDATED: - rfc_exception_updated = invoice_numbers_from_csv - if company_rfc in RFC_EXCEPTION_NUM_PARTE: - rfc_exception_num_parte = invoice_numbers_from_csv - - line_counts_csv: Dict[Tuple[str, str], int] = {} - partidas_principales_csv: Set[Tuple[str, str]] = set() - - def _get_row(row_norm: Dict[str, Any], *keys: str) -> str: - for k in keys: - v = row_norm.get(k) - if v is not None and str(v).strip(): - return str(v).strip() - return "" - - for row in rows_list: - row_norm = row_from_template(row, "imp_temp_details", normalize_header) - inv = _get_row(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") - linea = _get_row(row_norm, "LINEA", "RENGLON", "PARTIDA") - if inv and linea: - key = (inv, linea) - line_counts_csv[key] = line_counts_csv.get(key, 0) + 1 - u = _get_row(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper() - if u == "P" and inv and linea: - partidas_principales_csv.add((inv, linea)) - - error_count = 0 - processed_rows = 0 - error_lines_list = [] - errors_detail = [] - - with open(error_path, "w", encoding="utf-8") as f_err: - for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) - row_norm = row_from_template(row, "imp_temp_details", normalize_header) - err = validate_row_partidas_impo_temp( - row_norm, - i, - autonumerar=autonumerar, - actualizar=actualizar, - levantar_subpartidas=levantar_subpartidas, - calcular_costo_en_base_a_total=calcular_costo_en_base_a_total, - validar_decimales_pza=validar_decimales_pza, - invoice_id_by_number=invoice_id_by_number, - invoice_updated_by_number=invoice_updated_by_number, - rfc_exception_updated=rfc_exception_updated, - existing_line_keys_by_invoice=existing_line_keys_by_invoice, - line_counts_csv=line_counts_csv, - partidas_principales_csv=partidas_principales_csv, - partidas_principales_bd=partidas_principales_bd, - valid_class_codes=valid_class_codes, - class_um_by_code=class_um_by_code, - class_fraction_by_code=class_fraction_by_code, - class_desc_es_by_code=class_desc_es_by_code, - class_desc_en_by_code=class_desc_en_by_code, - valid_uom_codes=valid_uom_codes, - valid_bulks_codes=valid_bulks_codes, - valid_country_keys=valid_country_keys, - valid_fraction_ame=valid_fraction_ame, - valid_payment_methods=valid_payment_methods, - valid_valuation_methods=valid_valuation_methods, - authorized_sectors=authorized_sectors, - company_has_prosec=company_has_prosec, - rfc_exception_num_parte=rfc_exception_num_parte or None, - valid_part_numbers=valid_part_numbers, - warnings=None, - ) - if err: - error_count += 1 - error_lines_list.append(err["line"]) - f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") - if len(errors_detail) < 500: - errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) - processed_rows += 1 - - if error_lines_list: - common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) - return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) - except Exception as e: - logger.exception("Partidas import scan failed: %s", e) - return {"status": "failed", "error": str(e)} - - try: - from api.v1.modules.a76.invoices.models import InvoiceHeader - from api.v1.modules.a76.clients_and_providers.models import ClientProvider - from api.v1.modules.a76.customs_brokers.models import CustomsBroker - from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode - from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( - CodePedimentoRegimen, - ) - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType - from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection - from api.v1.modules.public.reference_data.incoterms.models import Incoterm - from api.v1.modules.a76.parts.models import Part - - models = { - "InvoiceHeader": InvoiceHeader, - "InvoiceType": InvoiceType, - "ClientProvider": ClientProvider, - "CustomsBroker": CustomsBroker, - "RegimenPedimento": RegimenPedimento, - "CodePedimentoRegimen": CodePedimentoRegimen, - "PedimentoCode": PedimentoCode, - "CurrencyType": CurrencyType, - "CustomsSection": CustomsSection, - "Incoterm": Incoterm, - "Part": Part, - } - - with CoreSessionLocal() as session, \ - open(file_path, 'r', encoding='utf-8-sig') as f_in, \ - open(error_path, 'w', encoding='utf-8') as f_err: - validator = ForeignKeyValidator(session, tenant_id, company_id) - invoice_id_cache: Dict[str, Optional[int]] = {} - - # Detect Delimiter - sample = f_in.read(2048) - f_in.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except: - dialect = 'excel' - - reader = csv.DictReader(f_in, dialect=dialect) - - for i, row in enumerate(reader, start=1): - # Check for Progress Update - if i % 1000 == 0: - self.update_state(state='PROGRESS', meta={ - 'current': i, - 'total': total_rows, - 'errors': error_count - }) - - # Solo columnas de la plantilla (respetar plantilla tal cual) - row_norm = row_from_template(row, template_id, normalize_header) - errors = validate_row_strict( - row_norm, - model_target, - i, - date_format, - validator, - inv_type_value, - invoice_id_cache, - models, - ) - - if errors: - error_count += 1 - # Write simple JSON error - f_err.write(json.dumps(errors) + "\n") - - processed_rows += 1 - - except Exception as e: - logger.error(f"Scan failed: {e}") - return {"status": "failed", "error": str(e)} - - # 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker) - error_lines_list = [] - errors_detail: List[Dict[str, Any]] = [] - try: - if os.path.exists(error_path): - with open(error_path, "r", encoding="utf-8") as f: - for line in f: - try: - err = json.loads(line) - if "line" in err: - error_lines_list.append(err["line"]) - if len(errors_detail) < 500: - errors_detail.append( - { - "line": err["line"], - "col": err.get("col", ""), - "msg": err.get("msg", ""), - } - ) - except Exception: - pass - if error_lines_list: - common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) - except Exception as e: - logger.warning(f"Failed to store error lines in Redis: {e}") - - return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) - -def validate_row_phase_1( - row: Dict[str, Any], - target: str, - line_num: int, - date_format: Optional[str], -) -> Optional[Dict[str, Any]]: - """ - Validation: Unique IDs, Dates, and Numeric constraint checks. - Target: 'invoice_header' or 'invoice_details' - """ - def check_decimal(col_name): - val = row.get(col_name) - if val and str(val).strip(): - if parse_decimal(val) is None: - return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"} - return None - - def check_int(col_name): - val = row.get(col_name) - if val and str(val).strip(): - if parse_int(val) is None: - return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"} - return None - - def check_date(col_name): - date_str = row.get(col_name) - if date_str and str(date_str).strip(): - if not is_valid_date(date_str, date_format): - expected = display_date_format(date_format) - return { - "line": line_num, - "col": col_name, - "msg": f"Formato de fecha inválido ({expected})", - } - return None - - def check_weight(col_name): - val = row.get(col_name) - if val and str(val).strip(): - if parse_weight_unit(val) is None: - return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"} - return None - - def check_currency(col_name): - val = row.get(col_name) - if val and str(val).strip(): - parsed_currency = parse_currency(val, None) - val_norm = normalize_header(val) - # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid - if parsed_currency.value == "manual" and "MANUAL" not in val_norm: - return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"} - return None - - def check_transport_type(col_name): - val = row.get(col_name) - if val and str(val).strip(): - if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES: - return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"} - return None - - # A. Invoice Header - if target == 'invoice_header': - # 1. Unique ID - if not row.get('NUMERO FACTURA') and not row.get('NUM FACTURA') and not row.get('ID'): - return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} - - # 2. Date Format - date_str = row.get('FECHA FACTURA') or row.get('FECHA') - if not date_str or not str(date_str).strip(): - return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"} - - err = check_date('FECHA FACTURA') or check_date('FECHA') - if err: return err - - err = check_date('FECHA EMISION') - if err: return err - - # 3. Numeric Fields - for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']: - err = check_decimal(col) - if err: return err - - # 4. Integer FKs - for col in ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA']: - err = check_int(col) - if err: return err - - # 5. Enums - for col in ['TIPO PESO']: - err = check_weight(col) - if err: return err - - for col in ['TIPO MONEDA']: - err = check_currency(col) - if err: return err - - for col in ['TIPO TRANSPORTE']: - err = check_transport_type(col) - if err: return err - - # B. Invoice Details (Parts) - elif target == 'invoice_details': - # 1. Line Number - if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'): - return {"line": line_num, "col": "LINEA", "msg": "Requerido"} - - # 2. Parent Link (Invoice Number) - if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')): - return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} - - # 3. Numeric Fields - for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']: - err = check_decimal(col) - if err: return err - - for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']: - err = check_int(col) - if err: return err - - return None - - -def validate_row_strict( - row: Dict[str, Any], - target: str, - line_num: int, - date_format: Optional[str], - validator: ForeignKeyValidator, - inv_type_value: str, - invoice_id_cache: Dict[str, Optional[int]], - models: Dict[str, Any], -) -> Optional[Dict[str, Any]]: - err = validate_row_phase_1(row, target, line_num, date_format) - if err: - return err - - InvoiceHeader = models["InvoiceHeader"] - InvoiceType = models["InvoiceType"] - ClientProvider = models["ClientProvider"] - CustomsBroker = models["CustomsBroker"] - RegimenPedimento = models["RegimenPedimento"] - CurrencyType = models["CurrencyType"] - CustomsSection = models["CustomsSection"] - Incoterm = models["Incoterm"] - Part = models["Part"] - - if target == "invoice_header": - if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): - return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"} - - provider_id = parse_int(row.get("CLAVE PROVEEDOR")) - err = validate_tenant_fk_id(validator, ClientProvider, provider_id, line_num, "CLAVE PROVEEDOR", required=True) - if err: - return err - - sold_to_id = parse_int(row.get("CLAVE VENDIDO A")) - err = validate_tenant_fk_id(validator, ClientProvider, sold_to_id, line_num, "CLAVE VENDIDO A", required=True) - if err: - return err - - shipped_to_id = parse_int(row.get("CLAVE ENVIADO A")) - err = validate_tenant_fk_id(validator, ClientProvider, shipped_to_id, line_num, "CLAVE ENVIADO A", required=True) - if err: - return err - - broker_id = parse_int(row.get("AGENTE ADUANAL")) - err = validate_tenant_fk_id(validator, CustomsBroker, broker_id, line_num, "AGENTE ADUANAL") - if err: - return err - - err = validate_public_code( - validator, - RegimenPedimento, - row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"), - line_num, - "CLAVEDOCUMENTO", - ) - if err: - return err - - err = validate_public_code( - validator, - CustomsSection, - row.get("ADUANA DE CRUCE"), - line_num, - "ADUANA DE CRUCE", - field_name="customs_code", - ) - if err: - return err - - err = validate_public_code( - validator, - CurrencyType, - row.get("CLAVE MONEDA"), - line_num, - "CLAVE MONEDA", - ) - if err: - return err - - err = validate_public_code( - validator, - Incoterm, - row.get("CLAVE INCOTERM"), - line_num, - "CLAVE INCOTERM", - ) - if err: - return err - - elif target == "invoice_details": - invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() - if not invoice_number: - return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} - - cache_key = f"{invoice_number}|{inv_type_value}" - if cache_key in invoice_id_cache: - invoice_id = invoice_id_cache[cache_key] - else: - invoice_id = ( - validator.session.query(InvoiceHeader.id) - .filter( - InvoiceHeader.tenant_id == validator.tenant_id, - InvoiceHeader.company_id == validator.company_id, - InvoiceHeader.invoice_number == invoice_number, - InvoiceHeader.invoice_type == inv_type_value, - ) - .scalar() - ) - invoice_id_cache[cache_key] = invoice_id - if not invoice_id: - return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"} - - part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip() - if not part_num: - return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} - if not validator.check_exists(Part, part_num, field_name="part_number"): - return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"} - - return None - -def parse_footer_config(config: Optional[str]) -> Dict[str, Any]: - if not config: - return {} - try: - if isinstance(config, str): - return json.loads(config) - if isinstance(config, dict): - return config - except Exception: - return {} - return {} - - -def display_date_format(date_format: Optional[str]) -> str: - if not date_format: - return "YYYY-MM-DD" - return date_format.upper() - - -def parse_date(date_text: Optional[str], date_format: Optional[str]) -> Optional[datetime.date]: - if not date_text: - return None - candidates = [] - fmt_map = { - "dd/mm/yyyy": "%d/%m/%Y", - "mm/dd/yyyy": "%m/%d/%Y", - "yyyy-mm-dd": "%Y-%m-%d", - } - if date_format and date_format in fmt_map: - candidates.append(fmt_map[date_format]) - candidates.extend(["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y"]) - for fmt in candidates: - try: - return datetime.strptime(str(date_text).strip(), fmt).date() - except ValueError: - continue - return None - - -def is_valid_date(date_text: Optional[str], date_format: Optional[str]) -> bool: - return parse_date(date_text, date_format) is not None - - -def normalize_header(name: Optional[str]) -> str: - if not name: - return "" - name = unicodedata.normalize("NFKD", str(name)).upper() - name = "".join(ch for ch in name if not unicodedata.combining(ch)) - name = re.sub(r"[^A-Z0-9]+", " ", name) - return re.sub(r"\s+", " ", name).strip() - - -def normalize_row(row: Dict[str, Any]) -> Dict[str, Any]: - return {normalize_header(k): v for k, v in row.items()} - - -def parse_int(value: Any) -> Optional[int]: - if value is None: - return None - text = str(value).strip() - if not text: - return None - try: - return int(text) - except ValueError: - return None - - -def parse_decimal(value: Any) -> Optional[Decimal]: - if value is None: - return None - text = str(value).strip() - if not text: - return None - text = text.replace(",", "") - try: - return Decimal(text) - except Exception: - return None - - -def decimal_or_zero(value: Any) -> Decimal: - """Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default).""" - return parse_decimal(value) or Decimal("0") - - -def int_or_zero(value: Any) -> int: - """Return parsed int or 0 for CSV nulls/empty (vanilla default).""" - return parse_int(value) if parse_int(value) is not None else 0 - - -def parse_currency(value: Optional[str], currency_type: Optional[str]): - from api.v1.modules.a76.invoices.models import Currency - if value: - normalized = normalize_header(value) - if normalized in {"MN", "M N", "NACIONAL", "LOCAL", "PESOS", "PESO"}: - return Currency.LOCAL - if normalized in {"ME", "M E", "EXTRANJERA", "EXTRANJERO", "FOREIGN", "USD", "DOLAR", "DOLARES"}: - return Currency.FOREIGN - if "MANUAL" in normalized: - return Currency.MANUAL - if currency_type and str(currency_type).strip().upper() == "MXN": - return Currency.LOCAL - if currency_type: - return Currency.FOREIGN - return Currency.MANUAL - - -def parse_weight_unit(value: Optional[str]): - from api.v1.modules.a76.invoices.models import WeightUnit - if not value: - return None - normalized = normalize_header(value) - if normalized in {"KG", "KGS", "KILOS", "KILOGRAMOS"}: - return WeightUnit.KGS - if normalized in {"LB", "LBS", "LIBRAS"}: - return WeightUnit.LBS - return None - - -def resolve_tenant_fk_id( - session: CoreSessionLocal, - model, - value: Optional[int], - tenant_id: int, - company_id: int, - cache: Dict[int, Optional[int]], -) -> Optional[int]: - if value is None: - return None - if value in cache: - return cache[value] - exists = ( - session.query(model.id) - .filter( - model.id == value, - model.tenant_id == tenant_id, - model.company_id == company_id, - ) - .scalar() - ) - cache[value] = value if exists is not None else None - return cache[value] - - -def resolve_public_code( - session: CoreSessionLocal, - model, - column, - value: Optional[str], - cache: Dict[str, Optional[str]], -) -> Optional[str]: - if not value: - return None - normalized = str(value).strip().upper() - if not normalized: - return None - if normalized in cache: - return cache[normalized] - exists = session.query(column).filter(column == normalized).scalar() - cache[normalized] = normalized if exists is not None else None - return cache[normalized] - -@celery_app.task(bind=True) -def insert_valid_rows(self, job_id: str, model_target: str): - """ - Pass 2: Re-read CSV, Skip Errors, Bulk Insert. - File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem. - """ - logger.info(f"Starting Commit for {job_id} target {model_target}") - - # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) - file_path = _ensure_worker_has_file_from_redis(job_id) - if not file_path: - alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) - if not os.path.exists(alt_path): - return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} - file_path = alt_path - else: - _ensure_worker_has_meta_from_redis(job_id, file_path) - - try: - tenant_id, company_id = common_meta.require_tenant_context(file_path) - except ValueError as e: - return {"status": "failed", "error": str(e)} - - meta = common_meta.load_meta(file_path) - meta_path = common_meta.get_meta_path(file_path) - error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) - error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) - - # Si el upload fue de series (template_id imp_temp_series), usar flujo series aunque model_target venga mal - use_series_flow = ( - model_target == "invoice_series" - or meta.get("template_id") == "imp_temp_series" - ) - - # --- Series de Importación Temporal: commit (INSERT/UPDATE item_line_series) --- - if use_series_flow: - try: - from api.v1.modules.a76.invoices.models import InvoiceHeader - from api.v1.modules.a76.items.models import LineItem - from api.v1.modules.a76.items.series.models import Serie - from .validators.series_impo_temp import ( - validate_row_series_impo_temp, - row_to_series_normalized, - ) - - actualizar = meta.get("actualizar", False) - autonumerar = meta.get("autonumerar", True) - validar_series_exception = meta.get("validar_series", False) - _fc = parse_footer_config(meta.get("footer_config")) - if _fc: - if "actualizar" in _fc: - actualizar = bool(_fc["actualizar"]) - elif _fc.get("mode") == "update": - actualizar = True - elif _fc.get("mode") == "replace": - actualizar = False - if "autonumerar" in _fc: - autonumerar = bool(_fc["autonumerar"]) - else: - as_val = _fc.get("autonumber_series", "true") - autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") - if "validar_series" in _fc: - validar_series_exception = bool(_fc["validar_series"]) - - with CoreSessionLocal() as session: - q = ( - session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.operation_type == "imp", - InvoiceHeader.invoice_type == "TEM", - ) - ) - rows_inv = q.all() - invoice_id_by_number: Dict[str, int] = {} - invoice_updated_by_number: Dict[str, bool] = {} - for num, iid, is_upd in rows_inv: - if num: - invoice_id_by_number[str(num).strip()] = iid - invoice_updated_by_number[str(num).strip()] = bool(is_upd) - - existing_series_keys: Set[Tuple[str, str, str]] = set() - existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} - if actualizar and not autonumerar: - q_ser = ( - session.query( - InvoiceHeader.invoice_number, - LineItem.line_number, - Serie.row, - Serie.serial_numbers, - Serie.model, - Serie.sub_model, - Serie.number_id, - ) - .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) - .join(Serie, Serie.line_item_id == LineItem.id) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.operation_type == "imp", - InvoiceHeader.invoice_type == "TEM", - ) - ) - for num, ln, rw, sn, md, sm, nid in q_ser.all(): - if num is not None: - k = (str(num).strip(), str(ln).strip(), str(rw).strip()) - existing_series_keys.add(k) - existing_series_data.setdefault(k, { - "serial_numbers": sn or "", - "model": md or "", - "sub_model": sm or "", - "number_id": nid or "", - }) - - inserted_count = 0 - updated_count = 0 - skipped_invalid = 0 - skipped_details: List[Dict[str, Any]] = [] - - with open(file_path, "r", encoding="utf-8-sig") as f: - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except Exception: - dialect = "excel" - reader = csv.DictReader(f, dialect=dialect) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - row_norm = row_from_template(row, "imp_temp_series", normalize_header) - err = validate_row_series_impo_temp( - row_norm, - i, - actualizar=actualizar, - autonumerar=autonumerar, - validar_series_exception=validar_series_exception, - invoice_id_by_number=invoice_id_by_number, - invoice_updated_by_number=invoice_updated_by_number, - existing_series_keys=existing_series_keys, - existing_series_data=existing_series_data, - warnings=None, - ) - if err: - skipped_invalid += 1 - skipped_details.append({ - "line": i, - "invoice": (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip(), - "reason": err.get("msg", ""), - }) - continue - - data = row_to_series_normalized(row_norm) - invoice_number = data["NUMERO FACTURA"] - linea_factura = data["LINEA FACTURA"] - linea_serie = data["LINEA SERIE"] - - if not invoice_number or invoice_number not in invoice_id_by_number: - skipped_invalid += 1 - continue - invoice_id = invoice_id_by_number[invoice_number] - line_number_val = parse_int(linea_factura) - if line_number_val is None: - skipped_invalid += 1 - skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."}) - continue - line_item = ( - session.query(LineItem) - .filter( - LineItem.invoice_id == invoice_id, - LineItem.line_number == line_number_val, - LineItem.tenant_id == tenant_id, - LineItem.company_id == company_id, - ) - .first() - ) - if not line_item: - skipped_invalid += 1 - skipped_details.append({ - "line": i, - "invoice": invoice_number, - "reason": f"Partida línea {linea_factura} no existe en la factura.", - }) - continue - - if autonumerar: - max_row = ( - session.query(Serie.row) - .filter(Serie.line_item_id == line_item.id) - .order_by(Serie.row.desc()) - .limit(1) - .scalar() - ) - row_num = (max_row or 0) + 1 - else: - row_num = parse_int(linea_serie) - if row_num is None: - skipped_invalid += 1 - skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."}) - continue - - existing_serie = ( - session.query(Serie) - .filter( - Serie.line_item_id == line_item.id, - Serie.row == row_num, - ) - .first() - ) - - if existing_serie: - if actualizar: - existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers - existing_serie.model = data["MODELO"] or existing_serie.model - existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model - existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id - session.add(existing_serie) - updated_count += 1 - else: - skipped_invalid += 1 - skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."}) - else: - new_serie = Serie( - tenant_id=tenant_id, - company_id=company_id, - line_item_id=line_item.id, - row=row_num, - serial_numbers=data["SERIE"] or None, - model=data["MODELO"] or None, - sub_model=data["SUB MODELO"] or None, - number_id=data["NUMERO ID"] or None, - ) - session.add(new_serie) - inserted_count += 1 - - session.commit() - - common_storage.cleanup_import_job(JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path) - status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed") - out = { - "status": status, - "inserted": inserted_count, - "updated": updated_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_fk": 0, - "skipped_duplicate": 0, - "skipped_details": skipped_details, - } - if status == "failed": - out["error"] = "No hay registros válidos en el archivo CSV." - elif status == "warning" and skipped_invalid: - out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados." - return out - except Exception as e: - logger.exception("Series import commit failed: %s", e) - return {"status": "failed", "error": str(e)} - - try: - from api.v1.modules.a76.invoices.models import ( - InvoiceHeader, - InvoiceComplianceMx, - InvoiceFinancials, - InvoiceLogistics, - InvoiceSalesDetails, - OperationType, - TransportType, - WeightUnit, - ) - from api.v1.modules.a76.clients_and_providers.models import ClientProvider - from api.v1.modules.a76.customs_brokers.models import CustomsBroker - from api.v1.modules.public.reference_data.currency_types.models import CurrencyType - from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode - from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen - from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento - from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType - from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection - from api.v1.modules.public.reference_data.incoterms.models import Incoterm - - from api.v1.modules.a76.items.models import LineItem - from api.v1.modules.a76.items.line_financials.models import LineFinancial - from api.v1.modules.a76.items.line_quantities.models import LineQuantity - from api.v1.modules.a76.items.line_customs.models import LineCustom - from api.v1.modules.a76.items.line_descriptions.models import LineDescription - from api.v1.modules.a76.parts.models import Part - from api.v1.modules.a76.classes.models import Class - from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure - from api.v1.modules.a76.general_catalogs.packages.models import Package - - footer_config = parse_footer_config(meta.get("footer_config")) - - date_format = footer_config.get("dateFormat") - # Validate and set default date_format if not provided - if not date_format: - date_format = "yyyy-mm-dd" # Default to ISO format - logger.info(f"No date_format specified in config, using default: {date_format}") - else: - logger.info(f"Using date_format from config: {date_format}") - - # Default types from config or fallback - op_type_value = OperationType(meta.get('operation_type', 'imp').lower()) - inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM' - - logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") - - headers_to_insert = [] - details_to_insert = [] - skipped_invalid = 0 - skipped_missing_invoice = 0 - skipped_missing_fk = 0 - skipped_fk_details = [] - inserted_count = 0 - response = None - - with CoreSessionLocal() as session: - invoice_id_cache = {} - cleared_invoices = set() # Track invoices where we've already cleared items in this job - provider_cache: Dict[int, Optional[int]] = {} - sold_to_cache: Dict[int, Optional[int]] = {} - shipped_to_cache: Dict[int, Optional[int]] = {} - broker_cache: Dict[int, Optional[int]] = {} - regimen_cache: Dict[str, Optional[str]] = {} - currency_type_cache: Dict[str, Optional[str]] = {} - customs_section_cache: Dict[str, Optional[str]] = {} - part_cache: Dict[str, Optional[int]] = {} - class_id_by_code: Dict[str, int] = {} - uom_id_by_code: Dict[str, int] = {} - package_id_by_key: Dict[str, int] = {} - if model_target == 'invoice_details': - for c in session.query(Class.id, Class.class_code).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): - if c[1]: - class_id_by_code[(c[1] or "").strip().upper()] = c[0] - for u in session.query(UnitOfMeasure.id, UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): - if u[1]: - uom_id_by_code[(u[1] or "").strip().upper()] = u[0] - for p in session.query(Package.id, Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): - if p[1]: - package_id_by_key[(p[1] or "").strip()] = p[0] - - validator = ForeignKeyValidator(session, tenant_id, company_id) - - with open(file_path, 'r', encoding='utf-8-sig') as f: - # Detect Delimiter - sample = f.read(2048) - f.seek(0) - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") - except: - dialect = 'excel' - - reader = csv.DictReader(f, dialect=dialect) - - template_id = meta.get("template_id") or ( - "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" - ) - - for i, row in enumerate(reader, start=1): - if i in error_lines: - continue - - row_norm = row_from_template(row, template_id, normalize_header) - - # Mapping Logic (solo campos que acepta el modelo de facturas) - if model_target == 'invoice_header': - invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() - invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format) - - if not invoice_number or not invoice_date: - skipped_invalid += 1 - logger.debug(f"Row {i}: Skipped - missing invoice_number or invalid invoice_date. " - f"Invoice: {invoice_number}, Date: {row_norm.get('FECHA FACTURA') or row_norm.get('FECHA')}") - continue - - # --- NEW: Foreign Key Validations --- - # 1. Invoice Type (Public) - if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): - skipped_missing_fk += 1 - reason = f"Tipo de factura '{inv_type_value}' no existe" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - provider_id = parse_int(row_norm.get('CLAVE PROVEEDOR')) - err = validate_tenant_fk_id( - validator, - ClientProvider, - provider_id, - i, - "CLAVE PROVEEDOR", - required=True, - ) - if err: - skipped_invalid += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - sold_to_id = parse_int(row_norm.get('CLAVE VENDIDO A')) - err = validate_tenant_fk_id( - validator, - ClientProvider, - sold_to_id, - i, - "CLAVE VENDIDO A", - required=True, - ) - if err: - skipped_invalid += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - shipped_to_id = parse_int(row_norm.get('CLAVE ENVIADO A')) - err = validate_tenant_fk_id( - validator, - ClientProvider, - shipped_to_id, - i, - "CLAVE ENVIADO A", - required=True, - ) - if err: - skipped_invalid += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - broker_id = parse_int(row_norm.get('AGENTE ADUANAL')) - err = validate_tenant_fk_id( - validator, - CustomsBroker, - broker_id, - i, - "AGENTE ADUANAL", - ) - if err: - skipped_missing_fk += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - err = validate_public_code( - validator, - RegimenPedimento, - row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'), - i, - "CLAVEDOCUMENTO", - ) - if err: - skipped_missing_fk += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - err = validate_public_code( - validator, - CustomsSection, - row_norm.get('ADUANA DE CRUCE'), - i, - "ADUANA DE CRUCE", - field_name="customs_code", - ) - if err: - skipped_missing_fk += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - err = validate_public_code( - validator, - CurrencyType, - row_norm.get('CLAVE MONEDA'), - i, - "CLAVE MONEDA", - ) - if err: - skipped_missing_fk += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - err = validate_public_code( - validator, - Incoterm, - row_norm.get('CLAVE INCOTERM'), - i, - "CLAVE INCOTERM", - ) - if err: - skipped_missing_fk += 1 - reason = f"{err['col']}: {err['msg']}" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - transport_type_val = row_norm.get('TIPO TRANSPORTE') - if transport_type_val and str(transport_type_val).strip().lower() not in TRANSPORT_TYPE_VALUES: - skipped_invalid += 1 - reason = "TIPO TRANSPORTE: Tipo de transporte invalido" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - currency_val = row_norm.get('TIPO MONEDA') - if currency_val and str(currency_val).strip(): - parsed_currency = parse_currency(currency_val, None) - val_norm = normalize_header(currency_val) - if parsed_currency.value == "manual" and "MANUAL" not in val_norm: - skipped_invalid += 1 - reason = "TIPO MONEDA: Moneda invalida" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - # 2. Client/Provider and broker checks are handled above - - # --- 4. Check for Existing Invoice (Upsert Logic) --- - existing_header = None - if invoice_number: - existing_header = ( - session.query(InvoiceHeader) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.invoice_number == invoice_number, - InvoiceHeader.invoice_type == inv_type_value - ) - .first() - ) - - if existing_header: - # UPDATE existing header - header = existing_header - header.invoice_date = invoice_date - header.operation_type = op_type_value - header.is_updated = True # Mark as updated - header.updated_date = datetime.utcnow() - header.document_type = resolve_public_code( - session, - RegimenPedimento, - RegimenPedimento.code, - (row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')), - regimen_cache, - ) - header.project_number = (row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None) - header.purchase_order = (row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None) - header.alternate_invoice = (row_norm.get('FACTURA ALTERNA') or None) - header.invoice_ref = (row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None) - header.emission_date = parse_date(row_norm.get('FECHA EMISION'), date_format) - header.observation_es = (row_norm.get('OBSERVACIONES E') or None) - header.observation_en = (row_norm.get('OBSERVACIONES I') or None) - - logger.info(f"Row {i}: Updating existing invoice {invoice_number}") - - # Clean up related data that will be re-inserted/updated - # Note: compliance, financials, logistics are 1-to-1 relationships and will be updated by assignment below - # but we might want to be explicit if ORM doesn't handle replace well. - # SQLAlchemy relationship assignment usually handles 1-to-1 updates correctly. - - else: - # CREATE new header - header = InvoiceHeader( - invoice_number=invoice_number, - invoice_date=invoice_date, - operation_type=op_type_value, - is_updated=False, - system="CSV", - capture_date=datetime.utcnow(), - invoice_type=inv_type_value, - document_type=resolve_public_code( - session, - RegimenPedimento, - RegimenPedimento.code, - (row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')), - regimen_cache, - ), - project_number=(row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None), - purchase_order=(row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None), - alternate_invoice=(row_norm.get('FACTURA ALTERNA') or None), - invoice_ref=(row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None), - emission_date=parse_date(row_norm.get('FECHA EMISION'), date_format), - observation_es=(row_norm.get('OBSERVACIONES E') or None), - observation_en=(row_norm.get('OBSERVACIONES I') or None), - tenant_id=tenant_id, - company_id=company_id, - ) - - compliance = InvoiceComplianceMx( - remesa=parse_int(row_norm.get('REMESA')), - aduana=resolve_public_code( - session, - CustomsSection, - CustomsSection.customs_code, - row_norm.get('ADUANA DE CRUCE'), - customs_section_cache, - ), - provider_id=resolve_tenant_fk_id( - session, - ClientProvider, - parse_int(row_norm.get('CLAVE PROVEEDOR')), - tenant_id, - company_id, - provider_cache, - ), - sold_to_id=resolve_tenant_fk_id( - session, - ClientProvider, - parse_int(row_norm.get('CLAVE VENDIDO A')), - tenant_id, - company_id, - sold_to_cache, - ), - shipped_to_id=resolve_tenant_fk_id( - session, - ClientProvider, - parse_int(row_norm.get('CLAVE ENVIADO A')), - tenant_id, - company_id, - shipped_to_cache, - ), - customs_broker_id=resolve_tenant_fk_id( - session, - CustomsBroker, - parse_int(row_norm.get('AGENTE ADUANAL')), - tenant_id, - company_id, - broker_cache, - ), - edocument=(row_norm.get('E DOCUMENT') or None), - vucem_operation_num=(row_norm.get('NUM OPERACION') or None), - tenant_id=tenant_id, - company_id=company_id, - ) - - financials_currency_type = resolve_public_code( - session, - CurrencyType, - CurrencyType.code, - row_norm.get('CLAVE MONEDA'), - currency_type_cache, - ) - financials = InvoiceFinancials( - currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type), - currency_type=financials_currency_type, - exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')), - freight=decimal_or_zero(row_norm.get('FLETES')), - insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')), - insurance=decimal_or_zero(row_norm.get('SEGUROS')), - packaging=decimal_or_zero(row_norm.get('EMBALAJES')), - other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')), - tenant_id=tenant_id, - company_id=company_id, - ) - - weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) - logistics = None - if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): - raw_transport = (row_norm.get('TIPO TRANSPORTE') or "none") - transport_str = str(raw_transport).strip().lower() or "none" - try: - transport_type = TransportType(transport_str) - except ValueError: - transport_type = TransportType.NONE - logistics = InvoiceLogistics( - carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), - driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), - transport_type=transport_type, - transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), - weight_type=weight_type or WeightUnit.KGS, - seal_number=(row_norm.get('PRECINTO') or None), - incoterm=(row_norm.get('CLAVE INCOTERM') or None), - entry_exit_date=parse_date(row_norm.get('FECHA EMISION'), date_format), - tenant_id=tenant_id, - company_id=company_id, - ) - - header.compliance_mx = compliance - header.financials = financials - if logistics: - header.logistics = logistics - - headers_to_insert.append(header) - - elif model_target == 'invoice_details': - invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or '').strip() - if not invoice_number: - skipped_invalid += 1 - continue - - cache_key = f"{invoice_number}|{inv_type_value}" - if cache_key in invoice_id_cache: - invoice_id = invoice_id_cache[cache_key] - else: - invoice_id = ( - session.query(InvoiceHeader.id) - .filter( - InvoiceHeader.tenant_id == tenant_id, - InvoiceHeader.company_id == company_id, - InvoiceHeader.invoice_number == invoice_number, - InvoiceHeader.invoice_type == inv_type_value, - ) - .scalar() - ) - invoice_id_cache[cache_key] = invoice_id - - if not invoice_id: - logger.warning( - "Invoice not found for details row %s (invoice_number=%s)", - i, - invoice_number, - ) - skipped_missing_invoice += 1 - continue - - part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() - if not part_num: - skipped_invalid += 1 - reason = "NUMPARTE: Requerido" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - if not validator.check_exists(Part, part_num, field_name="part_number"): - skipped_missing_fk += 1 - reason = f"NUMPARTE '{part_num}' no existe" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue - - # --- Prevent Duplicates: Clear existing line items for this invoice (Once per job) --- - if invoice_id not in cleared_invoices: - logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates") - session.query(LineItem).filter(LineItem.invoice_id == invoice_id).delete(synchronize_session=False) - session.query(InvoiceSalesDetails).filter(InvoiceSalesDetails.invoice_id == invoice_id).delete(synchronize_session=False) - cleared_invoices.add(invoice_id) - - # --- Partidas: LineItem with invoice_id (no Item parent) + full CSV mapping --- - part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() - part_id = part_cache.get(part_num) if part_num else None - if part_id is None and part_num: - p = session.query(Part.id).filter(Part.part_number == part_num, Part.tenant_id == tenant_id, Part.company_id == company_id).first() - if p: - part_id = p.id - part_cache[part_num] = part_id - - line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA')) - line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) - - class_code = (row_norm.get('CLASE') or '').strip().upper() - class_id = class_id_by_code.get(class_code) if class_code else None - uom_code = (row_norm.get('UNIDAD DE MEDIDA') or row_norm.get('UNIDAD MEDIDA') or '').strip().upper() - uom_id = uom_id_by_code.get(uom_code) if uom_code else None - bulk_key = (row_norm.get('CLAVE BULTOS') or row_norm.get('CLAVEBULTOS') or '').strip() - package_id = package_id_by_key.get(bulk_key) if bulk_key else None - - line = LineItem( - invoice_id=invoice_id, - line_number=line_num, - tenant_id=tenant_id, - company_id=company_id, - part_number_id=part_id, - class_id=class_id, - unit_of_measure=uom_id, - order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), - material_type=(row_norm.get('ID TYPE') or row_norm.get('IDTYPE') or None), - tax_payment=(str(row_norm.get('SE PAGO IMPUESTO') or row_norm.get('SEPAGOIMPUESTO') or '').strip().upper() == 'SI'), - payment_method=(row_norm.get('FORMA DE PAGO') or row_norm.get('FORMADEPAGO') or row_norm.get('FORMA PAGO') or None), - valuation_method=(row_norm.get('METODO DE VALORACION') or row_norm.get('METODODEVALORACION') or row_norm.get('METODO VALORACION') or None), - ) - session.add(line) - session.flush() - - price = parse_decimal(row_norm.get('COSTO UNITARIO') or row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO')) - if price is None: - total_val = parse_decimal(row_norm.get('TOTAL')) - qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD')) - price = (total_val / qty) if (total_val and qty and qty != 0) else None - qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD')) - commercial_total = (price * qty) if price and qty else parse_decimal(row_norm.get('TOTAL')) - - session.add(LineFinancial( - item_line_id=line.id, - unit_cost_capture=decimal_or_zero(price), - total_commercial_value=decimal_or_zero(commercial_total), - )) - - net_w = parse_decimal(row_norm.get('PESO NETO') or row_norm.get('PESONETO')) - gross_w = parse_decimal(row_norm.get('PESO BRUTO') or row_norm.get('PESOBRUTO')) - session.add(LineQuantity( - item_line_id=line.id, - quantity=decimal_or_zero(qty), - net_weight=decimal_or_zero(net_w), - gross_weight=decimal_or_zero(gross_w), - package_quantity=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), - package_id=package_id, - )) - - origin = (row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') or row_norm.get('PAIS') or '').strip() - fraction = (row_norm.get('FRACCION ARANCELARIA') or row_norm.get('FRACCION') or row_norm.get('FRACCIONARANCELARIA') or '').strip() - fraction_type = (row_norm.get('PREFERENCIA ARANCELARIA') or row_norm.get('PREFERENCIA') or '').strip() - sector = (row_norm.get('SECTOR') or '').strip() - american_fraction = (row_norm.get('FRACCION AMERICANA') or row_norm.get('FRACCIONAMERICANA') or '').strip() - session.add(LineCustom( - item_line_id=line.id, - origin_country=origin or None, - fraction=fraction or None, - fraction_type=fraction_type or None, - sector=sector or None, - american_fraction=american_fraction or None, - )) - - desc_es = (row_norm.get('DESCRIPCION ESPAÑOL') or row_norm.get('DESCRIPCIONE') or row_norm.get('DESCRIPCION') or '').strip() - desc_en = (row_norm.get('DESCRIPCION INGLES') or row_norm.get('DESCRIPCIONI') or '').strip() - brand = (row_norm.get('MARCA') or '').strip() - model = (row_norm.get('MODELO') or '').strip() - extra_desc = (row_norm.get('DESCRIPCION EXTRA') or row_norm.get('DESCRIPCIONEXTRA') or '').strip() - additional_info = (row_norm.get('INFORMACION ADICIONAL') or row_norm.get('INFORMACIONADICIONAL') or '').strip() - lot = (row_norm.get('LOTE') or '').strip() - entry_number = (row_norm.get('NUMERO ENTRADA') or row_norm.get('NUMEROENTRADA') or row_norm.get('NUM ENTRADA') or '').strip() - session.add(LineDescription( - item_line_id=line.id, - description_spanish=desc_es or None, - description_english=desc_en or None, - brand=brand or None, - model=model or None, - extra_description=extra_desc or None, - additional_info_spanish=additional_info or None, - lot=lot or None, - entry_number=entry_number or None, - )) - - session.add(InvoiceSalesDetails( - invoice_id=invoice_id, - line_number=line_num, - sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), - line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), - tenant_id=tenant_id, - company_id=company_id, - )) - details_to_insert.append(line) - - # 3. Bulk Insert (ORM Transaction) - try: - if model_target == 'invoice_header': - if headers_to_insert: - logger.info(f"Attempting to commit {len(headers_to_insert)} headers") - session.add_all(headers_to_insert) - session.commit() - inserted_count = len(headers_to_insert) - logger.info(f"Headers commit successful. Inserted: {inserted_count}") - else: - logger.warning(f"No headers to insert for job {job_id}") - else: - if details_to_insert: - logger.info(f"Attempting to commit {len(details_to_insert)} items and related data") - session.commit() # Everything was already added with session.add() - inserted_count = len(details_to_insert) - logger.info(f"Details commit successful. Inserted: {inserted_count}") - else: - logger.warning(f"No details to insert for job {job_id}") - - except Exception as db_err: - session.rollback() - logger.error(f"DB Error during {model_target} commit: {db_err}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(db_err)} - - # 4. Determine final status and prepare response (inside session block to access variables) - total_skipped = skipped_invalid + skipped_missing_fk + skipped_missing_invoice - - # Log summary - logger.info(f"Job {job_id} completed. Inserted: {inserted_count}, Skipped: {total_skipped} " - f"(invalid: {skipped_invalid}, missing_fk: {skipped_missing_fk}, missing_invoice: {skipped_missing_invoice})") - - # Prepare response based on results - if inserted_count == 0: - if total_skipped > 0: - logger.warning(f"No valid records to insert for job {job_id}. All {total_skipped} records were rejected.") - response = { - "status": "warning", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_invoice": skipped_missing_invoice, - "skipped_missing_fk": skipped_missing_fk, - "skipped_details": skipped_fk_details, - "message": f"No se insertaron registros. {total_skipped} fueron rechazados." - } - else: - logger.error(f"No valid records found in CSV for job {job_id}") - response = { - "status": "failed", - "error": "No hay registros válidos en el archivo CSV", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_invoice": skipped_missing_invoice, - "skipped_missing_fk": skipped_missing_fk, - "skipped_details": skipped_fk_details - } - else: - # Success case - at least some records were inserted - response = { - "status": "finished", - "inserted": inserted_count, - "skipped_invalid": skipped_invalid, - "skipped_missing_invoice": skipped_missing_invoice, - "skipped_missing_fk": skipped_missing_fk, - "skipped_details": skipped_fk_details - } - - except Exception as e: - logger.error(f"Task failed: {e}") - import traceback - logger.error(traceback.format_exc()) - return {"status": "failed", "error": str(e)} - - # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely - try: - common_storage.cleanup_import_job( - JOB_TYPE, job_id, - file_path=file_path, - error_path=error_path, - meta_path=meta_path, - ) - except Exception as cleanup_err: - logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) - - # Ensure response is defined (fallback in case of unexpected errors) - if response is None: - logger.error(f"Unexpected error: response not set for job {job_id}") - response = { - "status": "failed", - "error": "Error inesperado durante el procesamiento", - "inserted": 0, - "skipped_invalid": skipped_invalid, - "skipped_missing_invoice": skipped_missing_invoice, - "skipped_missing_fk": skipped_missing_fk, - "skipped_details": skipped_fk_details - } - - return response +import os +from datetime import datetime +from decimal import Decimal +import csv +import json +import logging +import re +import unicodedata +from typing import Dict, Any, Optional, List, Set, Tuple + +from core.celery_app import celery_app +from core.database import CoreSessionLocal +from core.paths import layout_path +from sqlalchemy import func + +from ..common import storage as common_storage +from ..common import meta as common_meta +from ..common import responses as common_responses +from .template_config import row_from_template +# Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process + +logger = logging.getLogger(__name__) + +# Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py) +JOB_TYPE = "" + +# Redis keys and TTL for import file/meta (exportados para routes; coinciden con common_storage cuando job_type="") +IMPORT_FILE_KEY_PREFIX = "import_file:" +IMPORT_META_KEY_PREFIX = "import_meta:" +IMPORT_ERROR_LINES_KEY_PREFIX = "import_error_lines:" +IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL + + +def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]: + """Usa common storage con job_type vacío (prefijo import_).""" + return common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Invoices import") + + +def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool: + return common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Invoices import") + + +def _delete_import_from_redis(job_id: str) -> None: + common_storage.delete_import_from_redis(JOB_TYPE, job_id) + +class ForeignKeyValidator: + def __init__(self, session, tenant_id, company_id): + self.session = session + self.tenant_id = tenant_id + self.company_id = company_id + self.cache = {} # {(model_name, value): bool} + + def check_exists(self, model, value, field_name="id", is_public=False): + if value is None: + return True # Assume optional if None, or let DB handle not-null + + key = (model.__name__, value) + if key in self.cache: + return self.cache[key] + + col = getattr(model, field_name) + if field_name == "short_name" and hasattr(model, "short_name"): + query = self.session.query(col).filter(func.upper(col) == (value.upper() if isinstance(value, str) else value)) + else: + query = self.session.query(col).filter(col == value) + if not is_public: + query = query.filter(model.tenant_id == self.tenant_id, model.company_id == self.company_id) + + exists = query.first() is not None + self.cache[key] = exists + return exists + + +TRANSPORT_TYPE_VALUES = { + "none", + "transport", + "box", + "licence plates", + "truck", + "vessel", + "rail_barge", + "container", + "airplane", + "gondola", + "flatbed", +} + + +def normalize_public_code(value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = str(value).strip().upper() + return text or None + + +def validate_public_code( + validator: ForeignKeyValidator, + model, + value: Optional[str], + line_num: int, + col_name: str, + field_name: str = "code", + required: bool = False, +) -> Optional[Dict[str, Any]]: + code = normalize_public_code(value) + if not code: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, code, field_name=field_name, is_public=True): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + + +def validate_tenant_fk_id( + validator: ForeignKeyValidator, + model, + value: Optional[int], + line_num: int, + col_name: str, + required: bool = False, +) -> Optional[Dict[str, Any]]: + if value is None: + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + if not validator.check_exists(model, value): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + + +def _validate_client_provider_ref( + validator: ForeignKeyValidator, + model, + raw_value: Any, + line_num: int, + col_name: str, + required: bool, +) -> Optional[Dict[str, Any]]: + """Valida CLAVE PROVEEDOR / VENDIDO A / ENVIADO A: acepta ID (entero) o short_name (texto).""" + if raw_value is None or not str(raw_value).strip(): + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + pid = parse_int(raw_value) + if pid is not None: + return validate_tenant_fk_id(validator, model, pid, line_num, col_name, required=False) + short_norm = str(raw_value).strip().upper() + if not validator.check_exists(model, short_norm, field_name="short_name"): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + + +def _validate_customs_broker_ref( + validator: ForeignKeyValidator, + model, + raw_value: Any, + line_num: int, + col_name: str, + required: bool = False, +) -> Optional[Dict[str, Any]]: + """Valida AGENTE ADUANAL: acepta ID (entero) o clave broker_key (texto).""" + if raw_value is None or not str(raw_value).strip(): + if required: + return {"line": line_num, "col": col_name, "msg": "Requerido"} + return None + pid = parse_int(raw_value) + if pid is not None: + return validate_tenant_fk_id(validator, model, pid, line_num, col_name, required=False) + clave = str(raw_value).strip() + if not validator.check_exists(model, clave, field_name="broker_key"): + return {"line": line_num, "col": col_name, "msg": "No existe en el catalogo"} + return None + +@celery_app.task(bind=True) +def scan_file(self, job_id: str, model_target: str, config: str = None): + """ + Pass 1: Read CSV, Validate types, Write Errors to JSONL. + File content is loaded from Redis (written by API on upload) so worker does not need shared filesystem. + """ + logger.info(f"Starting scan for job {job_id} target {model_target}") + + # 1. Get file from Redis and write to worker local disk + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + return {"status": "failed", "error": "File not found (missing or expired in queue). Please upload again."} + _ensure_worker_has_meta_from_redis(job_id, file_path) + + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + + total_rows = 0 + error_count = 0 + processed_rows = 0 + + # 3. Count Total (Quick Pass) or just estimate + try: + with open(file_path, 'r', encoding='utf-8-sig') as f: + total_rows = sum(1 for _ in f) - 1 # Minus header + except Exception as e: + return {"status": "failed", "error": f"Cannot read file: {e}"} + + footer_config = parse_footer_config(config) + date_format = footer_config.get("dateFormat") + + # Validate and set default date_format if not provided + if not date_format: + date_format = "yyyy-mm-dd" # Default to ISO format + logger.info(f"No date_format specified in config, using default: {date_format}") + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else + "imp_temp_details" if model_target == "invoice_details" else "imp_temp_series" + ) + + inv_type_value = normalize_public_code(footer_config.get("invoice_type") or meta.get("invoice_type") or "TEM") + if not inv_type_value: + inv_type_value = "TEM" + + # --- Series de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- + if model_target == "invoice_series": + try: + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.items.series.models import Serie + from .validators.series_impo_temp import ( + validate_row_series_impo_temp, + ) + + actualizar = meta.get("actualizar", False) + autonumerar = meta.get("autonumerar", True) + validar_series_exception = meta.get("validar_series", False) + _fc = parse_footer_config(meta.get("footer_config")) + if _fc: + if "actualizar" in _fc: + actualizar = bool(_fc["actualizar"]) + elif _fc.get("mode") == "update": + actualizar = True + elif _fc.get("mode") == "replace": + actualizar = False + if "autonumerar" in _fc: + autonumerar = bool(_fc["autonumerar"]) + else: + as_val = _fc.get("autonumber_series", "true") + autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") + if "validar_series" in _fc: + validar_series_exception = bool(_fc["validar_series"]) + + with CoreSessionLocal() as session: + # Invoice lookup: imp + TEM + q = ( + session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + rows_inv = q.all() + invoice_id_by_number: Dict[str, int] = {} + invoice_updated_by_number: Dict[str, bool] = {} + for num, iid, is_upd in rows_inv: + if num: + invoice_id_by_number[str(num).strip()] = iid + invoice_updated_by_number[str(num).strip()] = bool(is_upd) + + # Existing series keys: (invoice_number, linea_factura, linea_serie) + existing_series_keys: Set[Tuple[str, str, str]] = set() + existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + if actualizar and not autonumerar: + q_ser = ( + session.query( + InvoiceHeader.invoice_number, + LineItem.line_number, + Serie.row, + Serie.serial_numbers, + Serie.model, + Serie.sub_model, + Serie.number_id, + ) + .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) + .join(Serie, Serie.line_item_id == LineItem.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + for num, ln, rw, sn, md, sm, nid in q_ser.all(): + if num is not None: + k = (str(num).strip(), str(ln).strip(), str(rw).strip()) + existing_series_keys.add(k) + existing_series_data.setdefault(k, { + "serial_numbers": sn or "", + "model": md or "", + "sub_model": sm or "", + "number_id": nid or "", + }) + + with open(file_path, "r", encoding="utf-8-sig") as f_in, open(error_path, "w", encoding="utf-8") as f_err: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + errors_detail: List[Dict[str, Any]] = [] + error_lines_list: List[int] = [] + for i, row in enumerate(reader, start=1): + if i % 1000 == 0: + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) + row_norm = row_from_template(row, "imp_temp_series", normalize_header) + warnings_list: List[Dict[str, Any]] = [] + err = validate_row_series_impo_temp( + row_norm, + i, + actualizar=actualizar, + autonumerar=autonumerar, + validar_series_exception=validar_series_exception, + invoice_id_by_number=invoice_id_by_number, + invoice_updated_by_number=invoice_updated_by_number, + existing_series_keys=existing_series_keys, + existing_series_data=existing_series_data, + warnings=warnings_list, + ) + if err and not err.get("warning"): + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps(err) + "\n") + if len(errors_detail) < 500: + errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + for w in warnings_list: + if len(errors_detail) < 500: + errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows + ) + except Exception as e: + logger.exception("Series import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + # --- Partidas de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- + if model_target == "invoice_details" and template_id == "imp_temp_details": + try: + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.a76.general_catalogs.packages.models import Package + from api.v1.modules.public.reference_data.countries.models import Country + from api.v1.modules.public.reference_data.sectors.models import Sector + from api.v1.modules.public.reference_data.valuation_methods.models import ValuationMethod + from api.v1.modules.public.reference_data.payment_methods.models import PaymentMethod + from api.v1.modules.a76.general_catalogs.fractions.us_tariff_fractions.models import USTariffFraction + from api.v1.modules.a76.general_catalogs.company.models import Company + from api.v1.modules.a76.parts.models import Part + from .validators.partidas_impo_temp import validate_row_partidas_impo_temp + + _fc = parse_footer_config(meta.get("footer_config")) + autonumerar = meta.get("autonumerar", True) + actualizar = meta.get("actualizar", False) + levantar_subpartidas = meta.get("levantar_subpartidas", False) + calcular_costo_en_base_a_total = meta.get("calcular_costo_unitario_en_base_a_valor_total", False) + validar_decimales_pza = meta.get("validar_decimales_pza", False) + if _fc: + if "autonumerar" in _fc: + autonumerar = bool(_fc["autonumerar"]) + elif _fc.get("autonumber_partidas", "true") is not None: + autonumerar = str(_fc.get("autonumber_partidas", "true")).lower() in ("true", "1", "si", "sí", "yes") + if "actualizar" in _fc: + actualizar = bool(_fc["actualizar"]) + if "levantar_subpartidas" in _fc: + levantar_subpartidas = bool(_fc["levantar_subpartidas"]) + if "calcular_costo_unitario_en_base_a_valor_total" in _fc: + calcular_costo_en_base_a_total = bool(_fc["calcular_costo_unitario_en_base_a_valor_total"]) + if "validar_decimales_pza" in _fc: + validar_decimales_pza = bool(_fc["validar_decimales_pza"]) + + RFC_EXCEPTION_UPDATED = {"TPI121217SF6", "TCI170502858"} + RFC_EXCEPTION_NUM_PARTE = {"CTE980130518"} + + with CoreSessionLocal() as session: + q_inv = ( + session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + invoice_id_by_number: Dict[str, int] = {} + invoice_updated_by_number: Dict[str, bool] = {} + for num, iid, is_upd in q_inv.all(): + if num: + invoice_id_by_number[str(num).strip()] = iid + invoice_updated_by_number[str(num).strip()] = bool(is_upd) + + existing_line_keys_by_invoice: Dict[str, Set[str]] = {} + q_li = ( + session.query(InvoiceHeader.invoice_number, LineItem.line_number) + .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + for num, ln in q_li.all(): + if num is not None: + key = str(num).strip() + if key not in existing_line_keys_by_invoice: + existing_line_keys_by_invoice[key] = set() + existing_line_keys_by_invoice[key].add(str(ln).strip()) + + partidas_principales_bd: Set[Tuple[str, str]] = set() + try: + from api.v1.modules.a24.fa.fa_item_lines.models import FaLineItem + q_pp = ( + session.query(InvoiceHeader.invoice_number, LineItem.line_number) + .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) + .join(FaLineItem, FaLineItem.id == LineItem.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + FaLineItem.is_subitem == False, + FaLineItem.contains_subitems == True, + ) + ) + for num, ln in q_pp.all(): + if num is not None: + partidas_principales_bd.add((str(num).strip(), str(ln).strip())) + except Exception: + pass + + valid_class_codes: Set[str] = set() + class_um_by_code: Dict[str, str] = {} + class_fraction_by_code: Dict[str, str] = {} + class_desc_es_by_code: Dict[str, str] = {} + class_desc_en_by_code: Dict[str, str] = {} + for c in session.query(Class).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): + code = (c.class_code or "").strip().upper() + if code: + valid_class_codes.add(code) + class_um_by_code[code] = (c.unit_of_measure or "").strip().upper() + class_fraction_by_code[code] = (c.fraction or "").strip() + class_desc_es_by_code[code] = (c.description_es or "").strip() + class_desc_en_by_code[code] = (c.description_en or "").strip() + + valid_uom_codes: Set[str] = set() + for u in session.query(UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): + if u[0]: + valid_uom_codes.add((u[0] or "").strip().upper()) + + valid_bulks_codes: Set[str] = set() + for p in session.query(Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): + if p[0]: + valid_bulks_codes.add((p[0] or "").strip()) + + valid_country_keys: Set[str] = set() + for row in session.query(Country.m3_key, Country.ame_key).all(): + if row[0]: + valid_country_keys.add((row[0] or "").strip().upper()) + if row[1]: + valid_country_keys.add((row[1] or "").strip().upper()) + + valid_fraction_ame: Set[str] = set() + for row in session.query(USTariffFraction.code).filter(USTariffFraction.tenant_id == tenant_id, USTariffFraction.company_id == company_id).all(): + if row[0]: + valid_fraction_ame.add((row[0] or "").strip()) + + authorized_sectors: Set[str] = set() + for row in session.query(Sector.key).filter(Sector.authorized == True).all(): + if row[0]: + authorized_sectors.add((row[0] or "").strip().upper()) + + valid_payment_methods: Set[str] = set() + for row in session.query(PaymentMethod.key).all(): + if row[0] is not None: + valid_payment_methods.add(str(row[0]).strip()) + + valid_valuation_methods: Set[str] = set() + for row in session.query(ValuationMethod.key).all(): + if row[0]: + valid_valuation_methods.add((row[0] or "").strip()) + + company = session.query(Company).filter(Company.id == company_id).first() + company_has_prosec = bool(company.prosec) if company else False + company_rfc = (company.rfc or "").strip().upper() if company else "" + + valid_part_numbers: Set[str] = set() + for row in session.query(Part.part_number).filter(Part.tenant_id == tenant_id, Part.company_id == company_id).all(): + if row[0]: + valid_part_numbers.add((row[0] or "").strip().upper()) + + rfc_exception_updated: Set[str] = set() + rfc_exception_num_parte: Set[str] = set() + + with open(file_path, "r", encoding="utf-8-sig") as f_in: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + rows_list = list(reader) + + invoice_numbers_from_csv = set() + for row in rows_list: + inv = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() + if inv: + invoice_numbers_from_csv.add(inv) + if company_rfc in RFC_EXCEPTION_UPDATED: + rfc_exception_updated = invoice_numbers_from_csv + if company_rfc in RFC_EXCEPTION_NUM_PARTE: + rfc_exception_num_parte = invoice_numbers_from_csv + + line_counts_csv: Dict[Tuple[str, str], int] = {} + partidas_principales_csv: Set[Tuple[str, str]] = set() + + def _get_row(row_norm: Dict[str, Any], *keys: str) -> str: + for k in keys: + v = row_norm.get(k) + if v is not None and str(v).strip(): + return str(v).strip() + return "" + + for row in rows_list: + row_norm = row_from_template(row, "imp_temp_details", normalize_header) + inv = _get_row(row_norm, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") + linea = _get_row(row_norm, "LINEA", "RENGLON", "PARTIDA") + if inv and linea: + key = (inv, linea) + line_counts_csv[key] = line_counts_csv.get(key, 0) + 1 + u = _get_row(row_norm, "ES PARTIDA O SUBPARTIDA", "ESSUBPARTIDA").upper() + if u == "P" and inv and linea: + partidas_principales_csv.add((inv, linea)) + + error_count = 0 + processed_rows = 0 + error_lines_list = [] + errors_detail = [] + + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in enumerate(rows_list, start=1): + if i % 1000 == 0: + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + row_norm = row_from_template(row, "imp_temp_details", normalize_header) + err = validate_row_partidas_impo_temp( + row_norm, + i, + autonumerar=autonumerar, + actualizar=actualizar, + levantar_subpartidas=levantar_subpartidas, + calcular_costo_en_base_a_total=calcular_costo_en_base_a_total, + validar_decimales_pza=validar_decimales_pza, + invoice_id_by_number=invoice_id_by_number, + invoice_updated_by_number=invoice_updated_by_number, + rfc_exception_updated=rfc_exception_updated, + existing_line_keys_by_invoice=existing_line_keys_by_invoice, + line_counts_csv=line_counts_csv, + partidas_principales_csv=partidas_principales_csv, + partidas_principales_bd=partidas_principales_bd, + valid_class_codes=valid_class_codes, + class_um_by_code=class_um_by_code, + class_fraction_by_code=class_fraction_by_code, + class_desc_es_by_code=class_desc_es_by_code, + class_desc_en_by_code=class_desc_en_by_code, + valid_uom_codes=valid_uom_codes, + valid_bulks_codes=valid_bulks_codes, + valid_country_keys=valid_country_keys, + valid_fraction_ame=valid_fraction_ame, + valid_payment_methods=valid_payment_methods, + valid_valuation_methods=valid_valuation_methods, + authorized_sectors=authorized_sectors, + company_has_prosec=company_has_prosec, + rfc_exception_num_parte=rfc_exception_num_parte or None, + valid_part_numbers=valid_part_numbers, + warnings=None, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") + if len(errors_detail) < 500: + errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + except Exception as e: + logger.exception("Partidas import scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + # --- Encabezados de Importación Temporal: flujo específico (Clarion VALIDA_TODA / VALIDA_PARCIAL) --- + if model_target == "invoice_header" and template_id == "imp_temp_header": + try: + from api.v1.modules.a76.invoices.models import InvoiceHeader, InvoiceComplianceMx, InvoiceFinancials + from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos + from api.v1.modules.a76.pedmientos.models.pedimento_dates import PedimentoDates + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate + from api.v1.modules.a76.transportation.transporters.models import Transporter + from .validators.encabezados_impo_temp import ( + validate_row_encabezados_impo_temp, + parse_pedimento_col_a, + _pedimento_key_from_parsed, + ) + + def _ped_key_from_row(ped_str: str) -> Optional[str]: + parsed = parse_pedimento_col_a(ped_str) + if not parsed: + return None + return _pedimento_key_from_parsed(parsed[0], parsed[1], parsed[2]) + + _fc = parse_footer_config(meta.get("footer_config")) + actualizar = meta.get("actualizar", False) + autonumerar_remesas = meta.get("autonumerar_remesas", False) + control_remesa = bool(_fc.get("control_remesa", False)) + remesa_inicio = _fc.get("remesa_inicio") + remesa_fin = _fc.get("remesa_fin") + if remesa_inicio is not None: + try: + remesa_inicio = int(remesa_inicio) + except (TypeError, ValueError): + remesa_inicio = None + if remesa_fin is not None: + try: + remesa_fin = int(remesa_fin) + except (TypeError, ValueError): + remesa_fin = None + if _fc: + if "actualizar" in _fc: + actualizar = bool(_fc["actualizar"]) + if "autonumerar_remesas" in _fc: + autonumerar_remesas = bool(_fc["autonumerar_remesas"]) + + with CoreSessionLocal() as session: + q_inv = ( + session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + invoice_exists_by_number: Dict[str, bool] = {} + invoice_updated_by_number: Dict[str, bool] = {} + for num, iid, is_upd in q_inv.all(): + if num: + n = str(num).strip() + invoice_exists_by_number[n] = True + invoice_updated_by_number[n] = bool(is_upd) + + pedimento_data_by_key: Dict[str, List[Dict[str, Any]]] = {} + for p in ( + session.query( + Pedimentos.id, + Pedimentos.year, + Pedimentos.customs_office, + Pedimentos.license, + Pedimentos.pedimento_number, + Pedimentos.operation_type, + Pedimentos.regime, + Pedimentos.pedimento_type, + ) + .filter( + Pedimentos.tenant_id == tenant_id, + Pedimentos.company_id == company_id, + ) + .all() + ): + co = (p.customs_office or "").strip() + lic = (p.license or "").strip() + num = (p.pedimento_number or "").strip() + if not co or not lic or not num: + continue + key = _pedimento_key_from_parsed(co, lic, num) + entry_date = None + end_date = None + pd = ( + session.query(PedimentoDates.entry_date, PedimentoDates.end_date) + .filter(PedimentoDates.pedimento_id == p.id).first() + ) + if pd: + entry_date = pd[0] + end_date = pd[1] + info = { + "id": p.id, + "regime": (p.regime or "").strip(), + "operation_type": (p.operation_type or "").strip(), + "pedimento_type": (p.pedimento_type or "").strip(), + "entry_date": entry_date, + "end_date": end_date, + } + if key not in pedimento_data_by_key: + pedimento_data_by_key[key] = [] + pedimento_data_by_key[key].append(info) + + remesa_por_pedimento_bd: Dict[str, Set[int]] = {} + q_rem = ( + session.query( + InvoiceComplianceMx.remesa, + Pedimentos.year, + Pedimentos.customs_office, + Pedimentos.license, + Pedimentos.pedimento_number, + ) + .join(Pedimentos, Pedimentos.id == InvoiceComplianceMx.pedimento_id) + .filter( + InvoiceComplianceMx.tenant_id == tenant_id, + InvoiceComplianceMx.company_id == company_id, + InvoiceComplianceMx.pedimento_id.isnot(None), + InvoiceComplianceMx.remesa.isnot(None), + ) + ) + for rem, y, co, lic, num in q_rem.all(): + if co and lic and num and rem is not None: + key = _pedimento_key_from_parsed( + (co or "").strip(), + (lic or "").strip(), + (num or "").strip(), + ) + if key not in remesa_por_pedimento_bd: + remesa_por_pedimento_bd[key] = set() + remesa_por_pedimento_bd[key].add(int(rem)) + + valid_provider_ids: Set[int] = set() + valid_sold_to_ids: Set[int] = set() + valid_shipped_to_ids: Set[int] = set() + valid_provider_short_names: Set[str] = set() + valid_sold_to_short_names: Set[str] = set() + valid_shipped_to_short_names: Set[str] = set() + for cp in session.query(ClientProvider.id, ClientProvider.short_name).filter( + ClientProvider.tenant_id == tenant_id, + ClientProvider.company_id == company_id, + ).all(): + valid_provider_ids.add(cp[0]) + valid_sold_to_ids.add(cp[0]) + valid_shipped_to_ids.add(cp[0]) + if cp[1] and str(cp[1]).strip(): + sn_upper = str(cp[1]).strip().upper() + valid_provider_short_names.add(sn_upper) + valid_sold_to_short_names.add(sn_upper) + valid_shipped_to_short_names.add(sn_upper) + + valid_broker_ids: Set[int] = set() + valid_broker_claves: Set[str] = set() + for cb in session.query(CustomsBroker.id, CustomsBroker.broker_key).filter( + CustomsBroker.tenant_id == tenant_id, + CustomsBroker.company_id == company_id, + ).all(): + valid_broker_ids.add(cb[0]) + if cb[1] and str(cb[1]).strip(): + valid_broker_claves.add(str(cb[1]).strip()) + + valid_transporter_keys: Set[str] = set() + for t in session.query(Transporter.transporter_key).filter( + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ).all(): + if t[0]: + valid_transporter_keys.add((t[0] or "").strip().upper()) + + valid_incoterms: Set[str] = set() + for inc in session.query(Incoterm.code).all(): + if inc[0]: + valid_incoterms.add((inc[0] or "").strip().upper()) + + valid_aduana_codes: Set[str] = set() + for cs in session.query(CustomsSection.customs_code).all(): + if cs[0]: + valid_aduana_codes.add((cs[0] or "").strip()) + + valid_currency_codes: Set[str] = set() + for ct in session.query(CurrencyType.code).all(): + if ct[0]: + valid_currency_codes.add((ct[0] or "").strip().upper()) + + exchange_rate_by_date: Dict[str, Any] = {} + for er in session.query(ExchangeRate.date, ExchangeRate.value).filter( + ExchangeRate.tenant_id == tenant_id, + ExchangeRate.company_id == company_id, + ).all(): + if er[0] and er[1] is not None: + dk = er[0].strftime("%Y-%m-%d") if hasattr(er[0], "strftime") else str(er[0])[:10] + exchange_rate_by_date[dk] = er[1] + + invoice_has_partidas_by_number: Dict[str, bool] = {} + existing_tipo_moneda_by_number: Dict[str, str] = {} + q_li_count = ( + session.query(InvoiceHeader.invoice_number, func.count(LineItem.id)) + .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + .group_by(InvoiceHeader.invoice_number) + ) + for num, cnt in q_li_count.all(): + if num: + invoice_has_partidas_by_number[str(num).strip()] = cnt > 0 + q_fin = ( + session.query(InvoiceHeader.invoice_number, InvoiceFinancials.currency) + .join(InvoiceFinancials, InvoiceFinancials.invoice_id == InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + for num, cur in q_fin.all(): + if num and cur: + cur_str = (cur or "").strip().lower() + if cur_str == "foreign": + existing_tipo_moneda_by_number[str(num).strip()] = "ME" + elif cur_str == "local": + existing_tipo_moneda_by_number[str(num).strip()] = "MN" + else: + existing_tipo_moneda_by_number[str(num).strip()] = cur_str.upper()[:2] + + with open(file_path, "r", encoding="utf-8-sig") as f_in: + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f_in, dialect=dialect) + rows_list = list(reader) + + remesa_por_pedimento_csv: Dict[str, Dict[int, str]] = {} + for row in rows_list: + row_norm = row_from_template(row, "imp_temp_header", normalize_header) + ped = (row_norm.get("PEDIMENTO") or "").strip() + rem = row_norm.get("REMESA") + factura = (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or row_norm.get("FACTURA") or "").strip() + if not ped or not factura: + continue + key = _ped_key_from_row(ped) + if not key: + continue + try: + rem_int = int(rem) if rem is not None and str(rem).strip() else None + except (TypeError, ValueError): + rem_int = None + if rem_int is not None: + if key not in remesa_por_pedimento_csv: + remesa_por_pedimento_csv[key] = {} + if rem_int not in remesa_por_pedimento_csv[key]: + remesa_por_pedimento_csv[key][rem_int] = factura + + error_count = 0 + processed_rows = 0 + error_lines_list: List[int] = [] + errors_detail: List[Dict[str, Any]] = [] + + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in enumerate(rows_list, start=1): + if i % 1000 == 0: + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + row_norm = row_from_template(row, "imp_temp_header", normalize_header) + warnings_row: List[Dict[str, Any]] = [] + err = validate_row_encabezados_impo_temp( + row_norm, + i, + actualizar=actualizar, + invoice_exists_by_number=invoice_exists_by_number, + invoice_updated_by_number=invoice_updated_by_number, + pedimento_data_by_key=pedimento_data_by_key, + remesa_por_pedimento_bd=remesa_por_pedimento_bd, + remesa_por_pedimento_csv=remesa_por_pedimento_csv, + valid_provider_ids=valid_provider_ids, + valid_sold_to_ids=valid_sold_to_ids, + valid_shipped_to_ids=valid_shipped_to_ids, + valid_provider_short_names=valid_provider_short_names, + valid_sold_to_short_names=valid_sold_to_short_names, + valid_shipped_to_short_names=valid_shipped_to_short_names, + valid_broker_ids=valid_broker_ids, + valid_broker_claves=valid_broker_claves, + valid_transporter_keys=valid_transporter_keys, + valid_incoterms=valid_incoterms, + valid_aduana_codes=valid_aduana_codes, + valid_currency_codes=valid_currency_codes, + exchange_rate_by_date=exchange_rate_by_date, + invoice_has_partidas_by_number=invoice_has_partidas_by_number, + existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, + autonumerar_remesas=autonumerar_remesas, + control_remesa=control_remesa, + remesa_inicio=remesa_inicio, + remesa_fin=remesa_fin, + date_format=date_format, + parse_date_fn=parse_date, + warnings=warnings_row, + ) + if err: + error_count += 1 + error_lines_list.append(err["line"]) + f_err.write(json.dumps({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + "\n") + if len(errors_detail) < 500: + errors_detail.append({"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}) + for w in warnings_row: + if len(errors_detail) < 500: + errors_detail.append({"line": w["line"], "col": w.get("col", ""), "msg": w.get("msg", ""), "warning": True}) + processed_rows += 1 + + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + return common_responses.scan_result( + job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows + ) + except Exception as e: + logger.exception("Encabezados importación temporal scan failed: %s", e) + return {"status": "failed", "error": str(e)} + + try: + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import ( + CodePedimentoRegimen, + ) + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm + from api.v1.modules.a76.parts.models import Part + + models = { + "InvoiceHeader": InvoiceHeader, + "InvoiceType": InvoiceType, + "ClientProvider": ClientProvider, + "CustomsBroker": CustomsBroker, + "RegimenPedimento": RegimenPedimento, + "CodePedimentoRegimen": CodePedimentoRegimen, + "PedimentoCode": PedimentoCode, + "CurrencyType": CurrencyType, + "CustomsSection": CustomsSection, + "Incoterm": Incoterm, + "Part": Part, + } + + with CoreSessionLocal() as session, \ + open(file_path, 'r', encoding='utf-8-sig') as f_in, \ + open(error_path, 'w', encoding='utf-8') as f_err: + validator = ForeignKeyValidator(session, tenant_id, company_id) + invoice_id_cache: Dict[str, Optional[int]] = {} + + # Detect Delimiter + sample = f_in.read(2048) + f_in.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except: + dialect = 'excel' + + reader = csv.DictReader(f_in, dialect=dialect) + + for i, row in enumerate(reader, start=1): + # Check for Progress Update + if i % 1000 == 0: + self.update_state(state='PROGRESS', meta={ + 'current': i, + 'total': total_rows, + 'errors': error_count + }) + + # Solo columnas de la plantilla (respetar plantilla tal cual) + row_norm = row_from_template(row, template_id, normalize_header) + errors = validate_row_strict( + row_norm, + model_target, + i, + date_format, + validator, + inv_type_value, + invoice_id_cache, + models, + ) + + if errors: + error_count += 1 + # Write simple JSON error + f_err.write(json.dumps(errors) + "\n") + + processed_rows += 1 + + except Exception as e: + logger.error(f"Scan failed: {e}") + return {"status": "failed", "error": str(e)} + + # 4. Store error line numbers in Redis so insert_valid_rows can skip them (any worker) + error_lines_list = [] + errors_detail: List[Dict[str, Any]] = [] + try: + if os.path.exists(error_path): + with open(error_path, "r", encoding="utf-8") as f: + for line in f: + try: + err = json.loads(line) + if "line" in err: + error_lines_list.append(err["line"]) + if len(errors_detail) < 500: + errors_detail.append( + { + "line": err["line"], + "col": err.get("col", ""), + "msg": err.get("msg", ""), + } + ) + except Exception: + pass + if error_lines_list: + common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) + except Exception as e: + logger.warning(f"Failed to store error lines in Redis: {e}") + + return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) + +def validate_row_phase_1( + row: Dict[str, Any], + target: str, + line_num: int, + date_format: Optional[str], +) -> Optional[Dict[str, Any]]: + """ + Validation: Unique IDs, Dates, and Numeric constraint checks. + Target: 'invoice_header' or 'invoice_details' + """ + def check_decimal(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_decimal(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número decimal válido"} + return None + + def check_int(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_int(val) is None: + return {"line": line_num, "col": col_name, "msg": "Debe ser un número entero válido"} + return None + + def check_date(col_name): + date_str = row.get(col_name) + if date_str and str(date_str).strip(): + if not is_valid_date(date_str, date_format): + expected = display_date_format(date_format) + return { + "line": line_num, + "col": col_name, + "msg": f"Formato de fecha inválido ({expected})", + } + return None + + def check_weight(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if parse_weight_unit(val) is None: + return {"line": line_num, "col": col_name, "msg": "Unidad de peso inválida (ej. KGS, LBS)"} + return None + + def check_currency(col_name): + val = row.get(col_name) + if val and str(val).strip(): + parsed_currency = parse_currency(val, None) + val_norm = normalize_header(val) + # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + return {"line": line_num, "col": col_name, "msg": "Moneda inválida (ej. MN, ME, USD, PESOS)"} + return None + + def check_transport_type(col_name): + val = row.get(col_name) + if val and str(val).strip(): + if str(val).strip().lower() not in TRANSPORT_TYPE_VALUES: + return {"line": line_num, "col": col_name, "msg": "Tipo de transporte inválido (ej. box, truck, container)"} + return None + + # A. Invoice Header + if target == 'invoice_header': + # 1. Unique ID + if not row.get('NUMERO FACTURA') and not row.get('NUM FACTURA') and not row.get('ID'): + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} + + # 2. Date Format + date_str = row.get('FECHA FACTURA') or row.get('FECHA') + if not date_str or not str(date_str).strip(): + return {"line": line_num, "col": "FECHA FACTURA", "msg": "Requerido"} + + err = check_date('FECHA FACTURA') or check_date('FECHA') + if err: return err + + err = check_date('FECHA EMISION') + if err: return err + + # 3. Numeric Fields + for col in ['TIPO DE CAMBIO', 'FLETES', 'VALOR SEGUROS', 'SEGUROS', 'EMBALAJES', 'OTROS INCREMENTABLES']: + err = check_decimal(col) + if err: return err + + # 4. Integer FKs (CLAVE PROVEEDOR/VENDIDO/ENVIADO aceptan short_name; AGENTE ADUANAL acepta clave; solo REMESA exige entero) + int_fk_cols = ['CLAVE PROVEEDOR', 'CLAVE VENDIDO A', 'CLAVE ENVIADO A', 'AGENTE ADUANAL', 'REMESA'] + if target == 'invoice_header': + int_fk_cols = ['REMESA'] # proveedor/vendido/enviado por short_name; agente aduanal por clave + for col in int_fk_cols: + err = check_int(col) + if err: return err + + # 5. Enums + for col in ['TIPO PESO']: + err = check_weight(col) + if err: return err + + for col in ['TIPO MONEDA']: + err = check_currency(col) + if err: return err + + for col in ['TIPO TRANSPORTE']: + err = check_transport_type(col) + if err: return err + + # B. Invoice Details (Parts) + elif target == 'invoice_details': + # 1. Line Number + if not row.get('LINEA') and not row.get('RENGLON') and not row.get('PARTIDA'): + return {"line": line_num, "col": "LINEA", "msg": "Requerido"} + + # 2. Parent Link (Invoice Number) + if not (row.get('NUMERO FACTURA') or row.get('NUM FACTURA') or row.get('FACTURA')): + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} + + # 3. Numeric Fields + for col in ['PRECIO UNITARIO', 'PRECIOUNITARIO', 'VALOR COMERCIAL', 'VALORCOMERCIAL', 'CANTIDAD']: + err = check_decimal(col) + if err: return err + + for col in ['CANTIDAD BULTOS', 'CANTIDADBULTOS', 'LINEA', 'RENGLON', 'PARTIDA']: + err = check_int(col) + if err: return err + + return None + + +def validate_row_strict( + row: Dict[str, Any], + target: str, + line_num: int, + date_format: Optional[str], + validator: ForeignKeyValidator, + inv_type_value: str, + invoice_id_cache: Dict[str, Optional[int]], + models: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + err = validate_row_phase_1(row, target, line_num, date_format) + if err: + return err + + InvoiceHeader = models["InvoiceHeader"] + InvoiceType = models["InvoiceType"] + ClientProvider = models["ClientProvider"] + CustomsBroker = models["CustomsBroker"] + RegimenPedimento = models["RegimenPedimento"] + CurrencyType = models["CurrencyType"] + CustomsSection = models["CustomsSection"] + Incoterm = models["Incoterm"] + Part = models["Part"] + + if target == "invoice_header": + if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): + return {"line": line_num, "col": "TIPO FACTURA", "msg": "No existe en el catalogo"} + + err = _validate_client_provider_ref( + validator, ClientProvider, row.get("CLAVE PROVEEDOR"), line_num, "CLAVE PROVEEDOR", required=True + ) + if err: + return err + + err = _validate_client_provider_ref( + validator, ClientProvider, row.get("CLAVE VENDIDO A"), line_num, "CLAVE VENDIDO A", required=True + ) + if err: + return err + + err = _validate_client_provider_ref( + validator, ClientProvider, row.get("CLAVE ENVIADO A"), line_num, "CLAVE ENVIADO A", required=True + ) + if err: + return err + + err = _validate_customs_broker_ref( + validator, CustomsBroker, row.get("AGENTE ADUANAL"), line_num, "AGENTE ADUANAL", required=False + ) + if err: + return err + + err = validate_public_code( + validator, + RegimenPedimento, + row.get("REGIMEN") or row.get("CLAVEDOCUMENTO"), + line_num, + "CLAVEDOCUMENTO", + ) + if err: + return err + + err = validate_public_code( + validator, + CustomsSection, + row.get("ADUANA DE CRUCE"), + line_num, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + return err + + err = validate_public_code( + validator, + CurrencyType, + row.get("CLAVE MONEDA"), + line_num, + "CLAVE MONEDA", + ) + if err: + return err + + err = validate_public_code( + validator, + Incoterm, + row.get("CLAVE INCOTERM"), + line_num, + "CLAVE INCOTERM", + ) + if err: + return err + + elif target == "invoice_details": + invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() + if not invoice_number: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Requerido"} + + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] + else: + invoice_id = ( + validator.session.query(InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == validator.tenant_id, + InvoiceHeader.company_id == validator.company_id, + InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, + ) + .scalar() + ) + invoice_id_cache[cache_key] = invoice_id + if not invoice_id: + return {"line": line_num, "col": "NUMERO FACTURA", "msg": "Factura no existe"} + + part_num = (row.get("NUMPARTE") or row.get("NUMERO PARTE") or "").strip() + if not part_num: + return {"line": line_num, "col": "NUMPARTE", "msg": "Requerido"} + if not validator.check_exists(Part, part_num, field_name="part_number"): + return {"line": line_num, "col": "NUMPARTE", "msg": "No existe en el catalogo"} + + return None + +def parse_footer_config(config: Optional[str]) -> Dict[str, Any]: + if not config: + return {} + try: + if isinstance(config, str): + return json.loads(config) + if isinstance(config, dict): + return config + except Exception: + return {} + return {} + + +def display_date_format(date_format: Optional[str]) -> str: + if not date_format: + return "YYYY-MM-DD" + return date_format.upper() + + +def parse_date(date_text: Optional[str], date_format: Optional[str]) -> Optional[datetime.date]: + if not date_text: + return None + candidates = [] + fmt_map = { + "dd/mm/yyyy": "%d/%m/%Y", + "mm/dd/yyyy": "%m/%d/%Y", + "yyyy-mm-dd": "%Y-%m-%d", + } + if date_format and date_format in fmt_map: + candidates.append(fmt_map[date_format]) + candidates.extend(["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y"]) + for fmt in candidates: + try: + return datetime.strptime(str(date_text).strip(), fmt).date() + except ValueError: + continue + return None + + +def is_valid_date(date_text: Optional[str], date_format: Optional[str]) -> bool: + return parse_date(date_text, date_format) is not None + + +def normalize_header(name: Optional[str]) -> str: + if not name: + return "" + name = unicodedata.normalize("NFKD", str(name)).upper() + name = "".join(ch for ch in name if not unicodedata.combining(ch)) + name = re.sub(r"[^A-Z0-9]+", " ", name) + return re.sub(r"\s+", " ", name).strip() + + +def normalize_row(row: Dict[str, Any]) -> Dict[str, Any]: + return {normalize_header(k): v for k, v in row.items()} + + +def parse_int(value: Any) -> Optional[int]: + if value is None: + return None + text = str(value).strip() + if not text: + return None + try: + return int(text) + except ValueError: + pass + try: + f = float(text.replace(",", "")) + if f == int(f): + return int(f) + return None + except ValueError: + return None + + +def parse_decimal(value: Any) -> Optional[Decimal]: + if value is None: + return None + text = str(value).strip() + if not text: + return None + text = text.replace(",", "") + try: + return Decimal(text) + except Exception: + return None + + +def decimal_or_zero(value: Any) -> Decimal: + """Return parsed decimal or Decimal('0') for CSV nulls/empty (vanilla default).""" + return parse_decimal(value) or Decimal("0") + + +def int_or_zero(value: Any) -> int: + """Return parsed int or 0 for CSV nulls/empty (vanilla default).""" + return parse_int(value) if parse_int(value) is not None else 0 + + +def parse_currency(value: Optional[str], currency_type: Optional[str]): + from api.v1.modules.a76.invoices.models import Currency + if value: + normalized = normalize_header(value) + if normalized in {"MN", "M N", "NACIONAL", "LOCAL", "PESOS", "PESO"}: + return Currency.LOCAL + if normalized in {"ME", "M E", "EXTRANJERA", "EXTRANJERO", "FOREIGN", "USD", "DOLAR", "DOLARES"}: + return Currency.FOREIGN + if "MANUAL" in normalized: + return Currency.MANUAL + if currency_type and str(currency_type).strip().upper() == "MXN": + return Currency.LOCAL + if currency_type: + return Currency.FOREIGN + return Currency.MANUAL + + +def parse_weight_unit(value: Optional[str]): + from api.v1.modules.a76.invoices.models import WeightUnit + if not value: + return None + normalized = normalize_header(value) + if normalized in {"KG", "KGS", "KILOS", "KILOGRAMOS"}: + return WeightUnit.KGS + if normalized in {"LB", "LBS", "LIBRAS"}: + return WeightUnit.LBS + return None + + +def resolve_tenant_fk_id( + session: CoreSessionLocal, + model, + value: Optional[int], + tenant_id: int, + company_id: int, + cache: Dict[int, Optional[int]], +) -> Optional[int]: + if value is None: + return None + if value in cache: + return cache[value] + exists = ( + session.query(model.id) + .filter( + model.id == value, + model.tenant_id == tenant_id, + model.company_id == company_id, + ) + .scalar() + ) + cache[value] = value if exists is not None else None + return cache[value] + + +def resolve_client_provider_id( + session: CoreSessionLocal, + model, + value: Any, + tenant_id: int, + company_id: int, + cache: Dict[Any, Optional[int]], +) -> Optional[int]: + """Resuelve ID de ClientProvider por id (entero) o por short_name (texto). value puede ser int o str.""" + if value is None or (isinstance(value, str) and not value.strip()): + return None + if value in cache: + return cache[value] + pid = parse_int(value) + if pid is not None: + found = ( + session.query(model.id) + .filter( + model.id == pid, + model.tenant_id == tenant_id, + model.company_id == company_id, + ) + .scalar() + ) + cache[value] = found + return found + short_norm = str(value).strip().upper() + if short_norm in cache: + return cache[short_norm] + found = ( + session.query(model.id) + .filter( + func.upper(model.short_name) == short_norm, + model.tenant_id == tenant_id, + model.company_id == company_id, + ) + .scalar() + ) + cache[value] = found + cache[short_norm] = found + return found + + +def resolve_customs_broker_id( + session: CoreSessionLocal, + model, + value: Any, + tenant_id: int, + company_id: int, + cache: Dict[Any, Optional[int]], +) -> Optional[int]: + """Resuelve ID de CustomsBroker por id (entero) o por broker_key (clave). value puede ser int o str.""" + if value is None or (isinstance(value, str) and not value.strip()): + return None + if value in cache: + return cache[value] + pid = parse_int(value) + if pid is not None: + found = ( + session.query(model.id) + .filter( + model.id == pid, + model.tenant_id == tenant_id, + model.company_id == company_id, + ) + .scalar() + ) + cache[value] = found + return found + clave = str(value).strip() + if clave in cache: + return cache[clave] + found = ( + session.query(model.id) + .filter( + model.broker_key == clave, + model.tenant_id == tenant_id, + model.company_id == company_id, + ) + .scalar() + ) + cache[value] = found + cache[clave] = found + return found + + +def resolve_public_code( + session: CoreSessionLocal, + model, + column, + value: Optional[str], + cache: Dict[str, Optional[str]], +) -> Optional[str]: + if not value: + return None + normalized = str(value).strip().upper() + if not normalized: + return None + if normalized in cache: + return cache[normalized] + exists = session.query(column).filter(column == normalized).scalar() + cache[normalized] = normalized if exists is not None else None + return cache[normalized] + +@celery_app.task(bind=True) +def insert_valid_rows(self, job_id: str, model_target: str): + """ + Pass 2: Re-read CSV, Skip Errors, Bulk Insert. + File and meta are loaded from Redis if present (same as scan_file), so worker does not need shared filesystem. + """ + logger.info(f"Starting Commit for {job_id} target {model_target}") + + # Ensure we have the file on this worker: prefer Redis (so any worker can run commit) + file_path = _ensure_worker_has_file_from_redis(job_id) + if not file_path: + alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) + if not os.path.exists(alt_path): + return {"status": "failed", "error": "File not found (missing or expired). Please upload and confirm again."} + file_path = alt_path + else: + _ensure_worker_has_meta_from_redis(job_id, file_path) + + try: + tenant_id, company_id = common_meta.require_tenant_context(file_path) + except ValueError as e: + return {"status": "failed", "error": str(e)} + + meta = common_meta.load_meta(file_path) + meta_path = common_meta.get_meta_path(file_path) + error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) + error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) + + # Si el upload fue de series (template_id imp_temp_series), usar flujo series aunque model_target venga mal + use_series_flow = ( + model_target == "invoice_series" + or meta.get("template_id") == "imp_temp_series" + ) + + # --- Series de Importación Temporal: commit (INSERT/UPDATE item_line_series) --- + if use_series_flow: + try: + from api.v1.modules.a76.invoices.models import InvoiceHeader + from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.items.series.models import Serie + from .validators.series_impo_temp import ( + validate_row_series_impo_temp, + row_to_series_normalized, + ) + + actualizar = meta.get("actualizar", False) + autonumerar = meta.get("autonumerar", True) + validar_series_exception = meta.get("validar_series", False) + _fc = parse_footer_config(meta.get("footer_config")) + if _fc: + if "actualizar" in _fc: + actualizar = bool(_fc["actualizar"]) + elif _fc.get("mode") == "update": + actualizar = True + elif _fc.get("mode") == "replace": + actualizar = False + if "autonumerar" in _fc: + autonumerar = bool(_fc["autonumerar"]) + else: + as_val = _fc.get("autonumber_series", "true") + autonumerar = str(as_val).lower() in ("true", "1", "si", "sí", "yes") + if "validar_series" in _fc: + validar_series_exception = bool(_fc["validar_series"]) + + with CoreSessionLocal() as session: + q = ( + session.query(InvoiceHeader.invoice_number, InvoiceHeader.id, InvoiceHeader.is_updated) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + rows_inv = q.all() + invoice_id_by_number: Dict[str, int] = {} + invoice_updated_by_number: Dict[str, bool] = {} + for num, iid, is_upd in rows_inv: + if num: + invoice_id_by_number[str(num).strip()] = iid + invoice_updated_by_number[str(num).strip()] = bool(is_upd) + + existing_series_keys: Set[Tuple[str, str, str]] = set() + existing_series_data: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + if actualizar and not autonumerar: + q_ser = ( + session.query( + InvoiceHeader.invoice_number, + LineItem.line_number, + Serie.row, + Serie.serial_numbers, + Serie.model, + Serie.sub_model, + Serie.number_id, + ) + .join(LineItem, LineItem.invoice_id == InvoiceHeader.id) + .join(Serie, Serie.line_item_id == LineItem.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.operation_type == "imp", + InvoiceHeader.invoice_type == "TEM", + ) + ) + for num, ln, rw, sn, md, sm, nid in q_ser.all(): + if num is not None: + k = (str(num).strip(), str(ln).strip(), str(rw).strip()) + existing_series_keys.add(k) + existing_series_data.setdefault(k, { + "serial_numbers": sn or "", + "model": md or "", + "sub_model": sm or "", + "number_id": nid or "", + }) + + inserted_count = 0 + updated_count = 0 + skipped_invalid = 0 + skipped_details: List[Dict[str, Any]] = [] + + with open(file_path, "r", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except Exception: + dialect = "excel" + reader = csv.DictReader(f, dialect=dialect) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + row_norm = row_from_template(row, "imp_temp_series", normalize_header) + err = validate_row_series_impo_temp( + row_norm, + i, + actualizar=actualizar, + autonumerar=autonumerar, + validar_series_exception=validar_series_exception, + invoice_id_by_number=invoice_id_by_number, + invoice_updated_by_number=invoice_updated_by_number, + existing_series_keys=existing_series_keys, + existing_series_data=existing_series_data, + warnings=None, + ) + if err: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "invoice": (row_norm.get("NUMERO FACTURA") or row_norm.get("NUM FACTURA") or "").strip(), + "reason": err.get("msg", ""), + }) + continue + + data = row_to_series_normalized(row_norm) + invoice_number = data["NUMERO FACTURA"] + linea_factura = data["LINEA FACTURA"] + linea_serie = data["LINEA SERIE"] + + if not invoice_number or invoice_number not in invoice_id_by_number: + skipped_invalid += 1 + continue + invoice_id = invoice_id_by_number[invoice_number] + line_number_val = parse_int(linea_factura) + if line_number_val is None: + skipped_invalid += 1 + skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA FACTURA debe ser numérico."}) + continue + line_item = ( + session.query(LineItem) + .filter( + LineItem.invoice_id == invoice_id, + LineItem.line_number == line_number_val, + LineItem.tenant_id == tenant_id, + LineItem.company_id == company_id, + ) + .first() + ) + if not line_item: + skipped_invalid += 1 + skipped_details.append({ + "line": i, + "invoice": invoice_number, + "reason": f"Partida línea {linea_factura} no existe en la factura.", + }) + continue + + if autonumerar: + max_row = ( + session.query(Serie.row) + .filter(Serie.line_item_id == line_item.id) + .order_by(Serie.row.desc()) + .limit(1) + .scalar() + ) + row_num = (max_row or 0) + 1 + else: + row_num = parse_int(linea_serie) + if row_num is None: + skipped_invalid += 1 + skipped_details.append({"line": i, "invoice": invoice_number, "reason": "LINEA SERIE debe ser numérico."}) + continue + + existing_serie = ( + session.query(Serie) + .filter( + Serie.line_item_id == line_item.id, + Serie.row == row_num, + ) + .first() + ) + + if existing_serie: + if actualizar: + existing_serie.serial_numbers = data["SERIE"] or existing_serie.serial_numbers + existing_serie.model = data["MODELO"] or existing_serie.model + existing_serie.sub_model = data["SUB MODELO"] or existing_serie.sub_model + existing_serie.number_id = data["NUMERO ID"] or existing_serie.number_id + session.add(existing_serie) + updated_count += 1 + else: + skipped_invalid += 1 + skipped_details.append({"line": i, "invoice": invoice_number, "reason": "Serie ya existe (use actualizar)."}) + else: + new_serie = Serie( + tenant_id=tenant_id, + company_id=company_id, + line_item_id=line_item.id, + row=row_num, + serial_numbers=data["SERIE"] or None, + model=data["MODELO"] or None, + sub_model=data["SUB MODELO"] or None, + number_id=data["NUMERO ID"] or None, + ) + session.add(new_serie) + inserted_count += 1 + + session.commit() + + common_storage.cleanup_import_job(JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path) + status = "finished" if (inserted_count + updated_count) > 0 else ("warning" if skipped_invalid else "failed") + out = { + "status": status, + "inserted": inserted_count, + "updated": updated_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_fk": 0, + "skipped_duplicate": 0, + "skipped_details": skipped_details, + } + if status == "failed": + out["error"] = "No hay registros válidos en el archivo CSV." + elif status == "warning" and skipped_invalid: + out["message"] = f"No se insertaron registros. {skipped_invalid} fueron rechazados." + return out + except Exception as e: + logger.exception("Series import commit failed: %s", e) + return {"status": "failed", "error": str(e)} + + try: + from api.v1.modules.a76.invoices.models import ( + InvoiceHeader, + InvoiceComplianceMx, + InvoiceFinancials, + InvoiceLogistics, + InvoiceSalesDetails, + OperationType, + TransportType, + WeightUnit, + ) + from api.v1.modules.a76.clients_and_providers.models import ClientProvider + from api.v1.modules.a76.customs_brokers.models import CustomsBroker + from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos + from api.v1.modules.public.reference_data.currency_types.models import CurrencyType + from api.v1.modules.public.reference_data.pedimento_codes.models import PedimentoCode + from api.v1.modules.public.reference_data.code_pedimento_regimens.models import CodePedimentoRegimen + from api.v1.modules.public.reference_data.pedimento_regimens.models import RegimenPedimento + from api.v1.modules.public.reference_data.invoice_types.models import InvoiceType + from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection + from api.v1.modules.public.reference_data.incoterms.models import Incoterm + from .validators.encabezados_impo_temp import ( + parse_pedimento_col_a, + _pedimento_key_from_parsed, + row_to_transport_type_clarion, + ) + + from api.v1.modules.a76.items.models import LineItem + from api.v1.modules.a76.items.line_financials.models import LineFinancial + from api.v1.modules.a76.items.line_quantities.models import LineQuantity + from api.v1.modules.a76.items.line_customs.models import LineCustom + from api.v1.modules.a76.items.line_descriptions.models import LineDescription + from api.v1.modules.a76.parts.models import Part + from api.v1.modules.a76.classes.models import Class + from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure + from api.v1.modules.a76.general_catalogs.packages.models import Package + + footer_config = parse_footer_config(meta.get("footer_config")) + + date_format = footer_config.get("dateFormat") + # Validate and set default date_format if not provided + if not date_format: + date_format = "yyyy-mm-dd" # Default to ISO format + logger.info(f"No date_format specified in config, using default: {date_format}") + else: + logger.info(f"Using date_format from config: {date_format}") + + # Default types from config or fallback + op_type_value = OperationType(meta.get('operation_type', 'imp').lower()) + inv_type_value = normalize_public_code(footer_config.get('invoice_type') or 'TEM') or 'TEM' + + logger.info(f"Processing CSV with operation_type={op_type_value}, invoice_type={inv_type_value}, date_format={date_format}") + + headers_to_insert = [] + details_to_insert = [] + skipped_invalid = 0 + skipped_missing_invoice = 0 + skipped_missing_fk = 0 + skipped_fk_details = [] + inserted_count = 0 + response = None + + with CoreSessionLocal() as session: + invoice_id_cache = {} + cleared_invoices = set() # Track invoices where we've already cleared items in this job + provider_cache: Dict[Any, Optional[int]] = {} + sold_to_cache: Dict[Any, Optional[int]] = {} + shipped_to_cache: Dict[Any, Optional[int]] = {} + broker_cache: Dict[Any, Optional[int]] = {} + regimen_cache: Dict[str, Optional[str]] = {} + currency_type_cache: Dict[str, Optional[str]] = {} + customs_section_cache: Dict[str, Optional[str]] = {} + part_cache: Dict[str, Optional[int]] = {} + pedimento_id_cache: Dict[str, Optional[int]] = {} + _fc_insert = parse_footer_config(meta.get("footer_config")) + autonumerar_remesas_insert = _fc_insert.get("autonumerar_remesas", False) + class_id_by_code: Dict[str, int] = {} + uom_id_by_code: Dict[str, int] = {} + package_id_by_key: Dict[str, int] = {} + if model_target == 'invoice_details': + for c in session.query(Class.id, Class.class_code).filter(Class.tenant_id == tenant_id, Class.company_id == company_id).all(): + if c[1]: + class_id_by_code[(c[1] or "").strip().upper()] = c[0] + for u in session.query(UnitOfMeasure.id, UnitOfMeasure.code).filter(UnitOfMeasure.tenant_id == tenant_id, UnitOfMeasure.company_id == company_id).all(): + if u[1]: + uom_id_by_code[(u[1] or "").strip().upper()] = u[0] + for p in session.query(Package.id, Package.key).filter(Package.tenant_id == tenant_id, Package.company_id == company_id).all(): + if p[1]: + package_id_by_key[(p[1] or "").strip()] = p[0] + + validator = ForeignKeyValidator(session, tenant_id, company_id) + + with open(file_path, 'r', encoding='utf-8-sig') as f: + # Detect Delimiter + sample = f.read(2048) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t") + except: + dialect = 'excel' + + reader = csv.DictReader(f, dialect=dialect) + + template_id = meta.get("template_id") or ( + "imp_temp_header" if model_target == "invoice_header" else "imp_temp_details" + ) + + for i, row in enumerate(reader, start=1): + if i in error_lines: + continue + + row_norm = row_from_template(row, template_id, normalize_header) + + # Mapping Logic (solo campos que acepta el modelo de facturas) + if model_target == 'invoice_header': + invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or row_norm.get('FACTURA') or '').strip() + invoice_date = parse_date(row_norm.get('FECHA FACTURA') or row_norm.get('FECHA'), date_format) + + if not invoice_number or not invoice_date: + skipped_invalid += 1 + logger.debug(f"Row {i}: Skipped - missing invoice_number or invalid invoice_date. " + f"Invoice: {invoice_number}, Date: {row_norm.get('FECHA FACTURA') or row_norm.get('FECHA')}") + continue + + # --- NEW: Foreign Key Validations --- + # 1. Invoice Type (Public) + if not validator.check_exists(InvoiceType, inv_type_value, field_name="key", is_public=True): + skipped_missing_fk += 1 + reason = f"Tipo de factura '{inv_type_value}' no existe" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = _validate_client_provider_ref( + validator, + ClientProvider, + row_norm.get('CLAVE PROVEEDOR'), + i, + "CLAVE PROVEEDOR", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = _validate_client_provider_ref( + validator, + ClientProvider, + row_norm.get('CLAVE VENDIDO A'), + i, + "CLAVE VENDIDO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = _validate_client_provider_ref( + validator, + ClientProvider, + row_norm.get('CLAVE ENVIADO A'), + i, + "CLAVE ENVIADO A", + required=True, + ) + if err: + skipped_invalid += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = _validate_customs_broker_ref( + validator, + CustomsBroker, + row_norm.get('AGENTE ADUANAL'), + i, + "AGENTE ADUANAL", + required=False, + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + RegimenPedimento, + row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO'), + i, + "CLAVEDOCUMENTO", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CustomsSection, + row_norm.get('ADUANA DE CRUCE'), + i, + "ADUANA DE CRUCE", + field_name="customs_code", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + CurrencyType, + row_norm.get('CLAVE MONEDA'), + i, + "CLAVE MONEDA", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + err = validate_public_code( + validator, + Incoterm, + row_norm.get('CLAVE INCOTERM'), + i, + "CLAVE INCOTERM", + ) + if err: + skipped_missing_fk += 1 + reason = f"{err['col']}: {err['msg']}" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + transport_type_val = row_norm.get('TIPO TRANSPORTE') + transport_str_normalized = (row_to_transport_type_clarion(transport_type_val) or str(transport_type_val or "").strip().lower() or "none") + if transport_type_val and transport_str_normalized not in TRANSPORT_TYPE_VALUES: + skipped_invalid += 1 + reason = "TIPO TRANSPORTE: Tipo de transporte invalido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + currency_val = row_norm.get('TIPO MONEDA') + if currency_val and str(currency_val).strip(): + parsed_currency = parse_currency(currency_val, None) + val_norm = normalize_header(currency_val) + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + skipped_invalid += 1 + reason = "TIPO MONEDA: Moneda invalida" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + # 2. Client/Provider and broker checks are handled above + + # --- 4. Check for Existing Invoice (Upsert Logic) --- + existing_header = None + if invoice_number: + existing_header = ( + session.query(InvoiceHeader) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value + ) + .first() + ) + + # --- Resolve PEDIMENTO (Col A) to pedimento_id and REMESA (Col B) --- + pedimento_id = None + remesa_val = parse_int(row_norm.get('REMESA')) + ped_str = (row_norm.get('PEDIMENTO') or '').strip() + if ped_str: + parsed = parse_pedimento_col_a(ped_str) + if parsed: + customs_office_p, license_p, num_p = parsed + key_p = _pedimento_key_from_parsed(customs_office_p, license_p, num_p) + if key_p not in pedimento_id_cache: + ped_row = ( + session.query(Pedimentos.id) + .filter( + Pedimentos.tenant_id == tenant_id, + Pedimentos.company_id == company_id, + Pedimentos.customs_office == customs_office_p, + Pedimentos.license == license_p, + Pedimentos.pedimento_number == num_p, + ) + .first() + ) + pedimento_id_cache[key_p] = ped_row[0] if ped_row else None + pedimento_id = pedimento_id_cache[key_p] + if pedimento_id is not None and remesa_val is None and autonumerar_remesas_insert: + max_rem = ( + session.query(func.max(InvoiceComplianceMx.remesa)) + .filter(InvoiceComplianceMx.pedimento_id == pedimento_id) + .scalar() + ) + remesa_val = (max_rem or 0) + 1 + + if existing_header: + # UPDATE existing header + header = existing_header + header.invoice_date = invoice_date + header.operation_type = op_type_value + header.is_updated = True # Mark as updated + header.updated_date = datetime.utcnow() + header.document_type = resolve_public_code( + session, + RegimenPedimento, + RegimenPedimento.code, + (row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')), + regimen_cache, + ) + header.project_number = (row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None) + header.purchase_order = (row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None) + header.alternate_invoice = (row_norm.get('FACTURA ALTERNA') or None) + header.invoice_ref = (row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None) + header.emission_date = parse_date(row_norm.get('FECHA EMISION'), date_format) + header.observation_es = (row_norm.get('OBSERVACIONES E') or None) + header.observation_en = (row_norm.get('OBSERVACIONES I') or None) + + logger.info(f"Row {i}: Updating existing invoice {invoice_number}") + + # Clean up related data that will be re-inserted/updated + # Note: compliance, financials, logistics are 1-to-1 relationships and will be updated by assignment below + # but we might want to be explicit if ORM doesn't handle replace well. + # SQLAlchemy relationship assignment usually handles 1-to-1 updates correctly. + + else: + # CREATE new header + header = InvoiceHeader( + invoice_number=invoice_number, + invoice_date=invoice_date, + operation_type=op_type_value, + is_updated=False, + system="CSV", + capture_date=datetime.utcnow(), + invoice_type=inv_type_value, + document_type=resolve_public_code( + session, + RegimenPedimento, + RegimenPedimento.code, + (row_norm.get('REGIMEN') or row_norm.get('CLAVEDOCUMENTO')), + regimen_cache, + ), + project_number=(row_norm.get('NUM PROYECTO') or row_norm.get('NUMPROYECTO') or None), + purchase_order=(row_norm.get('ORDEN COMPRA') or row_norm.get('ORDENCOMPRA') or None), + alternate_invoice=(row_norm.get('FACTURA ALTERNA') or None), + invoice_ref=(row_norm.get('FACTURA EXPO REF') or row_norm.get('FACTURAEXPOREF') or None), + emission_date=parse_date(row_norm.get('FECHA EMISION'), date_format), + observation_es=(row_norm.get('OBSERVACIONES E') or None), + observation_en=(row_norm.get('OBSERVACIONES I') or None), + tenant_id=tenant_id, + company_id=company_id, + ) + + compliance = InvoiceComplianceMx( + pedimento_id=pedimento_id, + remesa=remesa_val, + aduana=resolve_public_code( + session, + CustomsSection, + CustomsSection.customs_code, + row_norm.get('ADUANA DE CRUCE'), + customs_section_cache, + ), + provider_id=resolve_client_provider_id( + session, + ClientProvider, + row_norm.get('CLAVE PROVEEDOR'), + tenant_id, + company_id, + provider_cache, + ), + sold_to_id=resolve_client_provider_id( + session, + ClientProvider, + row_norm.get('CLAVE VENDIDO A'), + tenant_id, + company_id, + sold_to_cache, + ), + shipped_to_id=resolve_client_provider_id( + session, + ClientProvider, + row_norm.get('CLAVE ENVIADO A'), + tenant_id, + company_id, + shipped_to_cache, + ), + customs_broker_id=resolve_customs_broker_id( + session, + CustomsBroker, + row_norm.get('AGENTE ADUANAL'), + tenant_id, + company_id, + broker_cache, + ), + edocument=(row_norm.get('E DOCUMENT') or None), + vucem_operation_num=(row_norm.get('NUM OPERACION') or None), + tenant_id=tenant_id, + company_id=company_id, + ) + + financials_currency_type = resolve_public_code( + session, + CurrencyType, + CurrencyType.code, + row_norm.get('CLAVE MONEDA'), + currency_type_cache, + ) + financials = InvoiceFinancials( + currency=parse_currency(row_norm.get('TIPO MONEDA'), financials_currency_type), + currency_type=financials_currency_type, + exchange_rate=decimal_or_zero(row_norm.get('TIPO DE CAMBIO')), + freight=decimal_or_zero(row_norm.get('FLETES')), + insurance_value=decimal_or_zero(row_norm.get('VALOR SEGUROS')), + insurance=decimal_or_zero(row_norm.get('SEGUROS')), + packaging=decimal_or_zero(row_norm.get('EMBALAJES')), + other_increments=decimal_or_zero(row_norm.get('OTROS INCREMENTABLES')), + tenant_id=tenant_id, + company_id=company_id, + ) + + weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) + logistics = None + if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): + raw_transport = row_norm.get('TIPO TRANSPORTE') + transport_str = (row_to_transport_type_clarion(raw_transport) or str(raw_transport or "").strip().lower() or "none") + try: + transport_type = TransportType(transport_str) + except ValueError: + transport_type = TransportType.NONE + logistics = InvoiceLogistics( + carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), + driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), + transport_type=transport_type, + transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), + weight_type=weight_type or WeightUnit.KGS, + seal_number=(row_norm.get('PRECINTO') or None), + incoterm=(row_norm.get('CLAVE INCOTERM') or None), + entry_exit_date=parse_date(row_norm.get('FECHA EMISION'), date_format), + tenant_id=tenant_id, + company_id=company_id, + ) + + header.compliance_mx = compliance + header.financials = financials + if logistics: + header.logistics = logistics + + headers_to_insert.append(header) + + elif model_target == 'invoice_details': + invoice_number = (row_norm.get('NUMERO FACTURA') or row_norm.get('NUM FACTURA') or '').strip() + if not invoice_number: + skipped_invalid += 1 + continue + + cache_key = f"{invoice_number}|{inv_type_value}" + if cache_key in invoice_id_cache: + invoice_id = invoice_id_cache[cache_key] + else: + invoice_id = ( + session.query(InvoiceHeader.id) + .filter( + InvoiceHeader.tenant_id == tenant_id, + InvoiceHeader.company_id == company_id, + InvoiceHeader.invoice_number == invoice_number, + InvoiceHeader.invoice_type == inv_type_value, + ) + .scalar() + ) + invoice_id_cache[cache_key] = invoice_id + + if not invoice_id: + logger.warning( + "Invoice not found for details row %s (invoice_number=%s)", + i, + invoice_number, + ) + skipped_missing_invoice += 1 + continue + + part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() + if not part_num: + skipped_invalid += 1 + reason = "NUMPARTE: Requerido" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + if not validator.check_exists(Part, part_num, field_name="part_number"): + skipped_missing_fk += 1 + reason = f"NUMPARTE '{part_num}' no existe" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue + + # --- Prevent Duplicates: Clear existing line items for this invoice (Once per job) --- + if invoice_id not in cleared_invoices: + logger.info(f"Clearing existing details for Invoice {invoice_number} (ID: {invoice_id}) to prevent duplicates") + session.query(LineItem).filter(LineItem.invoice_id == invoice_id).delete(synchronize_session=False) + session.query(InvoiceSalesDetails).filter(InvoiceSalesDetails.invoice_id == invoice_id).delete(synchronize_session=False) + cleared_invoices.add(invoice_id) + + # --- Partidas: LineItem with invoice_id (no Item parent) + full CSV mapping --- + part_num = (row_norm.get('NUM. PARTE') or row_norm.get('NUMPARTE') or row_norm.get('NUMERO PARTE') or row_norm.get('NUM PARTE') or '').strip() + part_id = part_cache.get(part_num) if part_num else None + if part_id is None and part_num: + p = session.query(Part.id).filter(Part.part_number == part_num, Part.tenant_id == tenant_id, Part.company_id == company_id).first() + if p: + part_id = p.id + part_cache[part_num] = part_id + + line_num_val = (row_norm.get('LINEA') or row_norm.get('RENGLON') or row_norm.get('PARTIDA')) + line_num = parse_int(line_num_val) or (len(details_to_insert) + 1) + + class_code = (row_norm.get('CLASE') or '').strip().upper() + class_id = class_id_by_code.get(class_code) if class_code else None + uom_code = (row_norm.get('UNIDAD DE MEDIDA') or row_norm.get('UNIDAD MEDIDA') or '').strip().upper() + uom_id = uom_id_by_code.get(uom_code) if uom_code else None + bulk_key = (row_norm.get('CLAVE BULTOS') or row_norm.get('CLAVEBULTOS') or '').strip() + package_id = package_id_by_key.get(bulk_key) if bulk_key else None + + line = LineItem( + invoice_id=invoice_id, + line_number=line_num, + tenant_id=tenant_id, + company_id=company_id, + part_number_id=part_id, + class_id=class_id, + unit_of_measure=uom_id, + order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), + material_type=(row_norm.get('ID TYPE') or row_norm.get('IDTYPE') or None), + tax_payment=(str(row_norm.get('SE PAGO IMPUESTO') or row_norm.get('SEPAGOIMPUESTO') or '').strip().upper() == 'SI'), + payment_method=(row_norm.get('FORMA DE PAGO') or row_norm.get('FORMADEPAGO') or row_norm.get('FORMA PAGO') or None), + valuation_method=(row_norm.get('METODO DE VALORACION') or row_norm.get('METODODEVALORACION') or row_norm.get('METODO VALORACION') or None), + ) + session.add(line) + session.flush() + + price = parse_decimal(row_norm.get('COSTO UNITARIO') or row_norm.get('PRECIO UNITARIO') or row_norm.get('PRECIOUNITARIO')) + if price is None: + total_val = parse_decimal(row_norm.get('TOTAL')) + qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD')) + price = (total_val / qty) if (total_val and qty and qty != 0) else None + qty = parse_decimal(row_norm.get('CANTIDAD IMPORTADA') or row_norm.get('CANTIDAD')) + commercial_total = (price * qty) if price and qty else parse_decimal(row_norm.get('TOTAL')) + + session.add(LineFinancial( + item_line_id=line.id, + unit_cost_capture=decimal_or_zero(price), + total_commercial_value=decimal_or_zero(commercial_total), + )) + + net_w = parse_decimal(row_norm.get('PESO NETO') or row_norm.get('PESONETO')) + gross_w = parse_decimal(row_norm.get('PESO BRUTO') or row_norm.get('PESOBRUTO')) + session.add(LineQuantity( + item_line_id=line.id, + quantity=decimal_or_zero(qty), + net_weight=decimal_or_zero(net_w), + gross_weight=decimal_or_zero(gross_w), + package_quantity=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), + package_id=package_id, + )) + + origin = (row_norm.get('PAIS ORIGEN') or row_norm.get('PAISORIGEN') or row_norm.get('PAIS') or '').strip() + fraction = (row_norm.get('FRACCION ARANCELARIA') or row_norm.get('FRACCION') or row_norm.get('FRACCIONARANCELARIA') or '').strip() + fraction_type = (row_norm.get('PREFERENCIA ARANCELARIA') or row_norm.get('PREFERENCIA') or '').strip() + sector = (row_norm.get('SECTOR') or '').strip() + american_fraction = (row_norm.get('FRACCION AMERICANA') or row_norm.get('FRACCIONAMERICANA') or '').strip() + session.add(LineCustom( + item_line_id=line.id, + origin_country=origin or None, + fraction=fraction or None, + fraction_type=fraction_type or None, + sector=sector or None, + american_fraction=american_fraction or None, + )) + + desc_es = (row_norm.get('DESCRIPCION ESPAÑOL') or row_norm.get('DESCRIPCIONE') or row_norm.get('DESCRIPCION') or '').strip() + desc_en = (row_norm.get('DESCRIPCION INGLES') or row_norm.get('DESCRIPCIONI') or '').strip() + brand = (row_norm.get('MARCA') or '').strip() + model = (row_norm.get('MODELO') or '').strip() + extra_desc = (row_norm.get('DESCRIPCION EXTRA') or row_norm.get('DESCRIPCIONEXTRA') or '').strip() + additional_info = (row_norm.get('INFORMACION ADICIONAL') or row_norm.get('INFORMACIONADICIONAL') or '').strip() + lot = (row_norm.get('LOTE') or '').strip() + entry_number = (row_norm.get('NUMERO ENTRADA') or row_norm.get('NUMEROENTRADA') or row_norm.get('NUM ENTRADA') or '').strip() + session.add(LineDescription( + item_line_id=line.id, + description_spanish=desc_es or None, + description_english=desc_en or None, + brand=brand or None, + model=model or None, + extra_description=extra_desc or None, + additional_info_spanish=additional_info or None, + lot=lot or None, + entry_number=entry_number or None, + )) + + session.add(InvoiceSalesDetails( + invoice_id=invoice_id, + line_number=line_num, + sales_order=(row_norm.get('ORDEN DE COMPRA') or row_norm.get('ORDENCOMPRA') or None), + line_bundles=int_or_zero(row_norm.get('CANTIDAD BULTOS') or row_norm.get('CANTIDADBULTOS')), + tenant_id=tenant_id, + company_id=company_id, + )) + details_to_insert.append(line) + + # 3. Bulk Insert (ORM Transaction) + try: + if model_target == 'invoice_header': + if headers_to_insert: + logger.info(f"Attempting to commit {len(headers_to_insert)} headers") + session.add_all(headers_to_insert) + session.commit() + inserted_count = len(headers_to_insert) + logger.info(f"Headers commit successful. Inserted: {inserted_count}") + else: + logger.warning(f"No headers to insert for job {job_id}") + else: + if details_to_insert: + logger.info(f"Attempting to commit {len(details_to_insert)} items and related data") + session.commit() # Everything was already added with session.add() + inserted_count = len(details_to_insert) + logger.info(f"Details commit successful. Inserted: {inserted_count}") + else: + logger.warning(f"No details to insert for job {job_id}") + + except Exception as db_err: + session.rollback() + logger.error(f"DB Error during {model_target} commit: {db_err}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(db_err)} + + # 4. Determine final status and prepare response (inside session block to access variables) + total_skipped = skipped_invalid + skipped_missing_fk + skipped_missing_invoice + + # Log summary + logger.info(f"Job {job_id} completed. Inserted: {inserted_count}, Skipped: {total_skipped} " + f"(invalid: {skipped_invalid}, missing_fk: {skipped_missing_fk}, missing_invoice: {skipped_missing_invoice})") + + # Prepare response based on results + if inserted_count == 0: + if total_skipped > 0: + logger.warning(f"No valid records to insert for job {job_id}. All {total_skipped} records were rejected.") + response = { + "status": "warning", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_invoice": skipped_missing_invoice, + "skipped_missing_fk": skipped_missing_fk, + "skipped_details": skipped_fk_details, + "message": f"No se insertaron registros. {total_skipped} fueron rechazados." + } + else: + logger.error(f"No valid records found in CSV for job {job_id}") + response = { + "status": "failed", + "error": "No hay registros válidos en el archivo CSV", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_invoice": skipped_missing_invoice, + "skipped_missing_fk": skipped_missing_fk, + "skipped_details": skipped_fk_details + } + else: + # Success case - at least some records were inserted + response = { + "status": "finished", + "inserted": inserted_count, + "skipped_invalid": skipped_invalid, + "skipped_missing_invoice": skipped_missing_invoice, + "skipped_missing_fk": skipped_missing_fk, + "skipped_details": skipped_fk_details + } + + except Exception as e: + logger.error(f"Task failed: {e}") + import traceback + logger.error(traceback.format_exc()) + return {"status": "failed", "error": str(e)} + + # 5. Cleanup: remove temp files and Redis keys so data is not kept indefinitely + try: + common_storage.cleanup_import_job( + JOB_TYPE, job_id, + file_path=file_path, + error_path=error_path, + meta_path=meta_path, + ) + except Exception as cleanup_err: + logger.warning("Failed to cleanup temp files or Redis: %s", cleanup_err) + + # Ensure response is defined (fallback in case of unexpected errors) + if response is None: + logger.error(f"Unexpected error: response not set for job {job_id}") + response = { + "status": "failed", + "error": "Error inesperado durante el procesamiento", + "inserted": 0, + "skipped_invalid": skipped_invalid, + "skipped_missing_invoice": skipped_missing_invoice, + "skipped_missing_fk": skipped_missing_fk, + "skipped_details": skipped_fk_details + } + + return response diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py b/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py index e6eb4a6a..265f5e56 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py @@ -12,41 +12,42 @@ from typing import Dict, List, Any, Optional # aliases = cabeceras alternativas que la plantilla .xls puede traer (ej. "Num Factura" → NUM FACTURA). TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { - # --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) --- + # --- Encabezado factura: Impo Temp (EstructuraEncFacImpoTemp.xls) - Clarion A-AD --- "imp_temp_header": [ + {"canonical": "PEDIMENTO", "aliases": ["NUMERO PEDIMENTO", "PEDIMENTO NUMERO", "NUMERO DE PEDIMENTO", "PED"]}, + {"canonical": "REMESA"}, {"canonical": "NUMERO FACTURA", "aliases": ["NUM FACTURA", "FACTURA", "ID"]}, {"canonical": "FECHA FACTURA", "aliases": ["FECHA"]}, - {"canonical": "FECHA EMISION"}, + {"canonical": "TIPO DE CAMBIO"}, + {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, {"canonical": "CLAVE PROVEEDOR"}, {"canonical": "CLAVE VENDIDO A"}, {"canonical": "CLAVE ENVIADO A"}, - {"canonical": "REGIMEN", "aliases": ["CLAVEDOCUMENTO"]}, - {"canonical": "ADUANA DE CRUCE"}, - {"canonical": "CLAVE MONEDA"}, - {"canonical": "CLAVE INCOTERM"}, - {"canonical": "TIPO MONEDA"}, - {"canonical": "TIPO DE CAMBIO"}, - {"canonical": "TIPO PESO"}, - {"canonical": "TIPO TRANSPORTE"}, - {"canonical": "REMESA"}, {"canonical": "AGENTE ADUANAL"}, + {"canonical": "CLAVE TRANSPORTISTA"}, + {"canonical": "NOMBRE CONDUCTOR"}, + {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "NUMERO TRANSPORTE"}, + {"canonical": "TIPO MONEDA"}, + {"canonical": "CLAVE MONEDA"}, {"canonical": "FLETES"}, {"canonical": "VALOR SEGUROS"}, {"canonical": "SEGUROS"}, {"canonical": "EMBALAJES"}, {"canonical": "OTROS INCREMENTABLES"}, - {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, - {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, - {"canonical": "FACTURA ALTERNA"}, - {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, + {"canonical": "CLAVE INCOTERM"}, + {"canonical": "PRECINTO"}, + {"canonical": "FECHA EMISION"}, + {"canonical": "TIPO PESO"}, + {"canonical": "E DOCUMENT", "aliases": ["E-DOCUMENT", "EDOCUMENT", "E DOCUMENT"]}, + {"canonical": "NUM OPERACION", "aliases": ["NUM. OPERACION", "NUMOPERACION", "NUM OPERACION"]}, + {"canonical": "ADUANA DE CRUCE"}, {"canonical": "OBSERVACIONES E"}, {"canonical": "OBSERVACIONES I"}, - {"canonical": "E DOCUMENT"}, - {"canonical": "NUM OPERACION"}, - {"canonical": "CLAVE TRANSPORTISTA"}, - {"canonical": "NOMBRE CONDUCTOR"}, - {"canonical": "NUMERO TRANSPORTE"}, - {"canonical": "PRECINTO"}, + {"canonical": "FACTURA ALTERNA"}, + {"canonical": "NUM PROYECTO", "aliases": ["NUMPROYECTO"]}, + {"canonical": "ORDEN COMPRA", "aliases": ["ORDENCOMPRA"]}, + {"canonical": "FACTURA EXPO REF", "aliases": ["FACTURAEXPOREF"]}, ], # --- Encabezado factura: Impo Def (EstructuraEncFacImpoDef.xls) - misma estructura --- "imp_def_header": None, # se resuelve igual que imp_temp_header diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/__init__.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/__init__.py index 83516424..ea14dcaf 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/__init__.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/__init__.py @@ -1 +1,14 @@ # Validators for invoice CSV imports (header, details, series). +from .encabezados_impo_temp import ( + validate_row_encabezados_impo_temp, + row_to_transport_type_clarion, + parse_pedimento_col_a, + _pedimento_key_from_parsed, +) + +__all__ = [ + "validate_row_encabezados_impo_temp", + "row_to_transport_type_clarion", + "parse_pedimento_col_a", + "_pedimento_key_from_parsed", +] diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py new file mode 100644 index 00000000..e4196ff3 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py @@ -0,0 +1,650 @@ +""" +Validaciones CSV para Encabezados de Facturas de Importación Temporal. +Paridad Clarion: VALIDA_TODA_FACIMPO_TEM, VALIDA_PARCIAL_FACIMPO_TEM, VALIDACIONES_FACIMPO_TEM. +Mapeo a BD en commit: LLENA_FACIMPO_TEM (tasks.insert_valid_rows). +Estructura CSV: PEDIMENTO (A), REMESA (B), NUMERO FACTURA (C), ... ADUANA DE CRUCE (AB), OBSERVACIONES E/I, FACTURA ALTERNA. +""" +from datetime import datetime +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, List, Optional, Set, Tuple + +# Longitudes máximas Clarion +MAX_LEN_PEDIMENTO = 18 # CC-LLLL-NNNNNNN o CCC-LLLL-NNNNNNN (sin año; ej. 01-1234-2312412 o 640-1234-2312412) +MAX_LEN_FACTURA = 15 + +# Formato PEDIMENTO: CC-LLLL-NNNNNNN (sin año: aduana 2-3, patente 4, número 7) + +REGIMENES_VALIDOS = frozenset({"ITE", "ITR"}) +TIPOS_MONEDA_VALIDOS = frozenset({"ME", "MN", "MC"}) +TIPO_PESO_VALIDOS = frozenset({"KILOS", "LIBRAS"}) + +# Clarion Col M → valor normalizado (minúscula para TransportType enum) +TIPO_TRANSPORTE_CLARION_TO_NORM = { + "NINGUNO": "none", + "TRANSPORTE": "transport", + "CAJA": "box", + "PLACAS": "licence plates", + "CAMION": "truck", + "BUQUE": "vessel", + "FERROBARCAZA": "rail_barge", + "CONTENEDOR": "container", + "PLATAFORMA": "flatbed", + "GONDOLA": "gondola", + "AVION": "airplane", +} +TIPO_TRANSPORTE_VALIDOS = frozenset(TIPO_TRANSPORTE_CLARION_TO_NORM.keys()) + + +def _clip(val: Any) -> str: + if val is None: + return "" + return str(val).strip() + + +def _get(row: Dict[str, Any], *keys: str) -> str: + for k in keys: + v = row.get(k) + if v is not None and str(v).strip(): + return _clip(v) + return "" + + +def _parse_decimal(val: Any) -> Optional[Decimal]: + if val is None: + return None + s = _clip(val) + if not s: + return None + s = s.replace(",", "") + try: + return Decimal(s) + except (InvalidOperation, ValueError): + return None + + +def _parse_int(val: Any) -> Optional[int]: + if val is None: + return None + s = _clip(val) + if not s: + return None + try: + return int(s) + except ValueError: + return None + + +def parse_pedimento_col_a(pedimento_str: str) -> Optional[Tuple[str, str, str]]: + """ + Parsea Col A (PEDIMENTO) formato CC-LLLL-NNNNNNN o CCC-LLLL-NNNNNNN (18 caracteres, sin año). + Retorna (customs_office_2o3, license_4, pedimento_number_7) o None si formato inválido. + """ + if not pedimento_str or not isinstance(pedimento_str, str): + return None + s = (pedimento_str or "").strip() + parts = s.split("-") + if len(parts) != 3: + return None + customs_office, license_val, pedimento_number = parts[0], parts[1], parts[2] + if len(customs_office) not in (2, 3) or not customs_office.isdigit(): + return None + if len(license_val) != 4 or not license_val.isdigit(): + return None + if len(pedimento_number) != 7 or not pedimento_number.isdigit(): + return None + return (customs_office, license_val, pedimento_number) + + +def _pedimento_key_from_parsed(customs_office: str, license_val: str, pedimento_number: str) -> str: + """Clave para lookup: CC-LLLL-NNNNNNN (sin año; como viene en CSV).""" + return f"{customs_office}-{license_val}-{pedimento_number}" + + +def _err(line_num: int, col: str, msg: str) -> Dict[str, Any]: + return {"line": line_num, "col": col, "msg": msg} + + +# --- Obligatorios VALIDA_TODA (cuando no es actualizar) --- +def _validaciones_obligatorios_toda( + row: Dict[str, Any], + line_num: int, + tiene_pedimento: bool, +) -> Optional[Dict[str, Any]]: + """Obligatorios: C, D, F, G, H, I, J; AB obligatoria si A tiene valor.""" + obligatorios: List[str] = [] + if not _get(row, "NUMERO FACTURA", "NUM FACTURA", "FACTURA"): + obligatorios.append("(Col.C) Número de Factura") + if not _get(row, "FECHA FACTURA", "FECHA"): + obligatorios.append("(Col.D) Fecha de la Factura") + if not _get(row, "REGIMEN", "CLAVEDOCUMENTO"): + obligatorios.append("(Col.F) Clave Régimen") + if not _get(row, "CLAVE PROVEEDOR"): + obligatorios.append("(Col.G) Clave del Proveedor") + if not _get(row, "CLAVE VENDIDO A"): + obligatorios.append("(Col.H) Clave del Vendido A") + if not _get(row, "CLAVE ENVIADO A"): + obligatorios.append("(Col.I) Clave del Enviado A") + if not _get(row, "AGENTE ADUANAL"): + obligatorios.append("(Col.J) Clave del Agente Aduanal") + if tiene_pedimento and not _get(row, "ADUANA DE CRUCE"): + obligatorios.append("(Col.AB) Aduana de Cruce") + if obligatorios: + return _err( + line_num, + "ARCHIVO CSV", + f"Existen campos vacíos que son obligatorios: {', '.join(obligatorios)}.", + ) + return None + + +# --- Pedimento y Remesa (Col A, B) --- +def _validaciones_pedimento_remesa( + row: Dict[str, Any], + line_num: int, + pedimento_data_by_key: Dict[str, List[Dict[str, Any]]], + remesa_por_pedimento_bd: Dict[str, Set[int]], + remesa_por_pedimento_csv: Dict[str, Dict[int, str]], + autonumerar_remesas: bool, + control_remesa: bool, + remesa_inicio: Optional[int], + remesa_fin: Optional[int], + date_format: Optional[str], + invoice_date_parsed: Optional[datetime], +) -> Optional[Dict[str, Any]]: + """Col A: formato, longitud, catálogo, tipo I, régimen ITE/ITR, match Col F, fechas. Col B: oblig si A, no 0, rango, unicidad.""" + col_a = _get(row, "PEDIMENTO") + col_b_raw = row.get("REMESA") + col_b = _clip(col_b_raw) + col_f = _get(row, "REGIMEN", "CLAVEDOCUMENTO").upper() + + if not col_a: + if col_b: + return _err( + line_num, + "REMESA", + f"Error: (Celda B{line_num}) Está asignado el número de Remesa y no se tiene un pedimento en (Celda A{line_num}).", + ) + return None + + if len(col_a) > MAX_LEN_PEDIMENTO: + return _err( + line_num, + "PEDIMENTO", + f"Error: (Celda A{line_num}) El Pedimento: {col_a} supera la longitud de caracteres.", + ) + parsed = parse_pedimento_col_a(col_a) + if not parsed: + return _err( + line_num, + "PEDIMENTO", + f"Error: (Celda A{line_num}) El Formato del Pedimento: {col_a} es incorrecto. Use CC-LLLL-NNNNNNN (ej. 01-1234-2312412, 18 caracteres sin año).", + ) + + customs_office, license_val, pedimento_number = parsed + key = _pedimento_key_from_parsed(customs_office, license_val, pedimento_number) + ped_info_list = pedimento_data_by_key.get(key) + if not ped_info_list: + return _err( + line_num, + "PEDIMENTO", + f"Error: (Celda A{line_num}) El Número de Pedimento: {col_a} no existe en el Catálogo de Pedimentos. " + f"Verifique que esté dado de alta (formato CC-LLLL-NNNNNNN: aduana 2-3, patente 4, número 7, sin año) para esta empresa.", + ) + + ped_info = ped_info_list[0] + if (ped_info.get("operation_type") or "").upper() != "IMP": + return _err( + line_num, + "PEDIMENTO", + f"Error: (Celda A{line_num}) Este Número de Pedimento: {col_a} no está marcado como Importación Temporal.", + ) + regimen_ped = (ped_info.get("regime") or "").strip().upper() + if regimen_ped not in REGIMENES_VALIDOS: + return _err( + line_num, + "PEDIMENTO", + f"Error: (Celda A{line_num}) El Pedimento: {col_a} tiene el Régimen {regimen_ped}, no válido (ITE o ITR).", + ) + if col_f and col_f not in REGIMENES_VALIDOS: + pass + elif col_f and col_f != regimen_ped: + return _err( + line_num, + "REGIMEN", + f"Error: (Celda F{line_num}) El Régimen Aduanero: {col_f} no coincide con el del Pedimento: {regimen_ped}.", + ) + + pedimento_type = (ped_info.get("pedimento_type") or "").strip().lower() + if pedimento_type == "consolidated" and invoice_date_parsed and ped_info.get("entry_date") and ped_info.get("end_date"): + entry = ped_info["entry_date"] + end = ped_info["end_date"] + if hasattr(entry, "date"): + entry = entry.date() + if hasattr(end, "date"): + end = end.date() + inv_d = invoice_date_parsed.date() if hasattr(invoice_date_parsed, "date") else invoice_date_parsed + if inv_d < entry or inv_d > end: + return _err( + line_num, + "FECHA FACTURA", + f"Error: (Celda D{line_num} y A{line_num}) La Fecha de la Factura no corresponde al rango de fechas del Pedimento {col_a}.", + ) + + if not autonumerar_remesas and not col_b: + return _err( + line_num, + "REMESA", + f"Error: (Celda B{line_num}) El Número de Remesa está vacío y se tiene un Pedimento en la Celda A{line_num}.", + ) + remesa_int = _parse_int(col_b_raw) + if col_b and remesa_int is not None and remesa_int == 0: + return _err( + line_num, + "REMESA", + f"Error: (Celda B{line_num}) El Número de Remesa no puede ser 0.", + ) + if control_remesa and remesa_int is not None and remesa_inicio is not None and remesa_fin is not None: + if remesa_int < remesa_inicio or remesa_int > remesa_fin: + return _err( + line_num, + "REMESA", + f"Error: (Celda B{line_num}) El Número de Remesa: {col_b} está fuera del rango configurado ({remesa_inicio}-{remesa_fin}).", + ) + + factura_actual = _get(row, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") + if remesa_int is not None and key in remesa_por_pedimento_csv: + other = remesa_por_pedimento_csv[key].get(remesa_int) + if other and other != factura_actual: + return _err( + line_num, + "REMESA", + f"Error: (Celda B{line_num}) El Número de Remesa ya está asignado a la factura {other} en este archivo CSV.", + ) + return None + + +# --- Factura longitud (Col C) --- +def _validaciones_factura_longitud(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + c = _get(row, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") + if c and len(c) > MAX_LEN_FACTURA: + return _err(line_num, "NUMERO FACTURA", f"Error: (Celda C{line_num}) El Número de Factura supera la longitud de caracteres.") + return None + + +# --- Régimen (Col F) --- +def _validaciones_regimen(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + f = _get(row, "REGIMEN", "CLAVEDOCUMENTO") + if f and f.upper() not in REGIMENES_VALIDOS: + return _err( + line_num, + "REGIMEN", + f"Error: (Celda F{line_num}) El Régimen Aduanero: {f} no es válido. Use ITE o ITR.", + ) + return None + + +# --- Tipo transporte y número (Col M, N) --- +def _validaciones_transporte(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + m = _get(row, "TIPO TRANSPORTE").upper() + n = _get(row, "NUMERO TRANSPORTE") + if m and m not in TIPO_TRANSPORTE_VALIDOS: + return _err( + line_num, + "TIPO TRANSPORTE", + f"Error: (Celda M{line_num}) El Tipo de Transporte: {m} no es válido. Válidos: NINGUNO, TRANSPORTE, CAJA, PLACAS, CAMION, BUQUE, FERROBARCAZA, CONTENEDOR, PLATAFORMA, GONDOLA, AVION.", + ) + if not m and n: + return _err( + line_num, + "NUMERO TRANSPORTE", + f"Error: (Celda N{line_num}) El Tipo de Transporte está vacío y está capturado un número de transporte.", + ) + if m == "NINGUNO" and n: + return _err( + line_num, + "NUMERO TRANSPORTE", + f"Error: (Celda N{line_num}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.", + ) + if m and m != "NINGUNO" and not n: + return _err( + line_num, + "NUMERO TRANSPORTE", + f"Error: (Celda N{line_num}) El Tipo de Transporte es {m} y no está capturado el número de transporte.", + ) + return None + + +# --- Tipo moneda y clave moneda (Col O, P) --- +def _validaciones_moneda( + row: Dict[str, Any], + line_num: int, + valid_currency_codes: Set[str], + invoice_has_partidas: Optional[bool], + existing_tipo_moneda: Optional[str], +) -> Optional[Dict[str, Any]]: + o = _get(row, "TIPO MONEDA").upper() + p = _get(row, "CLAVE MONEDA").upper() + if o and o not in TIPOS_MONEDA_VALIDOS: + return _err( + line_num, + "TIPO MONEDA", + f"Error: (Celda O{line_num}) La opción de Tipo Moneda: {row.get('TIPO MONEDA')} no es válida. Use ME, MN o MC.", + ) + if o == "MC": + if not p: + return _err( + line_num, + "CLAVE MONEDA", + f"Error: (Celda P{line_num}) La Clave de la Moneda es obligatoria cuando Tipo de Moneda es MC.", + ) + if p and valid_currency_codes and p not in valid_currency_codes: + return _err( + line_num, + "CLAVE MONEDA", + f"Error: (Celda P{line_num}) La Clave de la Moneda: {p} no existe en el Catálogo.", + ) + if invoice_has_partidas and existing_tipo_moneda and o and o != existing_tipo_moneda.upper(): + return _err( + line_num, + "TIPO MONEDA", + f"Error: (Celda O{line_num}) No se puede cambiar el Tipo de Moneda porque la factura ya tiene partidas. Use {existing_tipo_moneda}.", + ) + return None + + +# --- Tipo peso (Col Y) --- +def _validaciones_tipo_peso(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]: + y = _get(row, "TIPO PESO").upper() + if y and y not in TIPO_PESO_VALIDOS: + return _err( + line_num, + "TIPO PESO", + f"Error: (Celda Y{line_num}) La opción de Tipo de Peso: {row.get('TIPO PESO')} no es válida. Use KILOS o LIBRAS.", + ) + return None + + +# --- Catálogos: Proveedor, Vendido A, Enviado A, Agente, Transportista, Incoterm, Aduana --- +def _validaciones_catalogos( + row: Dict[str, Any], + line_num: int, + valid_provider_ids: Set[int], + valid_sold_to_ids: Set[int], + valid_shipped_to_ids: Set[int], + valid_provider_short_names: Set[str], + valid_sold_to_short_names: Set[str], + valid_shipped_to_short_names: Set[str], + valid_broker_ids: Set[int], + valid_broker_claves: Set[str], + valid_transporter_keys: Set[str], + valid_incoterms: Set[str], + valid_aduana_codes: Set[str], +) -> Optional[Dict[str, Any]]: + def check_id(val: Any, col: str, catalog_name: str, valid_set: Set[int]) -> Optional[Dict[str, Any]]: + if val is None or str(val).strip() == "": + return None + v = _parse_int(val) + if v is None: + return _err(line_num, col, f"Error: (Celda) {col} debe ser un número entero.") + if valid_set and v not in valid_set: + return _err(line_num, col, f"Error: La clave en {col} no existe en el Catálogo de {catalog_name}.") + return None + + def check_id_or_rfc( + val: Any, + col: str, + catalog_name: str, + valid_ids: Set[int], + valid_short_names: Set[str], + ) -> Optional[Dict[str, Any]]: + if val is None or str(val).strip() == "": + return None + v = _parse_int(val) + if v is not None: + if valid_ids and v not in valid_ids: + return _err(line_num, col, f"Error: La clave en {col} no existe en el Catálogo de {catalog_name}.") + return None + sn_norm = str(val).strip().upper() + if valid_short_names and sn_norm not in valid_short_names: + return _err(line_num, col, f"Error: La clave/corta en {col} no existe en el Catálogo de {catalog_name}.") + if not valid_short_names: + return _err(line_num, col, f"Error: (Celda) {col} debe ser un número entero o clave corta (short name) válida.") + return None + + def check_id_or_clave( + val: Any, + col: str, + catalog_name: str, + valid_ids: Set[int], + valid_claves: Set[str], + ) -> Optional[Dict[str, Any]]: + """Para AGENTE ADUANAL: acepta id (entero) o broker_key (clave).""" + if val is None or str(val).strip() == "": + return None + v = _parse_int(val) + if v is not None: + if valid_ids and v not in valid_ids: + return _err(line_num, col, f"Error: La clave en {col} no existe en el Catálogo de {catalog_name}.") + return None + clave = str(val).strip() + if valid_claves and clave not in valid_claves: + return _err(line_num, col, f"Error: La clave en {col} no existe en el Catálogo de {catalog_name}.") + if not valid_claves: + return _err(line_num, col, f"Error: (Celda) {col} debe ser un número entero o clave de agente aduanal válida.") + return None + + err = check_id_or_rfc( + row.get("CLAVE PROVEEDOR"), "CLAVE PROVEEDOR", "Clientes/Proveedores", + valid_provider_ids, valid_provider_short_names, + ) + if err: + return err + err = check_id_or_rfc( + row.get("CLAVE VENDIDO A"), "CLAVE VENDIDO A", "Clientes/Proveedores", + valid_sold_to_ids, valid_sold_to_short_names, + ) + if err: + return err + err = check_id_or_rfc( + row.get("CLAVE ENVIADO A"), "CLAVE ENVIADO A", "Clientes/Proveedores", + valid_shipped_to_ids, valid_shipped_to_short_names, + ) + if err: + return err + err = check_id_or_clave( + row.get("AGENTE ADUANAL"), "AGENTE ADUANAL", "Agentes Aduanales", + valid_broker_ids, valid_broker_claves, + ) + if err: + return err + + k = _get(row, "CLAVE TRANSPORTISTA").upper() + if k and valid_transporter_keys and k not in valid_transporter_keys: + return _err(line_num, "CLAVE TRANSPORTISTA", f"Error: (Celda K{line_num}) La Clave del Transportista: {k} no existe en el Catálogo.") + + v = _get(row, "CLAVE INCOTERM").upper() + if v and valid_incoterms and v not in valid_incoterms: + return _err(line_num, "CLAVE INCOTERM", f"Error: (Celda V{line_num}) La Clave de INCOTERM: {v} no existe en el Catálogo.") + + ab = _get(row, "ADUANA DE CRUCE") + if ab and valid_aduana_codes and ab not in valid_aduana_codes: + return _err(line_num, "ADUANA DE CRUCE", f"Error: (Celda AB{line_num}) La Aduana de Cruce: {ab} no existe en el Catálogo.") + + return None + + +def _validaciones_tipo_cambio( + row: Dict[str, Any], + line_num: int, + date_parsed: Optional[datetime], + exchange_rate_by_date: Dict[str, Any], + warnings: Optional[List[Dict[str, Any]]], +) -> Optional[Dict[str, Any]]: + """Si hay fecha y no hay tipo cambio, debe existir en catálogo. Si hay tipo cambio, puede advertir si difiere del catálogo.""" + d = _get(row, "FECHA FACTURA", "FECHA") + e = row.get("TIPO DE CAMBIO") + if not d or not date_parsed: + return None + date_key = date_parsed.isoformat()[:10] if hasattr(date_parsed, "isoformat") else str(date_parsed)[:10] + catalog_tc = exchange_rate_by_date.get(date_key) if exchange_rate_by_date else None + if not _clip(e): + if catalog_tc is None: + return _err( + line_num, + "TIPO DE CAMBIO", + f"Error: (Celda E{line_num}) El Tipo de Cambio para la Fecha {d} no se encontró en el Catálogo.", + ) + return None + val_e = _parse_decimal(e) + if val_e is not None and catalog_tc is not None: + cat_val = catalog_tc if isinstance(catalog_tc, (Decimal, int, float)) else getattr(catalog_tc, "valor", None) or getattr(catalog_tc, "value", None) + if cat_val is not None and abs(float(val_e) - float(cat_val)) > 0.0001 and warnings is not None: + warnings.append({ + "line": line_num, + "col": "TIPO DE CAMBIO", + "msg": f"Advertencia: (Celda E{line_num}) El Tipo de Cambio capturado ({e}) difiere del Catálogo para la fecha {d}.", + "warning": True, + }) + return None + + +def validate_row_encabezados_impo_temp( + row: Dict[str, Any], + line_num: int, + actualizar: bool, + invoice_exists_by_number: Dict[str, bool], + invoice_updated_by_number: Dict[str, bool], + pedimento_data_by_key: Dict[str, List[Dict[str, Any]]], + remesa_por_pedimento_bd: Dict[str, Set[int]], + remesa_por_pedimento_csv: Dict[str, Dict[int, str]], + valid_provider_ids: Set[int], + valid_sold_to_ids: Set[int], + valid_shipped_to_ids: Set[int], + valid_broker_ids: Set[int], + valid_transporter_keys: Set[str], + valid_incoterms: Set[str], + valid_aduana_codes: Set[str], + valid_currency_codes: Set[str], + valid_provider_short_names: Optional[Set[str]] = None, + valid_sold_to_short_names: Optional[Set[str]] = None, + valid_shipped_to_short_names: Optional[Set[str]] = None, + valid_broker_claves: Optional[Set[str]] = None, + exchange_rate_by_date: Optional[Dict[str, Any]] = None, + invoice_has_partidas_by_number: Optional[Dict[str, bool]] = None, + existing_tipo_moneda_by_number: Optional[Dict[str, str]] = None, + autonumerar_remesas: bool = False, + control_remesa: bool = False, + remesa_inicio: Optional[int] = None, + remesa_fin: Optional[int] = None, + date_format: Optional[str] = None, + parse_date_fn=None, + warnings: Optional[List[Dict[str, Any]]] = None, +) -> Optional[Dict[str, Any]]: + """ + Punto de entrada: valida una fila de CSV de Encabezados de Importación Temporal. + Clarion: VALIDA_TODA_FACIMPO_TEM vs VALIDA_PARCIAL_FACIMPO_TEM según actualizar y si la factura existe. + """ + factura = _get(row, "NUMERO FACTURA", "NUM FACTURA", "FACTURA") + if not factura: + return _err(line_num, "NUMERO FACTURA", "Error: (Col.C) La columna de Número de Factura está vacía y no se pueden hacer las validaciones.") + + if invoice_updated_by_number.get(factura.strip(), False): + return _err( + line_num, + "NUMERO FACTURA", + f"Error: (Celda C{line_num}) El Número de Factura: {factura} ya existe y está Actualizada, no se puede hacer cambios.", + ) + + if actualizar and factura.strip() not in invoice_exists_by_number: + return _err( + line_num, + "NUMERO FACTURA", + f"Error: (Col.C) Factura de importación {factura} no existe (modo Actualizar).", + ) + + use_partial = actualizar and invoice_exists_by_number.get(factura.strip(), False) + tiene_pedimento = bool(_get(row, "PEDIMENTO")) + + if not use_partial: + err = _validaciones_obligatorios_toda(row, line_num, tiene_pedimento) + if err: + return err + + invoice_date_parsed = None + if parse_date_fn: + date_str = _get(row, "FECHA FACTURA", "FECHA") + if date_str: + invoice_date_parsed = parse_date_fn(date_str, date_format) + + err = _validaciones_pedimento_remesa( + row, + line_num, + pedimento_data_by_key, + remesa_por_pedimento_bd, + remesa_por_pedimento_csv, + autonumerar_remesas, + control_remesa, + remesa_inicio, + remesa_fin, + date_format, + invoice_date_parsed, + ) + if err: + return err + + err = _validaciones_factura_longitud(row, line_num) + if err: + return err + err = _validaciones_regimen(row, line_num) + if err: + return err + err = _validaciones_transporte(row, line_num) + if err: + return err + + has_partidas = invoice_has_partidas_by_number.get(factura.strip(), False) if invoice_has_partidas_by_number else False + existing_moneda = existing_tipo_moneda_by_number.get(factura.strip()) if existing_tipo_moneda_by_number else None + err = _validaciones_moneda( + row, + line_num, + valid_currency_codes or set(), + has_partidas if use_partial else None, + existing_moneda if use_partial else None, + ) + if err: + return err + err = _validaciones_tipo_peso(row, line_num) + if err: + return err + err = _validaciones_catalogos( + row, + line_num, + valid_provider_ids or set(), + valid_sold_to_ids or set(), + valid_shipped_to_ids or set(), + valid_provider_short_names or set(), + valid_sold_to_short_names or set(), + valid_shipped_to_short_names or set(), + valid_broker_ids or set(), + valid_broker_claves or set(), + valid_transporter_keys or set(), + valid_incoterms or set(), + valid_aduana_codes or set(), + ) + if err: + return err + err = _validaciones_tipo_cambio( + row, line_num, invoice_date_parsed, exchange_rate_by_date or {}, warnings + ) + if err: + return err + + return None + + +def row_to_transport_type_clarion(val: Optional[str]) -> Optional[str]: + """Mapea valor Clarion Col M a valor enum TransportType (minúscula).""" + if not val: + return "none" + u = _clip(val).upper() + return TIPO_TRANSPORTE_CLARION_TO_NORM.get(u, "none")