""" Tareas Celery para importación CSV de Números de Parte. Flujo: scan_file (validación) → insert_valid_rows (commit). Paridad Clarion: actualizar (ACT), validación full/parcial, merge existente, reemplazar_sin_preguntar, RFC desde clase. Sin PartService de creación en API; los mappers CSV (row_to_part_data, apply_rfc_exception_from_class) son la fuente de verdad para reglas de negocio al crear/actualizar. """ import json import logging import os from typing import Dict, Any, List from core.celery_app import celery_app from core.database import CoreSessionLocal from ..common import storage as common_storage from ..common import normalize as common_normalize from ..common import csv_reader as common_csv from ..common import meta as common_meta from ..common import responses as common_responses from .template_config import row_from_template, detect_headers_or_data from .validators import validate_row_part, get_row_warnings from .common.mappers import ( row_to_part_data, row_to_part_data_merge_existing, apply_rfc_exception_from_class, ) from .common.fk_loader import load_parts_fk_sets logger = logging.getLogger(__name__) JOB_TYPE = "part" # Para routes.py (upload guarda con estos prefijos) PART_IMPORT_FILE_PREFIX = "part_import_file:" PART_IMPORT_META_PREFIX = "part_import_meta:" PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:" PART_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _read_plan_for_parts(fieldnames): if fieldnames: return common_csv.CsvReadPlan(header_mode="headerless", fieldnames=fieldnames) return common_csv.CsvReadPlan(header_mode="header") @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): logger.info("Parts import: starting scan for job %s", job_id) file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header) read_plan = _read_plan_for_parts(fieldnames) try: total_rows = common_csv.count_csv_rows(file_path, has_header=has_header, read_plan=read_plan) except Exception as e: return {"status": "failed", "error": str(e)} try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) actualizar = meta.get("actualizar", False) ( valid_class_codes, valid_uom_codes, valid_currency_codes, valid_fraction_mex_8, valid_country_m3, authorized_sector_keys, company_has_prosec, is_rfc_exception, ) = load_parts_fk_sets(tenant_id, company_id) from api.v1.modules.a76.parts.models import Part existing_part_numbers = set() try: with CoreSessionLocal() as session: for p in session.query(Part.part_number).filter( Part.tenant_id == tenant_id, Part.company_id == company_id, ).all(): if p[0]: existing_part_numbers.add((p[0] or "").strip().upper()) except Exception as e: logger.warning("Parts import: could not load existing part numbers: %s", e) error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv.iter_csv_rows_with_plan(file_path, read_plan=read_plan): self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) row_norm = row_from_template(row, common_normalize.normalize_header) row_errors = validate_row_part( row_norm, i, valid_class_codes=valid_class_codes, valid_uom_codes=valid_uom_codes, valid_currency_codes=valid_currency_codes, valid_fraction_mex_8=valid_fraction_mex_8, valid_country_m3=valid_country_m3, authorized_sector_keys=authorized_sector_keys, company_has_prosec=company_has_prosec, is_rfc_exception=is_rfc_exception, actualizar=actualizar, existing_part_numbers=existing_part_numbers, ) warnings_list = get_row_warnings(row_norm, i) has_blocking_error = any(not e.get("warning", False) for e in row_errors) if has_blocking_error: error_count += 1 error_lines_list.append(i) for e in row_errors: if not e.get("warning", False): f_err.write(json.dumps(e) + "\n") if len(errors_detail) < 500: for e in row_errors: if len(errors_detail) >= 500: break errors_detail.append({ "line": e.get("line", i), "col": e.get("col", ""), "msg": e.get("msg", ""), "solution": e.get("solution", ""), "warning": bool(e.get("warning", False)), }) for w in warnings_list: if len(errors_detail) >= 500: break errors_detail.append({ "line": w.get("line", i), "col": w.get("col", ""), "msg": w.get("msg", ""), "solution": w.get("solution", ""), "warning": True, }) processed_rows += 1 if error_lines_list: common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.error("Parts import scan failed: %s", e) return {"status": "failed", "error": str(e)} return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail ) @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): logger.info("Parts import: starting commit for job %s", job_id) file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import") if not file_path: alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) actualizar = meta.get("actualizar", False) reemplazar_sin_preguntar = meta.get("reemplazar_sin_preguntar", True) from api.v1.modules.a76.parts.models import Part from api.v1.modules.a76.classes.models import Class ( valid_class_codes, valid_uom_codes, valid_currency_codes, valid_fraction_mex_8, valid_country_m3, authorized_sector_keys, company_has_prosec, is_rfc_exception, ) = load_parts_fk_sets(tenant_id, company_id) inserted_count = 0 skipped_invalid = 0 skipped_missing_fk = 0 skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] response = None meta_path = common_meta.get_meta_path(file_path) fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header) read_plan = _read_plan_for_parts(fieldnames) try: with CoreSessionLocal() as session: existing_by_part_number = {} for p in session.query(Part).filter( Part.tenant_id == tenant_id, Part.company_id == company_id, ).all(): key = (p.part_number or "").strip().upper() if key: existing_by_part_number[key] = p for i, row in common_csv.iter_csv_rows_with_plan(file_path, read_plan=read_plan): if i in error_lines: continue row_norm = row_from_template(row, common_normalize.normalize_header) part_number_raw = (row_norm.get("NUMPARTE") or "").strip().upper() use_partial = ( actualizar and part_number_raw and part_number_raw in existing_by_part_number and reemplazar_sin_preguntar ) row_errors = validate_row_part( row_norm, i, valid_class_codes=valid_class_codes, valid_uom_codes=valid_uom_codes, valid_currency_codes=valid_currency_codes, valid_fraction_mex_8=valid_fraction_mex_8, valid_country_m3=valid_country_m3, authorized_sector_keys=authorized_sector_keys, company_has_prosec=company_has_prosec, is_rfc_exception=is_rfc_exception, actualizar=actualizar, existing_part_numbers=set(existing_by_part_number.keys()), ) if row_errors and any(not e.get("warning", False) for e in row_errors): skipped_invalid += 1 blocking_errors = [e for e in row_errors if not e.get("warning", False)] for blocking in blocking_errors: skipped_details.append({ "line": i, "reason": f"{blocking.get('col', '')}: {blocking.get('msg', '')}", "solution": blocking.get("solution", ""), }) continue if use_partial: existing = existing_by_part_number.get(part_number_raw) existing_data = { "description_spanish": existing.description_spanish, "description_english": existing.description_english, "part_class": existing.part_class, "unit_of_measure": existing.unit_of_measure, "unit_cost": existing.unit_cost, "currency_type": existing.currency_type, "currency_key": existing.currency_key, "unit_weight": existing.unit_weight, "weight_type": existing.weight_type, "fraction": existing.fraction, "us_fraction": existing.us_fraction, "part_photo": existing.part_photo, } data = row_to_part_data_merge_existing( row_norm, existing_data, valid_class_codes, valid_uom_codes, valid_currency_codes, ) else: data = row_to_part_data( row_norm, valid_class_codes, valid_uom_codes, valid_currency_codes, ) part_number = data.get("part_number") if not part_number: skipped_invalid += 1 continue if is_rfc_exception and data.get("part_class"): class_obj = ( session.query(Class) .filter( Class.tenant_id == tenant_id, Class.company_id == company_id, Class.class_code == (data.get("part_class") or "").strip().upper(), ) .first() ) if class_obj: data = apply_rfc_exception_from_class(data, class_obj) existing = existing_by_part_number.get(part_number) if existing: if not reemplazar_sin_preguntar: skipped_duplicate += 1 continue for key, value in data.items(): if hasattr(existing, key): setattr(existing, key, value) session.add(existing) inserted_count += 1 else: new_part = Part( tenant_id=tenant_id, company_id=company_id, client_id=company_id, **data, ) session.add(new_part) existing_by_part_number[part_number] = new_part inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() logger.error("Parts import DB error: %s", db_err) return common_responses.commit_result( "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, skipped_details, error=str(db_err), ) total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate if inserted_count == 0 and total_skipped > 0: response = common_responses.commit_result( "warning", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, skipped_details, message=f"No se insertaron registros. {total_skipped} rechazados.", ) elif inserted_count == 0: response = common_responses.commit_result( "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, skipped_details, error="No hay registros válidos en el archivo CSV", ) else: response = common_responses.commit_result( "finished", inserted_count, skipped_invalid, skipped_missing_fk, skipped_duplicate, skipped_details, ) except Exception as e: logger.exception("Parts import task failed") response = common_responses.commit_result( "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, skipped_details, error=str(e), ) common_storage.cleanup_import_job( JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path, ) if response is None: response = common_responses.commit_result( "failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate, skipped_details, error="Error inesperado", ) return response