""" Tareas Celery para importación CSV de Clases de Materiales. Flujo: scan_file (validación) → insert_valid_rows (commit). Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers. Sin ClassService de creación en API; los mappers CSV (row_to_class_data, row_to_class_data_merge_existing) son la fuente de verdad para reglas de negocio al crear/actualizar. """ import json import logging import os from typing import Dict, Any, List from core.celery_app import celery_app from core.database import CoreSessionLocal from ..common import storage as common_storage from ..common import normalize as common_normalize from ..common import csv_reader as common_csv from ..common import meta as common_meta from ..common import responses as common_responses from .template_config import row_from_template, detect_headers_or_data from .validators import validate_row_class, validate_row_class_partial from .common.mappers import row_to_class_data, row_to_class_data_merge_existing from .common.fk_loader import load_classes_fk_sets logger = logging.getLogger(__name__) JOB_TYPE = "cls" # Para routes.py CLS_IMPORT_FILE_PREFIX = "cls_import_file:" CLS_IMPORT_META_PREFIX = "cls_import_meta:" CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:" CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): logger.info("Classes import: starting scan for job %s", job_id) file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header) try: total_rows = common_csv.count_csv_rows(file_path, has_header=has_header) except Exception as e: return {"status": "failed", "error": str(e)} try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) actualizar = meta.get("actualizar", False) siempre_toda = meta.get("siempre_toda", False) valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets( tenant_id, company_id ) from api.v1.modules.a76.classes.models import Class existing_class_codes = set() try: with CoreSessionLocal() as session: for c in session.query(Class.class_code).filter( Class.tenant_id == tenant_id, Class.company_id == company_id, ).all(): if c[0]: existing_class_codes.add(c[0].strip().upper()) except Exception as e: logger.warning("Classes import: could not load existing class codes: %s", e) error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) row_norm = row_from_template(row, common_normalize.normalize_header) row_errors = validate_row_class( row_norm, i, valid_material_keys=valid_material_keys, valid_uom_codes=valid_uom_codes, actualizar=actualizar, siempre_toda=siempre_toda, existing_class_codes=existing_class_codes, valid_fraction_mex_8=valid_fraction_mex_8, valid_fraction_ame=valid_fraction_ame, valid_product_codes_cp=valid_product_codes_cp, ) if row_errors: has_blocking_error = any(not e.get("warning", False) for e in row_errors) if has_blocking_error: error_count += 1 error_lines_list.append(i) for e in row_errors: if not e.get("warning", False): f_err.write(json.dumps(e) + "\n") if len(errors_detail) < 500: for e in row_errors: if len(errors_detail) >= 500: break errors_detail.append({ "line": e.get("line", i), "col": e.get("col", ""), "msg": e.get("msg", ""), "solution": e.get("solution", ""), "warning": bool(e.get("warning", False)), }) processed_rows += 1 if error_lines_list: common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.error("Classes import scan failed: %s", e) return {"status": "failed", "error": str(e)} return common_responses.scan_result( job_id, processed_rows, error_count, errors_detail ) @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): logger.info("Classes import: starting commit for job %s", job_id) file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import") if not file_path: alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} meta = common_meta.load_meta(file_path) actualizar = meta.get("actualizar", False) siempre_toda = meta.get("siempre_toda", False) from api.v1.modules.a76.classes.models import Class valid_material_keys, valid_uom_codes, valid_fraction_mex_8, valid_fraction_ame, valid_product_codes_cp = load_classes_fk_sets( tenant_id, company_id ) inserted_count = 0 skipped_invalid = 0 skipped_details: List[Dict[str, Any]] = [] response = None meta_path = common_meta.get_meta_path(file_path) fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header) try: with CoreSessionLocal() as session: existing_by_code = {} for c in session.query(Class).filter( Class.tenant_id == tenant_id, Class.company_id == company_id, ).all(): key = (c.class_code or "").strip().upper() if key: existing_by_code[key] = c for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): if i in error_lines: continue row_norm = row_from_template(row, common_normalize.normalize_header) class_code_raw = (row_norm.get("CLASE") or "").strip().upper()[:8] use_partial = actualizar and class_code_raw and class_code_raw in existing_by_code and not siempre_toda if use_partial: row_errors = validate_row_class_partial( row_norm, i, valid_material_keys=valid_material_keys, valid_uom_codes=valid_uom_codes, valid_fraction_mex_8=valid_fraction_mex_8, valid_fraction_ame=valid_fraction_ame, valid_product_codes_cp=valid_product_codes_cp, ) else: row_errors = validate_row_class( row_norm, i, valid_material_keys=valid_material_keys, valid_uom_codes=valid_uom_codes, actualizar=actualizar, siempre_toda=siempre_toda, existing_class_codes=set(existing_by_code.keys()), valid_fraction_mex_8=valid_fraction_mex_8, valid_fraction_ame=valid_fraction_ame, valid_product_codes_cp=valid_product_codes_cp, ) if row_errors and any(not e.get("warning", False) for e in row_errors): skipped_invalid += 1 blocking_errors = [e for e in row_errors if not e.get("warning", False)] for blocking in blocking_errors: skipped_details.append({ "line": i, "reason": f"{blocking.get('col', '')}: {blocking.get('msg', '')}", "solution": blocking.get("solution", ""), }) continue if use_partial: existing = existing_by_code.get(class_code_raw) existing_data = { "description_es": existing.description_es, "description_en": existing.description_en, "material_key": existing.material_key, "unit_of_measure": existing.unit_of_measure, "fraction": existing.fraction, "us_fraction": existing.us_fraction, "sub_key": existing.sub_key, "physical_review": existing.physical_review, "iva_exempt_fraction": existing.iva_exempt_fraction, } data = row_to_class_data_merge_existing( row_norm, existing_data, valid_material_keys, valid_uom_codes, ) else: data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes) class_code = data.get("class_code") if not class_code: skipped_invalid += 1 continue existing = existing_by_code.get(class_code) if existing: existing.description_es = data["description_es"] existing.description_en = data["description_en"] existing.material_key = data["material_key"] existing.unit_of_measure = data["unit_of_measure"] existing.fraction = data["fraction"] existing.us_fraction = data["us_fraction"] existing.sub_key = data["sub_key"] existing.physical_review = data["physical_review"] existing.iva_exempt_fraction = data["iva_exempt_fraction"] session.add(existing) inserted_count += 1 else: new_class = Class( tenant_id=tenant_id, company_id=company_id, **data, ) session.add(new_class) existing_by_code[class_code] = new_class inserted_count += 1 try: session.commit() except Exception as db_err: session.rollback() logger.error("Classes import DB error: %s", db_err) return common_responses.commit_result( "failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(db_err), ) if inserted_count == 0 and skipped_invalid > 0: response = common_responses.commit_result( "warning", 0, skipped_invalid, 0, 0, skipped_details, message=f"No se insertaron registros. {skipped_invalid} rechazados.", ) elif inserted_count == 0: response = common_responses.commit_result( "failed", 0, skipped_invalid, 0, 0, skipped_details, error="No hay registros válidos en el archivo CSV", ) else: response = common_responses.commit_result( "finished", inserted_count, skipped_invalid, 0, 0, skipped_details, ) except Exception as e: logger.exception("Classes import task failed") response = common_responses.commit_result( "failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e), ) common_storage.cleanup_import_job( JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path, ) if response is None: response = common_responses.commit_result( "failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado", ) return response