""" Tareas Celery para importación CSV de Vehículos (Transportes). Flujo: scan_file (validación) → insert_valid_rows (commit). Usa layouts_csv.common (storage, normalize, meta, responses, csv_reader). """ import json import logging import os from typing import Dict, Any, Optional, List from core.celery_app import celery_app from core.database import CoreSessionLocal from ..common import storage as common_storage from ..common import normalize as common_normalize from ..common import meta as common_meta from ..common import responses as common_responses from ..common import csv_reader as common_csv_reader from .template_config import row_from_template from .validators import validate_row_vehicle from .common.mappers import row_to_vehicle_data logger = logging.getLogger(__name__) JOB_TYPE = "veh" # Para routes.py VEHL_IMPORT_FILE_PREFIX = "veh_import_file:" VEHL_IMPORT_META_PREFIX = "veh_import_meta:" VEHL_IMPORT_ERROR_LINES_PREFIX = "veh_import_error_lines:" VEHL_IMPORT_STATUS_PREFIX = "veh_import_status:" VEHL_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL def _get_redis(): import redis url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) return redis.Redis.from_url(url, decode_responses=False) def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, Any]: file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import") if not file_path: return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."} common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) try: total_rows = common_csv_reader.count_csv_rows(file_path) except Exception as e: return {"status": "failed", "error": str(e)} try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} error_count = 0 processed_rows = 0 errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path): if progress_callback and i % 500 == 0: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) err = validate_row_vehicle(row_norm, i) if err: error_count += 1 error_lines_list.append(err["line"]) f_err.write(json.dumps(err) + "\n") if len(errors_detail) < 500: errors_detail.append({ "line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", ""), }) processed_rows += 1 if error_lines_list: common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list) except Exception as e: logger.error("Vehicles import scan failed: %s", e) return {"status": "failed", "error": str(e)} return common_responses.scan_result(job_id, processed_rows, error_count, errors_detail) def run_scan_sync(job_id: str) -> Dict[str, Any]: result = _do_scan(job_id, progress_callback=None) try: r = _get_redis() r.set( f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", json.dumps(result).encode("utf-8"), ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: logger.warning("Vehicles import: failed to store scan status in Redis: %s", e) return result @celery_app.task(bind=True) def scan_file(self, job_id: str, config: str = None): logger.info("Vehicles import: starting scan for job %s", job_id) def on_progress(current: int, total: int, errors: int) -> None: self.update_state(state="PROGRESS", meta={"current": current, "total": total, "errors": errors}) result = _do_scan(job_id, progress_callback=on_progress) try: r = _get_redis() r.set( f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", json.dumps(result).encode("utf-8"), ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: logger.warning("Vehicles import: failed to store scan status in Redis: %s", e) return result def _do_commit(job_id: str) -> Dict[str, Any]: file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Vehicles import") if not file_path: alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id) if not os.path.exists(alt_path): return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."} file_path = alt_path else: common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Vehicles import") error_path = common_storage.error_path_for_job(JOB_TYPE, job_id) error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path) try: tenant_id, company_id = common_meta.require_tenant_context(file_path) except ValueError as e: return {"status": "failed", "error": str(e)} from api.v1.modules.a76.transportation.vehicles.services import VehicleService from api.v1.modules.a76.transportation.vehicles.dto import VehicleCreateDTO, VehicleUpdateDTO inserted_count = 0 updated_count = 0 skipped_invalid = 0 skipped_duplicate = 0 skipped_details: List[Dict[str, Any]] = [] seen_keys_in_file: Dict[str, int] = {} meta_path = common_meta.get_meta_path(file_path) try: with CoreSessionLocal() as session: for i, row in common_csv_reader.iter_csv_rows(file_path): if i in error_lines: continue row_norm = row_from_template(row, common_normalize.normalize_header) err = validate_row_vehicle(row_norm, i) if err: skipped_invalid += 1 vk = (row_norm.get("CLAVE") or "").strip()[:14] or "-" skipped_details.append({ "line": i, "vehicle_key": vk, "invoice": vk, "reason": f"{err.get('col', '')}: {err.get('msg', '')}", }) continue data = row_to_vehicle_data(row_norm) if not data or not data.get("vehicle_key"): skipped_invalid += 1 continue vk = data["vehicle_key"] if vk in seen_keys_in_file: skipped_duplicate += 1 skipped_details.append({ "line": i, "vehicle_key": vk, "invoice": vk, "reason": "Clave duplicada en el archivo (se usa la primera)", }) continue seen_keys_in_file[vk] = i existing = VehicleService.get_by_id(session, vk, tenant_id, company_id) try: if existing: update_data = VehicleUpdateDTO(**{k: v for k, v in data.items() if k != "vehicle_key"}) VehicleService.update(session, vk, tenant_id, update_data, company_id) updated_count += 1 else: create_data = VehicleCreateDTO(**data) VehicleService.create(session, create_data, tenant_id, company_id) inserted_count += 1 except Exception as db_err: session.rollback() skipped_invalid += 1 skipped_details.append({ "line": i, "vehicle_key": vk, "invoice": vk, "reason": str(db_err), }) continue try: session.commit() except Exception as db_err: session.rollback() logger.error("Vehicles import DB error: %s", db_err) return {"status": "failed", "error": str(db_err)} except Exception as e: logger.exception("Vehicles import task failed") return {"status": "failed", "error": str(e)} common_storage.cleanup_import_job( JOB_TYPE, job_id, file_path=file_path, error_path=error_path, meta_path=meta_path, ) try: r = _get_redis() r.delete(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}") except Exception as e: logger.warning("Vehicles import: failed to delete status key: %s", e) total_ok = inserted_count + updated_count if total_ok == 0 and (skipped_invalid + skipped_duplicate) > 0: return { "status": "warning", "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_duplicate": skipped_duplicate, "skipped_missing_fk": 0, "skipped_details": skipped_details, "message": f"No se insertaron registros. {skipped_invalid + skipped_duplicate} rechazados.", } if total_ok == 0: return { "status": "failed", "error": "No hay registros válidos en el archivo CSV", "inserted": 0, "updated": 0, "skipped_invalid": skipped_invalid, "skipped_duplicate": skipped_duplicate, "skipped_missing_fk": 0, "skipped_details": skipped_details, } return { "status": "finished", "inserted": inserted_count, "updated": updated_count, "skipped_invalid": skipped_invalid, "skipped_duplicate": skipped_duplicate, "skipped_missing_fk": 0, "skipped_details": skipped_details, } def run_commit_sync(job_id: str) -> Dict[str, Any]: result = _do_commit(job_id) try: r = _get_redis() r.set( f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", json.dumps(result).encode("utf-8"), ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: logger.warning("Vehicles import: failed to store commit status in Redis: %s", e) return result @celery_app.task(bind=True) def insert_valid_rows(self, job_id: str): logger.info("Vehicles import: starting commit for job %s", job_id) result = _do_commit(job_id) try: r = _get_redis() r.set( f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}", json.dumps(result).encode("utf-8"), ex=VEHL_IMPORT_REDIS_TTL, ) except Exception as e: logger.warning("Vehicles import: failed to store commit status in Redis: %s", e) return result