169 lines
6.1 KiB
Python
169 lines
6.1 KiB
Python
"""
|
|
Tareas Celery para importación CSV de BOMs.
|
|
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
|
Sin tabla BOM dedicada aún: insert_valid_rows solo valida y cuenta filas válidas.
|
|
Usa layouts_csv.common y common.fk_loader, validators.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
from typing import Dict, Any, List
|
|
|
|
from core.celery_app import celery_app
|
|
|
|
from ..common import storage as common_storage
|
|
from ..common import normalize as common_normalize
|
|
from ..common import csv_reader as common_csv
|
|
from ..common import meta as common_meta
|
|
from ..common import responses as common_responses
|
|
from .template_config import row_from_template
|
|
from .validators import validate_row_bom
|
|
from .common.fk_loader import load_boms_fk_sets
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
JOB_TYPE = "bom"
|
|
|
|
# Para routes.py
|
|
BOM_IMPORT_FILE_PREFIX = "bom_import_file:"
|
|
BOM_IMPORT_META_PREFIX = "bom_import_meta:"
|
|
BOM_IMPORT_ERROR_LINES_PREFIX = "bom_import_error_lines:"
|
|
BOM_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
|
|
|
|
|
@celery_app.task(bind=True)
|
|
def scan_file(self, job_id: str, config: str = None):
|
|
logger.info("BOMs import: starting scan for job %s", job_id)
|
|
|
|
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
|
if not file_path:
|
|
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
|
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
|
|
|
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
|
|
|
try:
|
|
total_rows = common_csv.count_csv_rows(file_path)
|
|
except Exception as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
try:
|
|
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
|
except ValueError as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
|
|
|
error_count = 0
|
|
processed_rows = 0
|
|
errors_detail: List[Dict[str, Any]] = []
|
|
error_lines_list: List[int] = []
|
|
|
|
try:
|
|
with open(error_path, "w", encoding="utf-8") as f_err:
|
|
for i, row in common_csv.iter_csv_rows(file_path):
|
|
if i % 500 == 0:
|
|
self.update_state(
|
|
state="PROGRESS",
|
|
meta={"current": i, "total": total_rows, "errors": error_count},
|
|
)
|
|
row_norm = row_from_template(row, common_normalize.normalize_header)
|
|
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
|
if err:
|
|
error_count += 1
|
|
error_lines_list.append(err["line"])
|
|
f_err.write(json.dumps(err) + "\n")
|
|
if len(errors_detail) < 500:
|
|
errors_detail.append({
|
|
"line": err["line"],
|
|
"col": err.get("col", ""),
|
|
"msg": err.get("msg", ""),
|
|
})
|
|
processed_rows += 1
|
|
|
|
if error_lines_list:
|
|
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
|
except Exception as e:
|
|
logger.error("BOMs import scan failed: %s", e)
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
return common_responses.scan_result(
|
|
job_id, processed_rows, error_count, errors_detail
|
|
)
|
|
|
|
|
|
@celery_app.task(bind=True)
|
|
def insert_valid_rows(self, job_id: str):
|
|
logger.info("BOMs import: starting commit for job %s", job_id)
|
|
|
|
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "BOMs import")
|
|
if not file_path:
|
|
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
|
if not os.path.exists(alt_path):
|
|
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
|
file_path = alt_path
|
|
else:
|
|
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "BOMs import")
|
|
|
|
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
|
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
|
|
|
try:
|
|
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
|
except ValueError as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
valid_part_numbers = load_boms_fk_sets(tenant_id, company_id)
|
|
|
|
inserted_count = 0
|
|
skipped_invalid = 0
|
|
valid_count = 0
|
|
skipped_details: List[Dict[str, Any]] = []
|
|
response = None
|
|
meta_path = common_meta.get_meta_path(file_path)
|
|
|
|
try:
|
|
for i, row in common_csv.iter_csv_rows(file_path):
|
|
if i in error_lines:
|
|
continue
|
|
|
|
row_norm = row_from_template(row, common_normalize.normalize_header)
|
|
err = validate_row_bom(row_norm, i, valid_part_numbers=valid_part_numbers)
|
|
if err:
|
|
skipped_invalid += 1
|
|
skipped_details.append({
|
|
"line": i,
|
|
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
|
|
})
|
|
continue
|
|
|
|
valid_count += 1
|
|
# Sin tabla BOM dedicada: no se escribe en DB; solo se cuentan filas válidas.
|
|
|
|
response = common_responses.commit_result(
|
|
"finished", inserted_count, skipped_invalid, 0, 0, skipped_details,
|
|
)
|
|
if valid_count > 0 and inserted_count == 0:
|
|
response["message"] = (
|
|
f"WIP: {valid_count} filas válidas. "
|
|
"La tabla BOM aún no existe en el sistema; no se insertó nada."
|
|
)
|
|
except Exception as e:
|
|
logger.exception("BOMs import task failed")
|
|
response = common_responses.commit_result(
|
|
"failed", 0, skipped_invalid, 0, 0, skipped_details, error=str(e),
|
|
)
|
|
|
|
common_storage.cleanup_import_job(
|
|
JOB_TYPE, job_id,
|
|
file_path=file_path,
|
|
error_path=error_path,
|
|
meta_path=meta_path,
|
|
)
|
|
|
|
if response is None:
|
|
response = common_responses.commit_result(
|
|
"failed", 0, skipped_invalid, 0, 0, skipped_details, error="Error inesperado",
|
|
)
|
|
return response
|