384 lines
16 KiB
Python
384 lines
16 KiB
Python
"""
|
|
Tareas Celery para importación CSV de Números de Parte.
|
|
Flujo: scan_file (validación) → insert_valid_rows (commit).
|
|
Paridad Clarion: actualizar (ACT), validación full/parcial, merge existente, reemplazar_sin_preguntar, RFC desde clase.
|
|
Sin PartService de creación en API; los mappers CSV (row_to_part_data, apply_rfc_exception_from_class) son la fuente de verdad para reglas de negocio al crear/actualizar.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
from typing import Dict, Any, List
|
|
|
|
from core.celery_app import celery_app
|
|
from core.database import CoreSessionLocal
|
|
|
|
from ..common import storage as common_storage
|
|
from ..common import normalize as common_normalize
|
|
from ..common import csv_reader as common_csv
|
|
from ..common import meta as common_meta
|
|
from ..common import responses as common_responses
|
|
from .template_config import row_from_template, detect_headers_or_data
|
|
from .validators import validate_row_part, get_row_warnings
|
|
from .common.mappers import (
|
|
row_to_part_data,
|
|
row_to_part_data_merge_existing,
|
|
apply_rfc_exception_from_class,
|
|
)
|
|
from .common.fk_loader import load_parts_fk_sets
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
JOB_TYPE = "part"
|
|
|
|
# Para routes.py (upload guarda con estos prefijos)
|
|
PART_IMPORT_FILE_PREFIX = "part_import_file:"
|
|
PART_IMPORT_META_PREFIX = "part_import_meta:"
|
|
PART_IMPORT_ERROR_LINES_PREFIX = "part_import_error_lines:"
|
|
PART_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
|
|
|
|
|
|
def _read_plan_for_parts(fieldnames):
|
|
if fieldnames:
|
|
return common_csv.CsvReadPlan(header_mode="headerless", fieldnames=fieldnames)
|
|
return common_csv.CsvReadPlan(header_mode="header")
|
|
|
|
|
|
@celery_app.task(bind=True)
|
|
def scan_file(self, job_id: str, config: str = None):
|
|
logger.info("Parts import: starting scan for job %s", job_id)
|
|
|
|
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import")
|
|
if not file_path:
|
|
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
|
|
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import")
|
|
|
|
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
|
|
|
fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header)
|
|
read_plan = _read_plan_for_parts(fieldnames)
|
|
try:
|
|
total_rows = common_csv.count_csv_rows(file_path, has_header=has_header, read_plan=read_plan)
|
|
except Exception as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
try:
|
|
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
|
except ValueError as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
meta = common_meta.load_meta(file_path)
|
|
actualizar = meta.get("actualizar", False)
|
|
|
|
(
|
|
valid_class_codes,
|
|
valid_uom_codes,
|
|
valid_currency_codes,
|
|
valid_fraction_mex_8,
|
|
valid_country_m3,
|
|
authorized_sector_keys,
|
|
company_has_prosec,
|
|
is_rfc_exception,
|
|
) = load_parts_fk_sets(tenant_id, company_id)
|
|
|
|
from api.v1.modules.a76.parts.models import Part
|
|
existing_part_numbers = set()
|
|
try:
|
|
with CoreSessionLocal() as session:
|
|
for p in session.query(Part.part_number).filter(
|
|
Part.tenant_id == tenant_id,
|
|
Part.company_id == company_id,
|
|
).all():
|
|
if p[0]:
|
|
existing_part_numbers.add((p[0] or "").strip().upper())
|
|
except Exception as e:
|
|
logger.warning("Parts import: could not load existing part numbers: %s", e)
|
|
|
|
error_count = 0
|
|
processed_rows = 0
|
|
errors_detail: List[Dict[str, Any]] = []
|
|
error_lines_list: List[int] = []
|
|
|
|
try:
|
|
with open(error_path, "w", encoding="utf-8") as f_err:
|
|
for i, row in common_csv.iter_csv_rows_with_plan(file_path, read_plan=read_plan):
|
|
self.update_state(
|
|
state="PROGRESS",
|
|
meta={"current": i, "total": total_rows, "errors": error_count},
|
|
)
|
|
row_norm = row_from_template(row, common_normalize.normalize_header)
|
|
row_errors = validate_row_part(
|
|
row_norm,
|
|
i,
|
|
valid_class_codes=valid_class_codes,
|
|
valid_uom_codes=valid_uom_codes,
|
|
valid_currency_codes=valid_currency_codes,
|
|
valid_fraction_mex_8=valid_fraction_mex_8,
|
|
valid_country_m3=valid_country_m3,
|
|
authorized_sector_keys=authorized_sector_keys,
|
|
company_has_prosec=company_has_prosec,
|
|
is_rfc_exception=is_rfc_exception,
|
|
actualizar=actualizar,
|
|
existing_part_numbers=existing_part_numbers,
|
|
)
|
|
warnings_list = get_row_warnings(row_norm, i)
|
|
has_blocking_error = any(not e.get("warning", False) for e in row_errors)
|
|
if has_blocking_error:
|
|
error_count += 1
|
|
error_lines_list.append(i)
|
|
for e in row_errors:
|
|
if not e.get("warning", False):
|
|
f_err.write(json.dumps(e) + "\n")
|
|
|
|
if len(errors_detail) < 500:
|
|
for e in row_errors:
|
|
if len(errors_detail) >= 500:
|
|
break
|
|
errors_detail.append({
|
|
"line": e.get("line", i),
|
|
"col": e.get("col", ""),
|
|
"msg": e.get("msg", ""),
|
|
"solution": e.get("solution", ""),
|
|
"warning": bool(e.get("warning", False)),
|
|
})
|
|
for w in warnings_list:
|
|
if len(errors_detail) >= 500:
|
|
break
|
|
errors_detail.append({
|
|
"line": w.get("line", i),
|
|
"col": w.get("col", ""),
|
|
"msg": w.get("msg", ""),
|
|
"solution": w.get("solution", ""),
|
|
"warning": True,
|
|
})
|
|
processed_rows += 1
|
|
|
|
if error_lines_list:
|
|
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
|
|
except Exception as e:
|
|
logger.error("Parts import scan failed: %s", e)
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
return common_responses.scan_result(
|
|
job_id, processed_rows, error_count, errors_detail
|
|
)
|
|
|
|
|
|
@celery_app.task(bind=True)
|
|
def insert_valid_rows(self, job_id: str):
|
|
logger.info("Parts import: starting commit for job %s", job_id)
|
|
|
|
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Parts import")
|
|
if not file_path:
|
|
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
|
|
if not os.path.exists(alt_path):
|
|
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
|
|
file_path = alt_path
|
|
else:
|
|
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Parts import")
|
|
|
|
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
|
|
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
|
|
|
|
try:
|
|
tenant_id, company_id = common_meta.require_tenant_context(file_path)
|
|
except ValueError as e:
|
|
return {"status": "failed", "error": str(e)}
|
|
|
|
meta = common_meta.load_meta(file_path)
|
|
actualizar = meta.get("actualizar", False)
|
|
reemplazar_sin_preguntar = meta.get("reemplazar_sin_preguntar", True)
|
|
|
|
from api.v1.modules.a76.parts.models import Part
|
|
from api.v1.modules.a76.classes.models import Class
|
|
|
|
(
|
|
valid_class_codes,
|
|
valid_uom_codes,
|
|
valid_currency_codes,
|
|
valid_fraction_mex_8,
|
|
valid_country_m3,
|
|
authorized_sector_keys,
|
|
company_has_prosec,
|
|
is_rfc_exception,
|
|
) = load_parts_fk_sets(tenant_id, company_id)
|
|
|
|
inserted_count = 0
|
|
skipped_invalid = 0
|
|
skipped_missing_fk = 0
|
|
skipped_duplicate = 0
|
|
skipped_details: List[Dict[str, Any]] = []
|
|
response = None
|
|
meta_path = common_meta.get_meta_path(file_path)
|
|
|
|
fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header)
|
|
read_plan = _read_plan_for_parts(fieldnames)
|
|
|
|
try:
|
|
with CoreSessionLocal() as session:
|
|
existing_by_part_number = {}
|
|
for p in session.query(Part).filter(
|
|
Part.tenant_id == tenant_id,
|
|
Part.company_id == company_id,
|
|
).all():
|
|
key = (p.part_number or "").strip().upper()
|
|
if key:
|
|
existing_by_part_number[key] = p
|
|
|
|
for i, row in common_csv.iter_csv_rows_with_plan(file_path, read_plan=read_plan):
|
|
if i in error_lines:
|
|
continue
|
|
|
|
row_norm = row_from_template(row, common_normalize.normalize_header)
|
|
part_number_raw = (row_norm.get("NUMPARTE") or "").strip().upper()
|
|
use_partial = (
|
|
actualizar
|
|
and part_number_raw
|
|
and part_number_raw in existing_by_part_number
|
|
and reemplazar_sin_preguntar
|
|
)
|
|
|
|
row_errors = validate_row_part(
|
|
row_norm,
|
|
i,
|
|
valid_class_codes=valid_class_codes,
|
|
valid_uom_codes=valid_uom_codes,
|
|
valid_currency_codes=valid_currency_codes,
|
|
valid_fraction_mex_8=valid_fraction_mex_8,
|
|
valid_country_m3=valid_country_m3,
|
|
authorized_sector_keys=authorized_sector_keys,
|
|
company_has_prosec=company_has_prosec,
|
|
is_rfc_exception=is_rfc_exception,
|
|
actualizar=actualizar,
|
|
existing_part_numbers=set(existing_by_part_number.keys()),
|
|
)
|
|
if row_errors and any(not e.get("warning", False) for e in row_errors):
|
|
skipped_invalid += 1
|
|
blocking_errors = [e for e in row_errors if not e.get("warning", False)]
|
|
for blocking in blocking_errors:
|
|
skipped_details.append({
|
|
"line": i,
|
|
"reason": f"{blocking.get('col', '')}: {blocking.get('msg', '')}",
|
|
"solution": blocking.get("solution", ""),
|
|
})
|
|
continue
|
|
|
|
if use_partial:
|
|
existing = existing_by_part_number.get(part_number_raw)
|
|
existing_data = {
|
|
"description_spanish": existing.description_spanish,
|
|
"description_english": existing.description_english,
|
|
"part_class": existing.part_class,
|
|
"unit_of_measure": existing.unit_of_measure,
|
|
"unit_cost": existing.unit_cost,
|
|
"currency_type": existing.currency_type,
|
|
"currency_key": existing.currency_key,
|
|
"unit_weight": existing.unit_weight,
|
|
"weight_type": existing.weight_type,
|
|
"fraction": existing.fraction,
|
|
"us_fraction": existing.us_fraction,
|
|
"part_photo": existing.part_photo,
|
|
}
|
|
data = row_to_part_data_merge_existing(
|
|
row_norm,
|
|
existing_data,
|
|
valid_class_codes,
|
|
valid_uom_codes,
|
|
valid_currency_codes,
|
|
)
|
|
else:
|
|
data = row_to_part_data(
|
|
row_norm,
|
|
valid_class_codes,
|
|
valid_uom_codes,
|
|
valid_currency_codes,
|
|
)
|
|
|
|
part_number = data.get("part_number")
|
|
if not part_number:
|
|
skipped_invalid += 1
|
|
continue
|
|
|
|
if is_rfc_exception and data.get("part_class"):
|
|
class_obj = (
|
|
session.query(Class)
|
|
.filter(
|
|
Class.tenant_id == tenant_id,
|
|
Class.company_id == company_id,
|
|
Class.class_code == (data.get("part_class") or "").strip().upper(),
|
|
)
|
|
.first()
|
|
)
|
|
if class_obj:
|
|
data = apply_rfc_exception_from_class(data, class_obj)
|
|
|
|
existing = existing_by_part_number.get(part_number)
|
|
if existing:
|
|
if not reemplazar_sin_preguntar:
|
|
skipped_duplicate += 1
|
|
continue
|
|
for key, value in data.items():
|
|
if hasattr(existing, key):
|
|
setattr(existing, key, value)
|
|
session.add(existing)
|
|
inserted_count += 1
|
|
else:
|
|
new_part = Part(
|
|
tenant_id=tenant_id,
|
|
company_id=company_id,
|
|
client_id=company_id,
|
|
**data,
|
|
)
|
|
session.add(new_part)
|
|
existing_by_part_number[part_number] = new_part
|
|
inserted_count += 1
|
|
|
|
try:
|
|
session.commit()
|
|
except Exception as db_err:
|
|
session.rollback()
|
|
logger.error("Parts import DB error: %s", db_err)
|
|
return common_responses.commit_result(
|
|
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
|
skipped_details, error=str(db_err),
|
|
)
|
|
|
|
total_skipped = skipped_invalid + skipped_missing_fk + skipped_duplicate
|
|
if inserted_count == 0 and total_skipped > 0:
|
|
response = common_responses.commit_result(
|
|
"warning", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
|
skipped_details,
|
|
message=f"No se insertaron registros. {total_skipped} rechazados.",
|
|
)
|
|
elif inserted_count == 0:
|
|
response = common_responses.commit_result(
|
|
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
|
skipped_details,
|
|
error="No hay registros válidos en el archivo CSV",
|
|
)
|
|
else:
|
|
response = common_responses.commit_result(
|
|
"finished", inserted_count, skipped_invalid, skipped_missing_fk,
|
|
skipped_duplicate, skipped_details,
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.exception("Parts import task failed")
|
|
response = common_responses.commit_result(
|
|
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
|
skipped_details, error=str(e),
|
|
)
|
|
|
|
common_storage.cleanup_import_job(
|
|
JOB_TYPE, job_id,
|
|
file_path=file_path,
|
|
error_path=error_path,
|
|
meta_path=meta_path,
|
|
)
|
|
|
|
if response is None:
|
|
response = common_responses.commit_result(
|
|
"failed", 0, skipped_invalid, skipped_missing_fk, skipped_duplicate,
|
|
skipped_details, error="Error inesperado",
|
|
)
|
|
return response
|