feature/abstraccion-de-funcionalidades-y-manejo-por-tareas-en-comun

This commit is contained in:
hreyes
2026-03-04 08:12:59 -07:00
parent 4c781b460a
commit e83205caf9
91 changed files with 4153 additions and 4571 deletions

View File

@@ -0,0 +1 @@
# common validators, mappers, fk_loader for classes CSV import

View File

@@ -0,0 +1,64 @@
"""
Helpers reutilizables para validación de filas CSV (clases de materiales).
"""
from typing import Dict, Any, Optional
def check_required(row: Dict[str, Any], col: str, line_num: int) -> Optional[Dict[str, Any]]:
val = (row.get(col) or "").strip()
if not val:
return {"line": line_num, "col": col, "msg": "Requerido"}
return None
def check_max_length(
row: Dict[str, Any],
col: str,
max_len: int,
line_num: int,
required: bool = False,
) -> Optional[Dict[str, Any]]:
val = (row.get(col) or "").strip()
if not val:
if required:
return {"line": line_num, "col": col, "msg": "Requerido"}
return None
if len(val) > max_len:
return {"line": line_num, "col": col, "msg": f"Máximo {max_len} caracteres"}
return None
def check_int_range(
row: Dict[str, Any],
col: str,
line_num: int,
min_val: int,
max_val: int,
) -> Optional[Dict[str, Any]]:
"""Solo valida si hay valor; devuelve error si no es int o está fuera de rango."""
val = row.get(col)
if val is None or val == "":
return None
try:
v = int(val)
if v < min_val or v > max_val:
return {"line": line_num, "col": col, "msg": "Valor fuera de rango"}
except (ValueError, TypeError):
return {"line": line_num, "col": col, "msg": "Debe ser número entero"}
return None
def check_in_set(
row: Dict[str, Any],
col: str,
line_num: int,
allowed: Optional[set],
msg: str = "No existe en el catálogo",
) -> Optional[Dict[str, Any]]:
"""Solo valida si hay valor y allowed no es None."""
val = (row.get(col) or "").strip()
if not val or allowed is None:
return None
if val not in allowed:
return {"line": line_num, "col": col, "msg": msg}
return None

View File

@@ -0,0 +1,37 @@
"""
Carga de conjuntos FK para validación/mapeo de import CSV de clases de materiales.
"""
from typing import Set, Tuple
from core.database import CoreSessionLocal
def load_classes_fk_sets(
tenant_id: int,
company_id: int,
) -> Tuple[Set[str], Set[str]]:
"""
Carga valid_material_keys (MaterialType.key) y valid_uom_codes (UnitOfMeasure.code).
Devuelve (valid_material_keys, valid_uom_codes).
"""
valid_material_keys: Set[str] = set()
valid_uom_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.public.reference_data.material_types.models import MaterialType
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
for m in session.query(MaterialType.key).all():
valid_material_keys.add(m[0])
for u in (
session.query(UnitOfMeasure.code)
.filter(
UnitOfMeasure.tenant_id == tenant_id,
UnitOfMeasure.company_id == company_id,
)
.all()
):
valid_uom_codes.add(u[0])
except Exception as e:
import logging
logging.getLogger(__name__).warning("Classes import: could not load FK sets: %s", e)
return valid_material_keys, valid_uom_codes

View File

@@ -0,0 +1,62 @@
"""
Mapeo fila CSV → datos para Class (clases de materiales).
"""
from typing import Dict, Any, Optional, Set
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _int_or_none(val: Any) -> Optional[int]:
if val is None or val == "":
return None
try:
return int(val)
except (ValueError, TypeError):
return None
def row_to_class_data(
row_norm: Dict[str, Any],
valid_material_keys: Set[str],
valid_uom_codes: Set[str],
) -> Dict[str, Any]:
"""
Mapea una fila normalizada del CSV a un diccionario de datos para Class.
Ajusta material_key y unit_of_measure a None si no están en los conjuntos.
"""
class_code = _str_or_none(row_norm.get("CLASE"), 8)
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
if material_key and material_key not in valid_material_keys:
material_key = None
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
if unit_of_measure and unit_of_measure not in valid_uom_codes:
unit_of_measure = None
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
physical_review = _int_or_none(row_norm.get("REVFISICA"))
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
return {
"class_code": class_code,
"description_es": desc_es,
"description_en": desc_en,
"material_key": material_key,
"unit_of_measure": unit_of_measure,
"fraction": fraction,
"us_fraction": us_fraction,
"sub_key": sub_key,
"physical_review": physical_review,
"iva_exempt_fraction": iva_exempt_fraction,
}

View File

@@ -1,528 +1,237 @@
"""
Tareas Celery para importación CSV de Clases de Materiales.
Flujo: scan_file (validación) → insert_valid_rows (commit).
Usa layouts_csv.common (storage, normalize, csv_reader, meta, responses) y common.fk_loader, validators, mappers.
"""
import os
import base64
import csv
import json
import logging
import re
import unicodedata
from typing import Dict, Any, Optional, List, Set
import os
from typing import Dict, Any, List
from core.celery_app import celery_app
from core.database import CoreSessionLocal
from core.paths import layout_path
from ..common import storage as common_storage
from ..common import normalize as common_normalize
from ..common import csv_reader as common_csv
from ..common import meta as common_meta
from ..common import responses as common_responses
from .template_config import row_from_template
from .validators import validate_row_class
from .common.mappers import row_to_class_data
from .common.fk_loader import load_classes_fk_sets
logger = logging.getLogger(__name__)
JOB_TYPE = "cls"
# Para routes.py
CLS_IMPORT_FILE_PREFIX = "cls_import_file:"
CLS_IMPORT_META_PREFIX = "cls_import_meta:"
CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
CLS_IMPORT_REDIS_TTL = 3600
def _get_redis():
import redis
url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0"))
return redis.Redis.from_url(url, decode_responses=False)
def _worker_upload_dir() -> str:
return layout_path("imports", "temp")
def _ensure_worker_has_file_from_redis(job_id: str) -> Optional[str]:
r = _get_redis()
data = r.get(f"{CLS_IMPORT_FILE_PREFIX}{job_id}")
if not data:
return None
try:
raw = base64.b64decode(data)
except Exception as e:
logger.warning(f"Classes import: failed to decode file from Redis: {e}")
return None
upload_dir = _worker_upload_dir()
os.makedirs(upload_dir, exist_ok=True)
file_path = os.path.join(upload_dir, f"cls_{job_id}.csv")
with open(file_path, "wb") as f:
f.write(raw)
return file_path
def _ensure_worker_has_meta_from_redis(job_id: str, file_path: str) -> bool:
r = _get_redis()
data = r.get(f"{CLS_IMPORT_META_PREFIX}{job_id}")
if not data:
return False
try:
meta = json.loads(data.decode("utf-8"))
except Exception as e:
logger.warning(f"Classes import: failed to decode meta from Redis: {e}")
return False
meta_path = file_path.replace(".csv", ".meta.json")
with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta, f)
return True
def _delete_import_from_redis(job_id: str) -> None:
try:
r = _get_redis()
r.delete(
f"{CLS_IMPORT_FILE_PREFIX}{job_id}",
f"{CLS_IMPORT_META_PREFIX}{job_id}",
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
)
except Exception as e:
logger.warning(f"Classes import: failed to delete Redis keys: {e}")
def normalize_header(name: Optional[str]) -> str:
if not name:
return ""
name = unicodedata.normalize("NFKD", str(name)).upper()
name = "".join(ch for ch in name if not unicodedata.combining(ch))
name = re.sub(r"[^A-Z0-9]+", " ", name)
return re.sub(r"\s+", " ", name).strip()
def _validate_row_class(
row: Dict[str, Any],
line_num: int,
valid_material_keys: Optional[Set[str]] = None,
valid_uom_codes: Optional[Set[str]] = None,
) -> Optional[Dict[str, Any]]:
class_code = (row.get("CLASE") or "").strip()
if not class_code:
return {"line": line_num, "col": "CLASE", "msg": "Requerido"}
if len(class_code) > 8:
return {"line": line_num, "col": "CLASE", "msg": "Máximo 8 caracteres"}
desc_es = (row.get("DESCRIPCIONE") or "").strip()
if desc_es and len(desc_es) > 500:
return {"line": line_num, "col": "DESCRIPCIONE", "msg": "Máximo 500 caracteres"}
desc_en = (row.get("DESCRIPCIONI") or "").strip()
if desc_en and len(desc_en) > 500:
return {"line": line_num, "col": "DESCRIPCIONI", "msg": "Máximo 500 caracteres"}
material_key = (row.get("CLAVEMAT") or "").strip()
if material_key:
if len(material_key) > 10:
return {"line": line_num, "col": "CLAVEMAT", "msg": "Máximo 10 caracteres"}
if valid_material_keys is not None and material_key not in valid_material_keys:
return {"line": line_num, "col": "CLAVEMAT", "msg": "Tipo de material no existe"}
uom = (row.get("UNIMED") or "").strip()
if uom:
if len(uom) > 5:
return {"line": line_num, "col": "UNIMED", "msg": "Máximo 5 caracteres"}
if valid_uom_codes is not None and uom not in valid_uom_codes:
return {"line": line_num, "col": "UNIMED", "msg": "Unidad de medida no existe"}
fraction = (row.get("FRACCION") or "").strip()
if fraction and len(fraction) > 20:
return {"line": line_num, "col": "FRACCION", "msg": "Máximo 20 caracteres"}
us_fraction = (row.get("FRACCIONAME") or "").strip()
if us_fraction and len(us_fraction) > 16:
return {"line": line_num, "col": "FRACCIONAME", "msg": "Máximo 16 caracteres"}
sub_key = (row.get("CLAVESUB") or "").strip()
if sub_key and len(sub_key) > 5:
return {"line": line_num, "col": "CLAVESUB", "msg": "Máximo 5 caracteres"}
iva_exempt = (row.get("FRACCIONEXENTAIVA") or "").strip()
if iva_exempt and len(iva_exempt) > 4:
return {"line": line_num, "col": "FRACCIONEXENTAIVA", "msg": "Máximo 4 caracteres"}
rev_fisica = row.get("REVFISICA")
if rev_fisica is not None and rev_fisica != "":
try:
v = int(rev_fisica)
if v < -32768 or v > 32767:
return {"line": line_num, "col": "REVFISICA", "msg": "Valor fuera de rango"}
except (ValueError, TypeError):
return {"line": line_num, "col": "REVFISICA", "msg": "Debe ser número entero"}
return None
CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
@celery_app.task(bind=True)
def scan_file(self, job_id: str, config: str = None):
logger.info(f"Classes import: starting scan for job {job_id}")
logger.info("Classes import: starting scan for job %s", job_id)
file_path = _ensure_worker_has_file_from_redis(job_id)
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
if not file_path:
return {"status": "failed", "error": "Archivo no encontrado (expirado o no subido). Sube de nuevo."}
_ensure_worker_has_meta_from_redis(job_id, file_path)
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
error_dir = layout_path("imports", "errors")
os.makedirs(error_dir, exist_ok=True)
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
total_rows = 0
try:
with open(file_path, "r", encoding="utf-8-sig") as f:
total_rows = sum(1 for _ in f) - 1
total_rows = common_csv.count_csv_rows(file_path)
except Exception as e:
return {"status": "failed", "error": str(e)}
meta_path = file_path.replace(".csv", ".meta.json")
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
except Exception as e:
logger.warning(f"Classes import: failed to read meta: {e}")
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
valid_material_keys: Set[str] = set()
valid_uom_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.public.reference_data.material_types.models import MaterialType
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
for m in session.query(MaterialType.key).all():
valid_material_keys.add(m[0])
for u in (
session.query(UnitOfMeasure.code)
.filter(
UnitOfMeasure.tenant_id == tenant_id,
UnitOfMeasure.company_id == company_id,
)
.all()
):
valid_uom_codes.add(u[0])
except Exception as e:
logger.warning(f"Classes import: could not load FK sets: {e}")
tenant_id, company_id = common_meta.require_tenant_context(file_path)
except ValueError as e:
return {"status": "failed", "error": str(e)}
valid_material_keys, valid_uom_codes = load_classes_fk_sets(tenant_id, company_id)
error_count = 0
processed_rows = 0
errors_detail: List[Dict[str, Any]] = []
error_lines_list: List[int] = []
try:
with open(file_path, "r", encoding="utf-8-sig") as f_in, open(
error_path, "w", encoding="utf-8"
) as f_err:
sample = f_in.read(2048)
f_in.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f_in, dialect=dialect)
for i, row in enumerate(reader, start=1):
with open(error_path, "w", encoding="utf-8") as f_err:
for i, row in common_csv.iter_csv_rows(file_path):
if i % 500 == 0:
self.update_state(
state="PROGRESS",
meta={"current": i, "total": total_rows, "errors": error_count},
)
row_norm = row_from_template(row, normalize_header)
err = _validate_row_class(
row_norm, i,
row_norm = row_from_template(row, common_normalize.normalize_header)
err = validate_row_class(
row_norm,
i,
valid_material_keys=valid_material_keys,
valid_uom_codes=valid_uom_codes,
)
if err:
error_count += 1
error_lines_list.append(err["line"])
f_err.write(json.dumps(err) + "\n")
if len(errors_detail) < 500:
errors_detail.append(
{"line": err["line"], "col": err.get("col", ""), "msg": err.get("msg", "")}
)
errors_detail.append({
"line": err["line"],
"col": err.get("col", ""),
"msg": err.get("msg", ""),
})
processed_rows += 1
if error_lines_list:
common_storage.store_error_lines(JOB_TYPE, job_id, error_lines_list)
except Exception as e:
logger.error(f"Classes import scan failed: {e}")
logger.error("Classes import scan failed: %s", e)
return {"status": "failed", "error": str(e)}
error_lines_list = []
try:
if os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
if "line" in err:
error_lines_list.append(err["line"])
except Exception:
pass
if error_lines_list:
r = _get_redis()
r.set(
f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}",
json.dumps(error_lines_list).encode("utf-8"),
ex=CLS_IMPORT_REDIS_TTL,
)
except Exception as e:
logger.warning(f"Classes import: failed to store error lines in Redis: {e}")
return {
"status": "waiting_confirmation",
"job_id": job_id,
"total_rows": processed_rows,
"error_count": error_count,
"valid_rows": processed_rows - error_count,
"errors": errors_detail,
}
def _str_or_none(val: Any, max_len: Optional[int] = None) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s:
return None
if max_len and len(s) > max_len:
return s[:max_len]
return s
def _int_or_none(val: Any) -> Optional[int]:
if val is None or val == "":
return None
try:
return int(val)
except (ValueError, TypeError):
return None
return common_responses.scan_result(
job_id, processed_rows, error_count, errors_detail
)
@celery_app.task(bind=True)
def insert_valid_rows(self, job_id: str):
logger.info(f"Classes import: starting commit for job {job_id}")
logger.info("Classes import: starting commit for job %s", job_id)
file_path = _ensure_worker_has_file_from_redis(job_id)
file_path = common_storage.ensure_file_from_redis(JOB_TYPE, job_id, "Classes import")
if not file_path:
alt_path = os.path.join(_worker_upload_dir(), f"cls_{job_id}.csv")
alt_path = common_storage.file_path_for_job(JOB_TYPE, job_id)
if not os.path.exists(alt_path):
return {
"status": "failed",
"error": "Archivo no encontrado (expirado). Sube y confirma de nuevo.",
}
return {"status": "failed", "error": "Archivo no encontrado (expirado). Sube y confirma de nuevo."}
file_path = alt_path
else:
_ensure_worker_has_meta_from_redis(job_id, file_path)
common_storage.ensure_meta_from_redis(JOB_TYPE, job_id, file_path, "Classes import")
error_dir = layout_path("imports", "errors")
error_path = os.path.join(error_dir, f"cls_{job_id}.jsonl")
error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
error_lines = common_storage.get_error_lines(JOB_TYPE, job_id, error_path)
error_lines = set()
try:
r = _get_redis()
raw = r.get(f"{CLS_IMPORT_ERROR_LINES_PREFIX}{job_id}")
if raw:
error_lines = set(json.loads(raw.decode("utf-8")))
except Exception as e:
logger.debug(f"Classes import: could not load error lines from Redis: {e}")
if not error_lines and os.path.exists(error_path):
with open(error_path, "r", encoding="utf-8") as f:
for line in f:
try:
err = json.loads(line)
error_lines.add(err["line"])
except Exception:
pass
meta_path = file_path.replace(".csv", ".meta.json")
tenant_id = None
company_id = None
meta = {}
if os.path.exists(meta_path):
try:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f) or {}
tenant_id = meta.get("tenant_id")
company_id = meta.get("company_id")
except Exception:
pass
if not tenant_id or not company_id:
return {"status": "failed", "error": "Falta contexto (tenant/company)"}
tenant_id, company_id = common_meta.require_tenant_context(file_path)
except ValueError as e:
return {"status": "failed", "error": str(e)}
from api.v1.modules.a76.classes.models import Class
valid_material_keys: Set[str] = set()
valid_uom_codes: Set[str] = set()
try:
with CoreSessionLocal() as session:
from api.v1.modules.public.reference_data.material_types.models import MaterialType
from api.v1.modules.a76.general_catalogs.units_of_measure.models import UnitOfMeasure
for m in session.query(MaterialType.key).all():
valid_material_keys.add(m[0])
for u in (
session.query(UnitOfMeasure.code)
.filter(
UnitOfMeasure.tenant_id == tenant_id,
UnitOfMeasure.company_id == company_id,
)
.all()
):
valid_uom_codes.add(u[0])
except Exception as e:
logger.warning(f"Classes import: could not load FK sets: {e}")
valid_material_keys, valid_uom_codes = load_classes_fk_sets(tenant_id, company_id)
inserted_count = 0
skipped_invalid = 0
skipped_details: List[Dict[str, Any]] = []
response = None
meta_path = common_meta.get_meta_path(file_path)
try:
with CoreSessionLocal() as session:
existing_by_code: Dict[str, Class] = {}
for c in (
session.query(Class)
.filter(
existing_by_code = {
c.class_code: c
for c in session.query(Class).filter(
Class.tenant_id == tenant_id,
Class.company_id == company_id,
).all()
}
for i, row in common_csv.iter_csv_rows(file_path):
if i in error_lines:
continue
row_norm = row_from_template(row, common_normalize.normalize_header)
err = validate_row_class(
row_norm,
i,
valid_material_keys=valid_material_keys,
valid_uom_codes=valid_uom_codes,
)
.all()
):
existing_by_code[c.class_code] = c
if err:
skipped_invalid += 1
skipped_details.append({
"line": i,
"reason": f"{err.get('col', '')}: {err.get('msg', '')}",
})
continue
with open(file_path, "r", encoding="utf-8-sig") as f:
sample = f.read(2048)
f.seek(0)
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
except Exception:
dialect = "excel"
reader = csv.DictReader(f, dialect=dialect)
data = row_to_class_data(row_norm, valid_material_keys, valid_uom_codes)
class_code = data.get("class_code")
if not class_code:
skipped_invalid += 1
continue
for i, row in enumerate(reader, start=1):
if i in error_lines:
continue
row_norm = row_from_template(row, normalize_header)
err = _validate_row_class(
row_norm, i,
valid_material_keys=valid_material_keys,
valid_uom_codes=valid_uom_codes,
existing = existing_by_code.get(class_code)
if existing:
existing.description_es = data["description_es"]
existing.description_en = data["description_en"]
existing.material_key = data["material_key"]
existing.unit_of_measure = data["unit_of_measure"]
existing.fraction = data["fraction"]
existing.us_fraction = data["us_fraction"]
existing.sub_key = data["sub_key"]
existing.physical_review = data["physical_review"]
existing.iva_exempt_fraction = data["iva_exempt_fraction"]
session.add(existing)
inserted_count += 1
else:
new_class = Class(
tenant_id=tenant_id,
company_id=company_id,
**data,
)
if err:
skipped_invalid += 1
skipped_details.append(
{"line": i, "reason": f"{err.get('col', '')}: {err.get('msg', '')}"}
)
continue
class_code = _str_or_none(row_norm.get("CLASE"), 8)
if not class_code:
skipped_invalid += 1
continue
existing = existing_by_code.get(class_code)
desc_es = _str_or_none(row_norm.get("DESCRIPCIONE"), 500)
desc_en = _str_or_none(row_norm.get("DESCRIPCIONI"), 500)
material_key = _str_or_none(row_norm.get("CLAVEMAT"), 10)
if material_key and material_key not in valid_material_keys:
material_key = None
unit_of_measure = _str_or_none(row_norm.get("UNIMED"), 5)
if unit_of_measure and unit_of_measure not in valid_uom_codes:
unit_of_measure = None
fraction = _str_or_none(row_norm.get("FRACCION"), 20)
us_fraction = _str_or_none(row_norm.get("FRACCIONAME"), 16)
sub_key = _str_or_none(row_norm.get("CLAVESUB"), 5)
physical_review = _int_or_none(row_norm.get("REVFISICA"))
iva_exempt_fraction = _str_or_none(row_norm.get("FRACCIONEXENTAIVA"), 4)
if existing:
existing.description_es = desc_es
existing.description_en = desc_en
existing.material_key = material_key
existing.unit_of_measure = unit_of_measure
existing.fraction = fraction
existing.us_fraction = us_fraction
existing.sub_key = sub_key
existing.physical_review = physical_review
existing.iva_exempt_fraction = iva_exempt_fraction
session.add(existing)
inserted_count += 1
else:
new_class = Class(
tenant_id=tenant_id,
company_id=company_id,
class_code=class_code,
description_es=desc_es,
description_en=desc_en,
material_key=material_key,
unit_of_measure=unit_of_measure,
fraction=fraction,
us_fraction=us_fraction,
sub_key=sub_key,
physical_review=physical_review,
iva_exempt_fraction=iva_exempt_fraction,
)
session.add(new_class)
existing_by_code[class_code] = new_class
inserted_count += 1
session.add(new_class)
existing_by_code[class_code] = new_class
inserted_count += 1
try:
session.commit()
except Exception as db_err:
session.rollback()
logger.error(f"Classes import DB error: {db_err}")
return {"status": "failed", "error": str(db_err)}
logger.error("Classes import DB error: %s", db_err)
return common_responses.commit_result(
"failed", 0, skipped_invalid, 0, 0,
skipped_details, error=str(db_err),
)
total_skipped = skipped_invalid
if inserted_count == 0 and total_skipped > 0:
response = {
"status": "warning",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
"message": f"No se insertaron registros. {total_skipped} rechazados.",
}
if inserted_count == 0 and skipped_invalid > 0:
response = common_responses.commit_result(
"warning", 0, skipped_invalid, 0, 0,
skipped_details,
message=f"No se insertaron registros. {skipped_invalid} rechazados.",
)
elif inserted_count == 0:
response = {
"status": "failed",
"error": "No hay registros válidos en el archivo CSV",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
response = common_responses.commit_result(
"failed", 0, skipped_invalid, 0, 0,
skipped_details,
error="No hay registros válidos en el archivo CSV",
)
else:
response = {
"status": "finished",
"inserted": inserted_count,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
response = common_responses.commit_result(
"finished", inserted_count, skipped_invalid, 0, 0,
skipped_details,
)
except Exception as e:
logger.error(f"Classes import task failed: {e}")
import traceback
logger.error(traceback.format_exc())
return {"status": "failed", "error": str(e)}
logger.exception("Classes import task failed")
response = common_responses.commit_result(
"failed", 0, skipped_invalid, 0, 0,
skipped_details, error=str(e),
)
try:
if file_path and os.path.exists(file_path):
os.remove(file_path)
if os.path.exists(error_path):
os.remove(error_path)
if os.path.exists(meta_path):
os.remove(meta_path)
_delete_import_from_redis(job_id)
except Exception as cleanup_err:
logger.warning(f"Classes import cleanup failed: {cleanup_err}")
common_storage.cleanup_import_job(
JOB_TYPE, job_id,
file_path=file_path,
error_path=error_path,
meta_path=meta_path,
)
if response is None:
response = {
"status": "failed",
"error": "Error inesperado",
"inserted": 0,
"skipped_invalid": skipped_invalid,
"skipped_missing_fk": 0,
"skipped_details": skipped_details,
}
response = common_responses.commit_result(
"failed", 0, skipped_invalid, 0, 0,
skipped_details, error="Error inesperado",
)
return response

View File

@@ -0,0 +1,3 @@
from .create import validate_row_class
__all__ = ["validate_row_class"]

View File

@@ -0,0 +1,55 @@
"""
Validaciones comunes de fila para import CSV de clases de materiales.
"""
from typing import Dict, Any, Optional, Set
from ..common.common_validators import (
check_max_length,
check_int_range,
check_in_set,
)
def validate_row_required(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
return check_max_length(row, "CLASE", 8, line_num, required=True)
def validate_row_lengths(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
checks = [
("DESCRIPCIONE", 500),
("DESCRIPCIONI", 500),
("CLAVEMAT", 10),
("UNIMED", 5),
("FRACCION", 20),
("FRACCIONAME", 16),
("CLAVESUB", 5),
("FRACCIONEXENTAIVA", 4),
]
for col, max_len in checks:
err = check_max_length(row, col, max_len, line_num)
if err:
return err
return None
def validate_row_types(row: Dict[str, Any], line_num: int) -> Optional[Dict[str, Any]]:
return check_int_range(row, "REVFISICA", line_num, -32768, 32767)
def validate_row_fks(
row: Dict[str, Any],
line_num: int,
valid_material_keys: Optional[Set[str]],
valid_uom_codes: Optional[Set[str]],
) -> Optional[Dict[str, Any]]:
err = check_in_set(
row, "CLAVEMAT", line_num, valid_material_keys, "Tipo de material no existe"
)
if err:
return err
err = check_in_set(
row, "UNIMED", line_num, valid_uom_codes, "Unidad de medida no existe"
)
if err:
return err
return None

View File

@@ -0,0 +1,40 @@
"""
Punto de entrada de validación para import de una fila de clase de material.
"""
from typing import Dict, Any, Optional, Set
from .common import (
validate_row_required,
validate_row_lengths,
validate_row_types,
validate_row_fks,
)
def validate_row_class(
row: Dict[str, Any],
line_num: int,
valid_material_keys: Optional[Set[str]] = None,
valid_uom_codes: Optional[Set[str]] = None,
) -> Optional[Dict[str, Any]]:
"""
Valida una fila de CSV de clases de materiales.
Encadena: requeridos → longitudes → tipos → FKs.
"""
err = validate_row_required(row, line_num)
if err:
return err
err = validate_row_lengths(row, line_num)
if err:
return err
err = validate_row_types(row, line_num)
if err:
return err
err = validate_row_fks(row, line_num, valid_material_keys, valid_uom_codes)
if err:
return err
return None