feature/utf8-normalizado

backend/api/v1/modules/a76/layouts_csv/classes/tasks.py

@@ -33,6 +33,12 @@ CLS_IMPORT_ERROR_LINES_PREFIX = "cls_import_error_lines:"
 CLS_IMPORT_REDIS_TTL = common_storage.IMPORT_REDIS_TTL
 
 
+def _read_plan_for_classes(fieldnames):
+    if fieldnames:
+        return common_csv.CsvReadPlan(header_mode="headerless", fieldnames=fieldnames)
+    return common_csv.CsvReadPlan(header_mode="header")
+
+
 @celery_app.task(bind=True)
 def scan_file(self, job_id: str, config: str = None):
     logger.info("Classes import: starting scan for job %s", job_id)
@@ -45,8 +51,9 @@ def scan_file(self, job_id: str, config: str = None):
     error_path = common_storage.error_path_for_job(JOB_TYPE, job_id)
 
     fieldnames, has_header = detect_headers_or_data(file_path, common_normalize.normalize_header)
+    read_plan = _read_plan_for_classes(fieldnames)
     try:
-        total_rows = common_csv.count_csv_rows(file_path, has_header=has_header)
+        total_rows = common_csv.count_csv_rows(file_path, has_header=has_header, read_plan=read_plan)
     except Exception as e:
         return {"status": "failed", "error": str(e)}
 
@@ -83,7 +90,7 @@ def scan_file(self, job_id: str, config: str = None):
 
     try:
         with open(error_path, "w", encoding="utf-8") as f_err:
-            for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
+            for i, row in common_csv.iter_csv_rows_with_plan(file_path, read_plan=read_plan):
                 self.update_state(
                         state="PROGRESS",
                         meta={"current": i, "total": total_rows, "errors": error_count},
@@ -171,6 +178,7 @@ def insert_valid_rows(self, job_id: str):
     meta_path = common_meta.get_meta_path(file_path)
 
     fieldnames, _ = detect_headers_or_data(file_path, common_normalize.normalize_header)
+    read_plan = _read_plan_for_classes(fieldnames)
 
     try:
         with CoreSessionLocal() as session:
@@ -183,7 +191,7 @@ def insert_valid_rows(self, job_id: str):
                 if key:
                     existing_by_code[key] = c
 
-            for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames):
+            for i, row in common_csv.iter_csv_rows_with_plan(file_path, read_plan=read_plan):
                 if i in error_lines:
                     continue
 

backend/api/v1/modules/a76/layouts_csv/classes/template_config.py

@@ -7,28 +7,12 @@ import io
 from typing import Dict, List, Any, Optional, Tuple
 
 from ..common.cell_value import cell_to_str
+from ..common import csv_reader as common_csv_reader
 
 
 # Valores que indican que la primera fila es cabecera (primera columna normalizada)
 FIRST_COLUMN_HEADER_VALUES = ("CLAVE CLASE", "CLASE")
 
-_ENCODING_FALLBACKS: Tuple[str, ...] = ("utf-8-sig", "utf-8", "cp1252", "latin-1")
-
-
-def _read_text_sample(file_path: str, sample_bytes: int = 2048) -> str:
-    with open(file_path, "rb") as f:
-        raw = f.read(sample_bytes)
-    last_err: Optional[Exception] = None
-    for enc in _ENCODING_FALLBACKS:
-        try:
-            return raw.decode(enc)
-        except Exception as e:
-            last_err = e
-    if last_err:
-        raise last_err
-    return ""
-
-
 def detect_headers_or_data(
     file_path: str,
     normalize_header_fn,
@@ -42,24 +26,27 @@ def detect_headers_or_data(
     - Si no -> has_header=False, fieldnames=TEMPLATE_DOWNLOAD_HEADERS (la primera fila es dato).
     """
     try:
-        # `encoding` se mantiene por compatibilidad; si falla, hacemos fallback para CSVs tipo Excel (cp1252/latin-1).
-        if encoding and encoding.lower() not in ("auto", "detect"):
-            try:
-                with open(file_path, "r", encoding=encoding) as f:
-                    sample = f.read(2048)
-            except Exception:
-                sample = _read_text_sample(file_path, sample_bytes=2048)
-        else:
-            sample = _read_text_sample(file_path, sample_bytes=2048)
+        sample, _ = common_csv_reader.read_text_sample(
+            file_path,
+            requested_encoding=encoding,
+            sample_chars=2048,
+        )
     except Exception:
+        try:
+            sample, _ = common_csv_reader.read_text_sample(
+                file_path,
+                requested_encoding="auto",
+                sample_chars=2048,
+            )
+        except Exception:
+            return None, True
+
+    if not sample:
         return None, True
     lines = sample.splitlines()
     if not lines:
         return None, True
-    try:
-        dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
-    except Exception:
-        dialect = csv.excel
+    dialect = common_csv_reader.detect_csv_dialect(sample, delimiters=",;\t")
     reader = csv.reader(io.StringIO(lines[0]), dialect=dialect)
     first_row = next(reader, None)
     if not first_row:
@@ -124,10 +111,15 @@ def row_from_template(row: Dict[str, Any], normalize_header_fn) -> Dict[str, Any
         if key_norm in lookup:
             out[lookup[key_norm]] = cell_to_str(value)
         elif key_norm.startswith("CLAVE CLASE"):
-            # CSV leído con delimitador incorrecto: primera columna es "CLAVE CLASE,..." -> usar primer valor como CLASE
+            # CSV leído con delimitador incorrecto: primera columna puede venir colapsada.
             if "CLASE" not in out and value:
                 val_str = cell_to_str(value)
-                first_val = (val_str.split(",")[0] if "," in val_str else val_str).strip()
+                first_val = val_str
+                for delimiter in (",", ";", "\t"):
+                    if delimiter in first_val:
+                        first_val = first_val.split(delimiter)[0]
+                        break
+                first_val = first_val.lstrip("\ufeff").strip()
                 if first_val:
                     out["CLASE"] = first_val
     return out