full adjustment

app/helpers/extractCsv.py

@@ -1,3 +1,5 @@
+#helper/csv_mapper
+
 import csv
 import requests
 from io import StringIO
@@ -5,6 +7,10 @@ from io import StringIO
 from typing import List, Dict, Any
 import os
 from dotenv import load_dotenv
+#
+import pandas as pd
+import math
+
 
 load_dotenv()
 
@@ -17,6 +23,21 @@ class Settings:
 settings = Settings()
 
 
+def _find_header_row(self, rows, delimiter):
+        """ Search row with RFC an name"""
+
+        for idx, row in enumerate(rows):
+            #Normalice: Convert to String, clean space, mayus
+            clean_row = [str(cell).strip().upper() for cell in row]
+
+            has_rfc = any('RFC' in cell for cell in clean_row)
+            has_nombre =any('NOMBRE' in cell for cell in clean_row)
+
+            if has_rfc and has_nombre:
+                return idx
+        return None
+
+
 class CSVExtractor:
     """Extract raw data of CSV"""
 
@@ -36,46 +57,40 @@ class CSVExtractor:
         
     
     @staticmethod
-    def read_csv(content: str) -> List[Dict[str, Any]]:
+    def read_csv(content):
         """ Converts CSV to list of dictionary"""
-        csv_file = StringIO(content)
-        #detect delimiter auto
-        sample = csv_file.read(1024)
-        csv_file.seek(0)
-        try:
-            dialect = csv.Sniffer().sniff(sample)
-            reader  =csv.DictReader(csv_file, dialect=dialect)
-        except:
-            csv_file.seek(0)
-            reader = csv.DictReader(csv_file)
-        
-        rows = [row for row in reader]
+        for delimiter in[',', ';', '\t']:
 
-        if rows: 
-            clean_row = {}
+            try: 
+                lines = content.split('\n')
 
-            for key, value in rows[0].items():
+                header_row_idx = None
+                for idx, line in enumerate(lines):
+                    if 'RFC' in line.upper():
+                        header_row_idx = idx
+                        break
+                
+                if header_row_idx is None:
+                    continue
 
-                if key is not None:
-                    clean_key = key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
-                else:
-                    clean_key = ''
-            
-                clean_row[clean_key] = value
+                df = pd.read_csv(
+                    StringIO(content),
+                    delimiter=delimiter,
+                    encoding='utf-8', 
+                    skiprows=header_row_idx,
+                    dtype=str,
+                    keep_default_na=False,
+                    na_filter=False)
 
-            for i, row in enumerate(rows):
-                new_row = {}
-                for old_key, value in row.items():
-                    if old_key is not None:
-                        new_key = old_key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
-                    else: 
-                        new_key = ''
-                    #FIX: manage value None for avoid error strip
-                    if value is not None and isinstance(value, str):
-                        new_value = value.strip()
-                    else:
-                        new_value = value
+                df.columns = df.columns.str.strip()
 
-                    new_row[new_key] = value
-                rows[i] = new_row
-        return rows
+                df = df.dropna(how='all')
+                result = df.to_dict('records')
+                if result:
+                    return result
+
+            except Exception as e:
+                print(f"Error with delimiter '{delimiter}': {e}")
+                continue
+        raise ValueError("Don't read CSV with pandas")
+