adision de bases para trabajo completo de back-v1.0.0

app/helpers/csv_mapper.py

@@ -0,0 +1,56 @@
+from typing import List, Dict, Any
+from app.modules.edos.schema import EdosUpload
+
+class CSVMapper:
+    """ Mapper: filter and mapping only necesary data, based on a schema"""
+    
+    FIELD_MAPPING = {
+        'csv_column_name': 'schema_field',
+        'No.' : 'publicacion_sat',
+        'RFC' : 'numero',
+        'Nombre del Contribuyente' : 'razon_social',
+        'Situación del contribuyente' : 'situacion',
+        'Número y fecha de oficio global definitivo SAT' : 'numero_definitivo',
+        'Publicación página SAT definitivo' : 'fecha_definitivo',
+        'Número y fecha de oficio global definitivo DOF' : 'numero_def_dof',
+        'Publicación DOF definitivo' : 'fecha_def_dof',
+        'Número y fecha de oficio global de sentencia favorable SAT' : 'publicacion_dof',
+        'Publicación página SAT sentencia favorable' : 'numero_fav_sat',
+    }
+
+    REQUIRED_FIELD = [ 'publicacion_sat', 'numero', 'razon_social', 'situacion'] #Campos obligatorios
+
+    @staticmethod
+    def filter_by_schema(raw_rows: List[Dict]) -> List[Dict]:
+        """ Filter only fields defined on schema"""
+        filtered = []
+
+        for row in raw_rows:
+            mapped_row = {}
+            for csv_field, schema_field in CSVMapper.FIELD_MAPPING.items():
+                if csv_field in row and row[csv_field]:
+                    mapped_row[schema_field] = CSVMapper._clean_value(
+                        row[csv_field], schema_field
+                    )
+                if all(field in mapped_row for field in CSVMapper.REQUIRED_FIELD):
+                    filtered.append(mapped_row)
+                else: 
+                    print(f"omited fields: {mapped_row}")
+        return filtered
+    
+    @staticmethod
+    def _clean_value(value: str, field_type: str) -> Any: 
+        """ Cleanen and typified values"""
+        if not value or value.strip() == '':
+            return None
+        
+        if 'date' in field_type:
+            from datetime import datetime
+            return datetime.strptime(value.strip(), '%Y-%m-%d')
+        elif 'amount' in field_type or 'price' in field_type:
+            return float(value.strip().replace(',','.'))
+        elif 'int' in field_type.lower():
+            return int(value.strip())
+        else:
+            return value.strip()
+        

app/helpers/db_adapter.py

@@ -0,0 +1,42 @@
+from typing import List, Dict
+from database import sessionLocal
+from app.modules.edos.models import EDOS
+
+class DBAdapter:
+    """ Adapter: insert cleen data on a db"""
+
+    def __inti__(self):
+        self.db = sessionLocal()
+
+    def insert_filtered_data(self, mapped_rows: List[Dict]) -> Dict:
+        """ Insert only data filtered and mapped"""
+        results = {
+            'total_processed': len(mapped_rows),
+            'inserted': 0,
+            'errors': [],
+            'duplicates_skiped': 0
+        }
+        for row in mapped_rows:
+            try: 
+                #verified dupled data
+                existing = self.db.query(EDOS).filter_by(
+                    unique_field=row.get('unique_field')
+                ).first()
+
+                if existing:
+                    results['duplicates_skiped'] += 1
+                    continue
+
+                instance = EDOS(**row)
+                self.db.add(instance)
+                self.db.commit()
+                results['inserted'] += 1
+            except Exception as e:
+                self.db.rollback()
+                results['errors'].append({
+                    'row': row,
+                    'error': str(e)
+                })
+            self.db.close()
+            return results
+        

app/helpers/extractCsv.py

@@ -0,0 +1,81 @@
+import csv
+import requests
+from io import StringIO
+#
+from typing import List, Dict, Any
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+class Settings:
+    """configuration"""
+    CSV_DOWNLOAD_TIMEOUT = int(os.getenv('CSV_DOWNLOAD_TIMEOUT', 30))
+    CSV_MAX_SIZE_MB = int(os.getenv('CSV_MAX_SIZE_MB', 10))
+
+settings = Settings()
+
+
+class CSVExtractor:
+    """Extract raw data of CSV"""
+
+    @staticmethod
+    def download_csv(url: str) -> str:
+        """ Download CSV of URL"""
+        try: 
+
+            response = requests.get(url, timeout=30)
+            response.raise_for_status()
+
+            encoding = response.encoding or 'utf-8'
+            content = response.text
+            return content, encoding
+        except Exception as e:
+            raise Exception(f"Error Download CSV: {str(e)}")
+        
+    
+    @staticmethod
+    def read_csv(content: str) -> List[Dict[str, Any]]:
+        """ Converts CSV to list of dictionary"""
+        csv_file = StringIO(content)
+        #detect delimiter auto
+        sample = csv_file.read(1024)
+        csv_file.seek(0)
+        try:
+            dialect = csv.Sniffer().sniff(sample)
+            reader  =csv.DictReader(csv_file, dialect=dialect)
+        except:
+            csv_file.seek(0)
+            reader = csv.DictReader(csv_file)
+        
+        rows = [row for row in reader]
+
+        if rows: 
+            clean_row = {}
+
+            for key, value in rows[0].items():
+
+                if key is not None:
+                    clean_key = key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
+                else:
+                    clean_key = ''
+            
+                clean_row[clean_key] = value
+
+            for i, row in enumerate(rows):
+                new_row = {}
+                for old_key, value in row.items():
+                    if old_key is not None:
+                        new_key = old_key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
+                    else: 
+                        new_key = ''
+                    #FIX: manage value None for avoid error strip
+                    if value is not None and isinstance(value, str):
+                        new_value = value.strip()
+                    else:
+                        new_value = value
+
+                    new_row[new_key] = value
+                rows[i] = new_row
+        return rows