ADUANASOFT/Verificacion_EFOS_Backend
10
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

full adjustment

Browse Source
This commit is contained in:
gerardoe
2026-04-09 14:03:06 -07:00
parent 75ecb00a94
commit 092a7048d8
21 changed files with 416 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/helpers/__pycache__/csv_mapper.cpython-311.pyc
View File

Binary file not shown.

BIN
app/helpers/__pycache__/db_adapter.cpython-311.pyc
View File

Binary file not shown.

BIN
app/helpers/__pycache__/extractCsv.cpython-311.pyc
View File

Binary file not shown.

106
app/helpers/csv_mapper.py
View File

@@ -1,56 +1,74 @@
from typing import List, Dict, Any
#helpers/csv_mapper
from typing import List, Dict, Any, Optional
from app.modules.edos.schema import EdosUpload
from datetime import date, datetime
class CSVMapper:
""" Mapper: filter and mapping only necesary data, based on a schema"""
FIELD_MAPPING = {
'csv_column_name': 'schema_field',
'No.' : 'publicacion_sat',
'RFC' : 'numero',
'Nombre del Contribuyente' : 'razon_social',
'Situación del contribuyente' : 'situacion',
'Número y fecha de oficio global definitivo SAT' : 'numero_definitivo',
'Publicación página SAT definitivo' : 'fecha_definitivo',
'Número y fecha de oficio global definitivo DOF' : 'numero_def_dof',
'Publicación DOF definitivo' : 'fecha_def_dof',
'Número y fecha de oficio global de sentencia favorable SAT' : 'publicacion_dof',
'Publicación página SAT sentencia favorable' : 'numero_fav_sat',
'RFC': 'numero',
'Nombre del Contribuyente': 'razon_social',
'Situación del contribuyente': 'situacion',
'Número y fecha de oficio global definitivo SAT': 'numero_definitivo',
'Publicación página SAT definitivo': 'fecha_definitivo',
'Número y fecha de oficio global definitivo DOF': 'numero_def_dof',
'Publicación DOF definitivo': 'fecha_def_dof',
'Número y fecha de oficio global de sentencia favorable SAT': 'numero_fav_sat',
'Publicación página SAT sentencia favorable': 'publicacion_sat',
'Publicación DOF sentencia favorable': 'publicacion_dof',
}
REQUIRED_FIELD = [ 'publicacion_sat', 'numero', 'razon_social', 'situacion'] #Campos obligatorios
REQUIRED_FIELD = [ 'numero', 'razon_social', 'situacion'] #Campos obligatorios
@staticmethod
def filter_by_schema(raw_rows: List[Dict]) -> List[Dict]:
""" Filter only fields defined on schema"""
filtered = []
def map_rows(csv_rows: List[Dict]) -> List[Dict]:
mapped_rows = []
for row in raw_rows:
mapped_row = {}
for csv_field, schema_field in CSVMapper.FIELD_MAPPING.items():
if csv_field in row and row[csv_field]:
mapped_row[schema_field] = CSVMapper._clean_value(
row[csv_field], schema_field
)
if all(field in mapped_row for field in CSVMapper.REQUIRED_FIELD):
filtered.append(mapped_row)
else:
print(f"omited fields: {mapped_row}")
return filtered
for row in csv_rows:
mapped = {}
for csv_field, bd_field in CSVMapper.FIELD_MAPPING.items():
value = row.get(csv_field, '')
if bd_field == 'razon_social':
value = value.replace('\n', ' ').replace('\r', ' ')
if bd_field == 'situacion':
value = CSVMapper._normalize_situacion(value)
if 'fecha' in bd_field and value:
value = CSVMapper._parse_date(value)
mapped[bd_field] = value if value else None
missing = [f for f in CSVMapper.REQUIRED_FIELD if not mapped.get(f)]
if missing:
print(f"Fila omitida - faltan: {missing}")
continue
mapped_rows.append(mapped)
return mapped_rows
@staticmethod
def _clean_value(value: str, field_type: str) -> Any:
""" Cleanen and typified values"""
if not value or value.strip() == '':
def _normalize_situacion(value: str) -> str:
""" converts text of CVS to format ENUM"""
if not value:
return None
if 'Sentencia Favorable' in value:
return 'sentencia_favorable'
if 'Definitivo' in value:
return 'definitivo'
return value.lower()
@staticmethod
def _parse_date(value: str) -> Optional[date]:
"""Converts DD/MM/YYYY to date"""
if not value:
return None
try:
return datetime.strptime(value.strip(), '%dd%m%Y').date()
except:
return None
if 'date' in field_type:
from datetime import datetime
return datetime.strptime(value.strip(), '%Y-%m-%d')
elif 'amount' in field_type or 'price' in field_type:
return float(value.strip().replace(',','.'))
elif 'int' in field_type.lower():
return int(value.strip())
else:
return value.strip()

25
app/helpers/db_adapter.py
View File

@@ -1,15 +1,24 @@
#helper/db_adapter
from typing import List, Dict
from database import sessionLocal
from app.modules.edos.models import EDOS
class DBAdapter:
""" Adapter: insert cleen data on a db"""
""" Adapter: insert cleen data on a db"""
def __inti__(self):
def __init__(self):
self.db = sessionLocal()
def insert_filtered_data(self, mapped_rows: List[Dict]) -> Dict:
""" Insert only data filtered and mapped"""
print(f" [DBAdapter] Recibidas {len(mapped_rows)} filas")
if mapped_rows:
print(f" [DBAdapter] Primera fila: {mapped_rows[0]}")
results = {
'total_processed': len(mapped_rows),
'inserted': 0,
@@ -19,8 +28,11 @@ class DBAdapter:
for row in mapped_rows:
try:
#verified dupled data
unique_field_value = row.get('numero')
existing = self.db.query(EDOS).filter_by(
unique_field=row.get('unique_field')
numero=unique_field_value
).first()
if existing:
@@ -28,8 +40,11 @@ class DBAdapter:
continue
instance = EDOS(**row)
self.db.add(instance)
print(f" [DBAdapter] Added instance for {row.get('numero')}")
self.db.commit()
print(f" [DBAdapter] Committed successfully")
results['inserted'] += 1
except Exception as e:
self.db.rollback()
@@ -37,6 +52,6 @@ class DBAdapter:
'row': row,
'error': str(e)
})
self.db.close()
return results
self.db.close()
return results

89
app/helpers/extractCsv.py
View File

@@ -1,3 +1,5 @@
#helper/csv_mapper
import csv
import requests
from io import StringIO
@@ -5,6 +7,10 @@ from io import StringIO
from typing import List, Dict, Any
import os
from dotenv import load_dotenv
#
import pandas as pd
import math
load_dotenv()
@@ -17,6 +23,21 @@ class Settings:
settings = Settings()
def _find_header_row(self, rows, delimiter):
""" Search row with RFC an name"""
for idx, row in enumerate(rows):
#Normalice: Convert to String, clean space, mayus
clean_row = [str(cell).strip().upper() for cell in row]
has_rfc = any('RFC' in cell for cell in clean_row)
has_nombre =any('NOMBRE' in cell for cell in clean_row)
if has_rfc and has_nombre:
return idx
return None
class CSVExtractor:
"""Extract raw data of CSV"""
@@ -36,46 +57,40 @@ class CSVExtractor:
@staticmethod
def read_csv(content: str) -> List[Dict[str, Any]]:
def read_csv(content):
""" Converts CSV to list of dictionary"""
csv_file = StringIO(content)
#detect delimiter auto
sample = csv_file.read(1024)
csv_file.seek(0)
try:
dialect = csv.Sniffer().sniff(sample)
reader =csv.DictReader(csv_file, dialect=dialect)
except:
csv_file.seek(0)
reader = csv.DictReader(csv_file)
rows = [row for row in reader]
for delimiter in[',', ';', '\t']:
if rows:
clean_row = {}
try:
lines = content.split('\n')
for key, value in rows[0].items():
header_row_idx = None
for idx, line in enumerate(lines):
if 'RFC' in line.upper():
header_row_idx = idx
break
if header_row_idx is None:
continue
if key is not None:
clean_key = key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
else:
clean_key = ''
clean_row[clean_key] = value
df = pd.read_csv(
StringIO(content),
delimiter=delimiter,
encoding='utf-8',
skiprows=header_row_idx,
dtype=str,
keep_default_na=False,
na_filter=False)
for i, row in enumerate(rows):
new_row = {}
for old_key, value in row.items():
if old_key is not None:
new_key = old_key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
else:
new_key = ''
#FIX: manage value None for avoid error strip
if value is not None and isinstance(value, str):
new_value = value.strip()
else:
new_value = value
df.columns = df.columns.str.strip()
new_row[new_key] = value
rows[i] = new_row
return rows
df = df.dropna(how='all')
result = df.to_dict('records')
if result:
return result
except Exception as e:
print(f"Error with delimiter '{delimiter}': {e}")
continue
raise ValueError("Don't read CSV with pandas")

BIN
app/modules/edos/__pycache__/models.cpython-311.pyc
View File

Binary file not shown.

BIN
app/modules/edos/__pycache__/route.cpython-311.pyc
View File

Binary file not shown.

BIN
app/modules/edos/__pycache__/schema.cpython-311.pyc
View File

Binary file not shown.

BIN
app/modules/edos/__pycache__/service.cpython-311.pyc
View File

Binary file not shown.

4
app/modules/edos/models.py
View File

@@ -16,12 +16,12 @@ class EDOS(Base):
__tablename__="edos"
id = Column(Integer, primary_key=True, nullable=False)
numero = Column(Integer, nullable=False)
numero = Column(String(25), nullable=False)
razon_social = Column(String(100), nullable=False)
situacion = Column(SQLEnum(Situacion, name="situcion", create_type=False), nullable=False)
numero_definitivo = Column(String(60), nullable=False)
fecha_definitivo = Column(Date, nullable=True )
publicaccion_sat = Column(Date, nullable=True)
publicacion_sat = Column(Date, nullable=True)
numero_def_dof = Column(String(100), nullable=True)
fecha_def_dof = Column(Date, nullable=True)
publicacion_dof = Column(Date, nullable=True)

3
app/modules/edos/route.py
View File

@@ -13,7 +13,7 @@ async def import_csv(data: Dict[str, Any] = Body(..., example={"url": "http://ex
csv_url = data.get('url')
dry_run = data.get('dry_run', True)
dry_run = data.get('dry_run', False)
max_rows = data.get('max_rows')
if not csv_url:
@@ -26,6 +26,7 @@ async def import_csv(data: Dict[str, Any] = Body(..., example={"url": "http://ex
)
try:
result = XMACSVService.process_csv_from_url(
csv_url= csv_url,
dry_run=dry_run,

2
app/modules/edos/schema.py
View File

@@ -16,7 +16,7 @@ class EdosUpload(BaseModel):
situacion : Situacion
numero_definitivo : str
fecha_definitivo : date
publicaccion_sat : date
publicacion_sat : date
numero_def_dof : str
fecha_def_dof : date
publicacion_dof : date

76
app/modules/edos/service.py
View File

@@ -4,71 +4,95 @@ from typing import Optional
from fastapi import HTTPException
from app.modules.edos.models import EDOS
from app.modules.edos.schema import EdosResponse, EdosUpload, messageResponse
from app.helpers.csv_mapper import CSVMapper
from app.helpers.db_adapter import DBAdapter
from app.helpers.extractCsv import CSVExtractor
class XMACSVService:
""" Orquester"""
"""Orchestrator: extract, map, adapt pipeline"""
@staticmethod
def process_csv_from_url(
csv_url: str,
dry_run: bool = False,
max_rows: int = None
) -> dict:
""" flow download and parsing, filter and adapt"""
) -> dict:
"""Flow: download CSV, parse, map to schema, insert into database"""
try:
# X - EXTRACT
print(f"📥 Descargando CSV: {csv_url}")
# =========================================================
# X - EXTRACT: Download and parse CSV
# =========================================================
print(f"Downloading CSV: {csv_url}")
# ✅ Ahora download_csv retorna (content, encoding)
content, encoding = CSVExtractor.download_csv(csv_url)
print(f" ✅ Descargado {len(content)} bytes, encoding: {encoding}")
print(f" Downloaded {len(content)} bytes, encoding: {encoding}")
# Debug: show first lines
print("First 10 lines of CSV:")
lines = content.split('\n')
for i, line in enumerate(lines[:10]):
print(f"Line {i}: {repr(line[:200])}")
# Parsear CSV
# Parse CSV to raw rows
raw_rows = CSVExtractor.read_csv(content)
print(f" ✅ Parseadas {len(raw_rows)} filas")
print(f" Parsed {len(raw_rows)} rows")
if not raw_rows:
return {
'total_extracted': 0,
'total_mapped': 0,
'inserted': 0,
'errors': ['No se encontraron datos en el CSV']
'dry_run': dry_run,
'errors': ['No data found in CSV']
}
# Mostrar columnas encontradas
print(f" 📋 Columnas: {list(raw_rows[0].keys())}")
print(f" Columns: {list(raw_rows[0].keys())}")
# M - MAP (si tienes mapper)
# Por ahora, usar datos crudos
mapped_rows = raw_rows
# =========================================================
# M - MAP: Transform CSV rows to database schema
# =========================================================
mapped_rows = CSVMapper.map_rows(raw_rows)
print(f" Mapped {len(mapped_rows)} rows to database schema")
if max_rows:
mapped_rows = mapped_rows[:max_rows]
print(f" 🔒 Limitado a {max_rows} filas")
print(f" Limited to {max_rows} rows")
# =========================================================
# A - ADAPT: Insert into database
# =========================================================
inserted_count = 0
errors = []
# A - ADAPT
if not dry_run:
print(f" 💾 Insertando {len(mapped_rows)} filas en BD")
# Aquí iría la inserción en BD
print(f" Inserting {len(mapped_rows)} rows into database")
adapter = DBAdapter()
result = adapter.insert_filtered_data(mapped_rows)
inserted_count = result.get('inserted', 0)
errors = result.get('errors', [])
print(f" Inserted: {inserted_count}")
print(f" Duplicates skipped: {result.get('duplicates_skiped', 0)}")
if errors:
print(f" Errors: {len(errors)}")
else:
print(f" DRY RUN - No data inserted")
# =========================================================
# Return response
# =========================================================
return {
'total_extracted': len(raw_rows),
'total_mapped': len(mapped_rows),
'inserted': len(mapped_rows) if not dry_run else 0,
'inserted': inserted_count,
'dry_run': dry_run,
'columns': list(raw_rows[0].keys()) if raw_rows else [],
'sample': raw_rows[:2] if raw_rows else []
'sample': mapped_rows[:2] if mapped_rows else [],
'errors': errors[:10] if errors else [] # Limit errors in response
}
except Exception as e:
print(f"❌ Error: {str(e)}")
print(f"Error: {str(e)}")
import traceback
traceback.print_exc()
raise
raise