ADUANASOFT/Verificacion_EFOS_Backend
10
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

full adjustment

Browse Source
This commit is contained in:
gerardoe
2026-04-09 14:03:06 -07:00
parent 75ecb00a94
commit 092a7048d8
21 changed files with 416 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/helpers/__pycache__/csv_mapper.cpython-311.pyc
View File

Binary file not shown.

BIN
app/helpers/__pycache__/db_adapter.cpython-311.pyc
View File

Binary file not shown.

BIN
app/helpers/__pycache__/extractCsv.cpython-311.pyc
View File

Binary file not shown.

106
app/helpers/csv_mapper.py
View File

@@ -1,56 +1,74 @@
from typing import List, Dict, Any
#helpers/csv_mapper
from typing import List, Dict, Any, Optional
from app.modules.edos.schema import EdosUpload
from datetime import date, datetime
class CSVMapper:
""" Mapper: filter and mapping only necesary data, based on a schema"""
FIELD_MAPPING = {
'csv_column_name': 'schema_field',
'No.' : 'publicacion_sat',
'RFC' : 'numero',
'Nombre del Contribuyente' : 'razon_social',
'Situación del contribuyente' : 'situacion',
'Número y fecha de oficio global definitivo SAT' : 'numero_definitivo',
'Publicación página SAT definitivo' : 'fecha_definitivo',
'Número y fecha de oficio global definitivo DOF' : 'numero_def_dof',
'Publicación DOF definitivo' : 'fecha_def_dof',
'Número y fecha de oficio global de sentencia favorable SAT' : 'publicacion_dof',
'Publicación página SAT sentencia favorable' : 'numero_fav_sat',
'RFC': 'numero',
'Nombre del Contribuyente': 'razon_social',
'Situación del contribuyente': 'situacion',
'Número y fecha de oficio global definitivo SAT': 'numero_definitivo',
'Publicación página SAT definitivo': 'fecha_definitivo',
'Número y fecha de oficio global definitivo DOF': 'numero_def_dof',
'Publicación DOF definitivo': 'fecha_def_dof',
'Número y fecha de oficio global de sentencia favorable SAT': 'numero_fav_sat',
'Publicación página SAT sentencia favorable': 'publicacion_sat',
'Publicación DOF sentencia favorable': 'publicacion_dof',
}
REQUIRED_FIELD = [ 'publicacion_sat', 'numero', 'razon_social', 'situacion'] #Campos obligatorios
REQUIRED_FIELD = [ 'numero', 'razon_social', 'situacion'] #Campos obligatorios
@staticmethod
def filter_by_schema(raw_rows: List[Dict]) -> List[Dict]:
""" Filter only fields defined on schema"""
filtered = []
def map_rows(csv_rows: List[Dict]) -> List[Dict]:
mapped_rows = []
for row in raw_rows:
mapped_row = {}
for csv_field, schema_field in CSVMapper.FIELD_MAPPING.items():
if csv_field in row and row[csv_field]:
mapped_row[schema_field] = CSVMapper._clean_value(
row[csv_field], schema_field
)
if all(field in mapped_row for field in CSVMapper.REQUIRED_FIELD):
filtered.append(mapped_row)
else:
print(f"omited fields: {mapped_row}")
return filtered
for row in csv_rows:
mapped = {}
for csv_field, bd_field in CSVMapper.FIELD_MAPPING.items():
value = row.get(csv_field, '')
if bd_field == 'razon_social':
value = value.replace('\n', ' ').replace('\r', ' ')
if bd_field == 'situacion':
value = CSVMapper._normalize_situacion(value)
if 'fecha' in bd_field and value:
value = CSVMapper._parse_date(value)
mapped[bd_field] = value if value else None
missing = [f for f in CSVMapper.REQUIRED_FIELD if not mapped.get(f)]
if missing:
print(f"Fila omitida - faltan: {missing}")
continue
mapped_rows.append(mapped)
return mapped_rows
@staticmethod
def _clean_value(value: str, field_type: str) -> Any:
""" Cleanen and typified values"""
if not value or value.strip() == '':
def _normalize_situacion(value: str) -> str:
""" converts text of CVS to format ENUM"""
if not value:
return None
if 'Sentencia Favorable' in value:
return 'sentencia_favorable'
if 'Definitivo' in value:
return 'definitivo'
return value.lower()
@staticmethod
def _parse_date(value: str) -> Optional[date]:
"""Converts DD/MM/YYYY to date"""
if not value:
return None
try:
return datetime.strptime(value.strip(), '%dd%m%Y').date()
except:
return None
if 'date' in field_type:
from datetime import datetime
return datetime.strptime(value.strip(), '%Y-%m-%d')
elif 'amount' in field_type or 'price' in field_type:
return float(value.strip().replace(',','.'))
elif 'int' in field_type.lower():
return int(value.strip())
else:
return value.strip()

25
app/helpers/db_adapter.py
View File

@@ -1,15 +1,24 @@
#helper/db_adapter
from typing import List, Dict
from database import sessionLocal
from app.modules.edos.models import EDOS
class DBAdapter:
""" Adapter: insert cleen data on a db"""
""" Adapter: insert cleen data on a db"""
def __inti__(self):
def __init__(self):
self.db = sessionLocal()
def insert_filtered_data(self, mapped_rows: List[Dict]) -> Dict:
""" Insert only data filtered and mapped"""
print(f" [DBAdapter] Recibidas {len(mapped_rows)} filas")
if mapped_rows:
print(f" [DBAdapter] Primera fila: {mapped_rows[0]}")
results = {
'total_processed': len(mapped_rows),
'inserted': 0,
@@ -19,8 +28,11 @@ class DBAdapter:
for row in mapped_rows:
try:
#verified dupled data
unique_field_value = row.get('numero')
existing = self.db.query(EDOS).filter_by(
unique_field=row.get('unique_field')
numero=unique_field_value
).first()
if existing:
@@ -28,8 +40,11 @@ class DBAdapter:
continue
instance = EDOS(**row)
self.db.add(instance)
print(f" [DBAdapter] Added instance for {row.get('numero')}")
self.db.commit()
print(f" [DBAdapter] Committed successfully")
results['inserted'] += 1
except Exception as e:
self.db.rollback()
@@ -37,6 +52,6 @@ class DBAdapter:
'row': row,
'error': str(e)
})
self.db.close()
return results
self.db.close()
return results

89
app/helpers/extractCsv.py
View File

@@ -1,3 +1,5 @@
#helper/csv_mapper
import csv
import requests
from io import StringIO
@@ -5,6 +7,10 @@ from io import StringIO
from typing import List, Dict, Any
import os
from dotenv import load_dotenv
#
import pandas as pd
import math
load_dotenv()
@@ -17,6 +23,21 @@ class Settings:
settings = Settings()
def _find_header_row(self, rows, delimiter):
""" Search row with RFC an name"""
for idx, row in enumerate(rows):
#Normalice: Convert to String, clean space, mayus
clean_row = [str(cell).strip().upper() for cell in row]
has_rfc = any('RFC' in cell for cell in clean_row)
has_nombre =any('NOMBRE' in cell for cell in clean_row)
if has_rfc and has_nombre:
return idx
return None
class CSVExtractor:
"""Extract raw data of CSV"""
@@ -36,46 +57,40 @@ class CSVExtractor:
@staticmethod
def read_csv(content: str) -> List[Dict[str, Any]]:
def read_csv(content):
""" Converts CSV to list of dictionary"""
csv_file = StringIO(content)
#detect delimiter auto
sample = csv_file.read(1024)
csv_file.seek(0)
try:
dialect = csv.Sniffer().sniff(sample)
reader =csv.DictReader(csv_file, dialect=dialect)
except:
csv_file.seek(0)
reader = csv.DictReader(csv_file)
rows = [row for row in reader]
for delimiter in[',', ';', '\t']:
if rows:
clean_row = {}
try:
lines = content.split('\n')
for key, value in rows[0].items():
header_row_idx = None
for idx, line in enumerate(lines):
if 'RFC' in line.upper():
header_row_idx = idx
break
if header_row_idx is None:
continue
if key is not None:
clean_key = key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
else:
clean_key = ''
clean_row[clean_key] = value
df = pd.read_csv(
StringIO(content),
delimiter=delimiter,
encoding='utf-8',
skiprows=header_row_idx,
dtype=str,
keep_default_na=False,
na_filter=False)
for i, row in enumerate(rows):
new_row = {}
for old_key, value in row.items():
if old_key is not None:
new_key = old_key.strip().replace('"', '').replace("'", "").replace('\ufeff', '')
else:
new_key = ''
#FIX: manage value None for avoid error strip
if value is not None and isinstance(value, str):
new_value = value.strip()
else:
new_value = value
df.columns = df.columns.str.strip()
new_row[new_key] = value
rows[i] = new_row
return rows
df = df.dropna(how='all')
result = df.to_dict('records')
if result:
return result
except Exception as e:
print(f"Error with delimiter '{delimiter}': {e}")
continue
raise ValueError("Don't read CSV with pandas")