From 092a7048d847bec9d6af28ae9d55a0305a884e34 Mon Sep 17 00:00:00 2001 From: gerardoe Date: Thu, 9 Apr 2026 14:03:06 -0700 Subject: [PATCH] full adjustment --- README.md | 113 +++++++++++++++++- .../__pycache__/csv_mapper.cpython-311.pyc | Bin 3523 -> 3861 bytes .../__pycache__/db_adapter.cpython-311.pyc | Bin 2459 -> 2957 bytes .../__pycache__/extractCsv.cpython-311.pyc | Bin 4222 -> 4689 bytes app/helpers/csv_mapper.py | 106 +++++++++------- app/helpers/db_adapter.py | 25 +++- app/helpers/extractCsv.py | 89 ++++++++------ .../edos/__pycache__/models.cpython-311.pyc | Bin 2098 -> 2127 bytes .../edos/__pycache__/route.cpython-311.pyc | Bin 3447 -> 3447 bytes .../edos/__pycache__/schema.cpython-311.pyc | Bin 1848 -> 1847 bytes .../edos/__pycache__/service.cpython-311.pyc | Bin 3374 -> 4763 bytes app/modules/edos/models.py | 4 +- app/modules/edos/route.py | 3 +- app/modules/edos/schema.py | 2 +- app/modules/edos/service.py | 76 ++++++++---- .../versions/876421e4eea2_adjustmen_of_bds.py | 54 +++++++++ ...421e4eea2_adjustmen_of_bds.cpython-311.pyc | Bin 0 -> 3140 bytes ...2733a7f_adjustmen_of_bdsv2.cpython-311.pyc | Bin 0 -> 1718 bytes .../d91de687dac7_full_bd.cpython-311.pyc | Bin 0 -> 1074 bytes .../c242c2733a7f_adjustmen_of_bdsv2.py | 32 +++++ migrations/versions/d91de687dac7_full_bd.py | 30 +++++ 21 files changed, 416 insertions(+), 118 deletions(-) create mode 100644 migrations/versions/876421e4eea2_adjustmen_of_bds.py create mode 100644 migrations/versions/__pycache__/876421e4eea2_adjustmen_of_bds.cpython-311.pyc create mode 100644 migrations/versions/__pycache__/c242c2733a7f_adjustmen_of_bdsv2.cpython-311.pyc create mode 100644 migrations/versions/__pycache__/d91de687dac7_full_bd.cpython-311.pyc create mode 100644 migrations/versions/c242c2733a7f_adjustmen_of_bdsv2.py create mode 100644 migrations/versions/d91de687dac7_full_bd.py diff --git a/README.md b/README.md index c711f08..b5be315 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,115 @@ una vez importado el model, lo lee, lo pasa por la logica en la que se descratan `notas de Gerardo` se quedaron implementadas los servicios crud de las funciones que seran publicos, ajustados a el tipo de usuario ('Root', 'Admin') -lo mas dificil de implementar para el back va a ser, implementar las funciones de ajuste "automatico" de cargado de files, con un url, desde las listas, del sat y las publicaciones del diario. +lo mas dificil de implementar para el back va a ser, implementar las funciones de ajuste "automatico" de cargado de file con un url, desde las listas del sat y las publicaciones del diario. -si el tiempo nos cae encima (que es lo mas seguro, voy a concentrarme en trabajar en las listas de el sat(edos, efos y creditos), para tratar de dejar enpoints funcionales). \ No newline at end of file +si el tiempo nos cae encima (que es lo mas seguro, voy a concentrarme en trabajar en las listas de el sat(edos, efos y creditos), para tratar de dejar enpoints funcionales). + + +# Flujo del tomado automatico de las listas del sat +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ PIPELINE DE TRANSFORMACIÓN │ +│ (CSV → JSONB → Base de Datos) │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + + ┌─────────────────┐ + │ SAT ORIGEN │ + │ archivo.csv │ + └────────┬────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ LECTOR POR LOTES (chunks) │ +│ ┌──────────────────────┐ │ +│ │ Bloque 1: filas 1-100│ │ +│ │ Bloque 2: filas 101-200│ │ +│ │ ... │ │ +│ │ Bloque N: últimas │ │ +│ └──────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ TRANSFORMADOR POR BLOQUE │ +├─────────────────────────────────────────────────────────────────────────────────────┤ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ 1. Agrupar │───▶│ 2. Limpiar │───▶│ 3. Armar │───▶│ 4. Validar │ │ +│ │ por RFC │ │ fechas │ │ JSONB │ │ esquema │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐ │ +│ │ FUNCIONES REUTILIZABLES │ │ +│ │ • normalize_date() → convierte números Excel y strings a ISO │ │ +│ │ • get_latest_status() → identifica el estatus más reciente │ │ +│ │ • build_jsonb() → construye la estructura final │ │ +│ └─────────────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ OUTPUT: JSONB POR RFC │ +├─────────────────────────────────────────────────────────────────────────────────────┤ +│ { │ +│ "rfc": "ASA110718HJ5", │ +│ "nombre": "ASESORÍA Y SERVICIOS ACANTO, S.A. DE C.V.", │ +│ "ultimo_estatus": { │ +│ "situacion": "definitivo", │ +│ "fecha": "2020-06-24", │ +│ "oficio": "500-05-2020-13598" │ +│ }, │ +│ "historial": [...], │ +│ "raw_metadata": {...} │ +│ } │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ REPOSITORIO / SERVICE LAYER │ +├─────────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐ │ +│ │ interface EfosRepository │ │ +│ │ ┌─────────────────────┐ ┌─────────────────────┐ │ │ +│ │ │ save_batch() │ │ find_by_rfc() │ │ │ +│ │ │ update_status() │ │ search_by_name() │ │ │ +│ │ │ get_statistics() │ │ count_by_status() │ │ │ +│ │ └─────────────────────┘ └─────────────────────┘ │ │ +│ └─────────────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌─────────────────┼─────────────────┐ │ +│ ▼ ▼ ▼ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ PostgreSQL │ │ SQLite │ │ MongoDB │ │ +│ │ (producción) │ │ (dev/local) │ │ (alternativa│ │ +│ └─────────────┘ └─────────────┘ │ JSON nativo)│ │ +│ └─────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ FRONT-END API (FastAPI) │ +├─────────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ GET /contribuyentes/{rfc} → consulta por RFC │ +│ GET /contribuyentes?q={nombre} → búsqueda por nombre │ +│ GET /contribuyentes?status={x} → filtrar por situación │ +│ GET /stats → contar por status (rojo/amarillo/verde) │ +│ │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + + +# 09/04/2026 - queda pendiente; +- revisar, verificar y tabajar todos los cruds. +- los que trabajan completamente bien, son los de usuarios y los de clients. +- la bd, funciona correctamente y las migraciones estan en su top. +- en este punto el endpoint de tomado automatico de las listas del sat, ya funcionan con el url del CSV. falta ajustar que detecte en que bd escriba, por el momento solo funciona con EDOS. +@ @ +* Nota importante * +- !para hacer chunks del cargado de la lista de Efos(articulo69-b) usar una libreria y no hacerle caso al ENFERMO DE HUGO, para hacerlo a mano! + + +urls importantes: +https://wu1agsprosta001.blob.core.windows.net/agsc-publicaciones/Datos_abiertos/Documents_AGAFF/Listado_completo_69-B.csv +* https://wu1agsprosta001.blob.core.windows.net/agsc-publicaciones/Datos_abiertos/Documents_AGGC/Listado_69_B_Bis_Completo.csv + + * https://www.sat.gob.mx/minisitio/DatosAbiertos/contribuyentes_publicados.html#collapseTwo1 diff --git a/app/helpers/__pycache__/csv_mapper.cpython-311.pyc b/app/helpers/__pycache__/csv_mapper.cpython-311.pyc index 2ce6224aeb82830a3661e94eca5368deefcfcec6..a34a42432c8163cb1de173456d8978e4d1ee4ff2 100644 GIT binary patch literal 3861 zcmb7HUu@gP89!2@{@G6C(2gBj?bybSBR8>=#t96^)#_Tdm!)>o*y#|p0YU31hZ!kq zN6Lv6dGJH9B}LG=4?~QuL)D^7U1Y!v>?PRC+GW6AB}14A0s#yR!?2e@MfQ;SY2Q(n zWZCY}olYM=-uJstcfY&)zS9>?O#uYucmLW@?t=C=`cf(GZRObvC=U=um{F0Af@ zv6?I9(%dmOL)%%E(>yVcqq|hE=8O3>Z_LXe7EK`RzJoCLkVWVbe9I-~$DU~v_WqN; zhTXRNO~o*6cT|a+wreE0VEf)mnTnp2)G%WQ#;|T&NvXPo!>sMbk}2ChDosU`pOhK=#VM zBUO+1VEuc-je~dG55fMZii!E5U$YzlYyj+s_ShS$_C~p><{Pm59_#_!0<5&!{?X}o zCZ$wLCYceTtPBYWMKxt2NJ%VcU{jKFf}T_tgrppo4T&rWK#6ohn3W6}gGG=8BYs`h zB-@irYckR8Mj~bOq-f}IMN)0Qp_pkY4kX*nWq^p~gpyQDWnQ;~2|0dU+SziT1`@HJ zu$!s~=e1%^Nb{m0nfAd{dRA59&JXNu00-fK?YcZZnrWZXwOJwySXPBmJ!ukUHoYJx zO*tI`lBdgqeDeFG1BllF`T;QYnHfx2Y$e&M8TW8h2pR|F7@pTjS}@-M*o5r_b}>ZM=d(r=NsX5`ayeNVBT!jQzxJ z|2xONPp5R(?hbT7`vAi((g;w}M8yFRwl8kXi$uR^*d8LAX_AaH&|E1VP+ahR=7tmf zbMFH*hZdPEilS@(S!5OTzSpcE;d!IlGgm$TJItv5Jb{nDz6ZEP*P=Vi=IF2NDOne0 z>f^~Gmvv+I$Buu}o8@wq*j3k;Mu7T$--6?I;~f8!&2mYIXZOIB(OtYQda@qE zo6QwFLfD`6{0jY=tsg0a?th(Sa{d~wrrq=I!EW5&$}$kM0#kN*`)o0REk@X4n#%i# zb53R5=Ane~@O>5m@k@qoo-Dd{WRbbY=AdP7$ujjnyVjOvK4kBAOrtLmqzBiWY=&Jn z+jq)l61IoNMj0D);|Rji){Fh{?tHJyDg;!cpYCi2MEX(-TY+H}QcqmhQjyHDYS`@q zfV2JkhE+W-sYV2p3XX;bBMjpC63TbXlxwc}tHfV!{OLyi?aNyW9a!3dV2u)me)A)t4H>2X@$fZjcrY3AoF=fplFHw`1$SGBd z%XW*H)QKjkN=6oI0lYy>NyLyvCp8|pJofgL3zx^DqT|ozQbb9bwukbP#9<$4qaGW| zJeAM53r@Q2r^F-_S;e+*7H@0ZyriaO+p8&t0U3wTh+_Ca2<7rR>89R%6}~cu_Hx)k z`<#Qb0DNs8Zok{G$#<5}VgD66KiCRe-GT!qj;F*L9d z8eR_#Kc4yOOd&K;3{5PLQS|L~zBkYJJ`Htmgof5bLyylELL%ogoX#ITpF5j7yJjwXmTwh;!e(Uyz+(SKDISZ!v(Pb9?3h{h zT6{+dvCS_$Z8?(fcx`P01k^%Hq}US4w?wR#&}Oi6Blz-q@a5dOLa?_O?9Ds1)I+D< zi-MhbUkB*{{>T7`x?@CRofCt92+9LELI&J(rr|D@MG$A;wgb9*Tj!?1o&((8QXhTc zBzrPl0;Jh_nV5!P%C}6&r^4tv(}JlB3A*=8VQlKkAuBNH+N}96yYIYH;xyuej zt2x*X)bcT$T~5_+%H&JNnVvzzrQmER@(QS#7i)WIzimGV7~^#iOQ_^Q!PXLD{OuOs zwme`p2)VI*Ll~;XAJ6m0t%kOhv-t)As>OeAdDuGG@{8N=-@dnae=+Y1ISkj4OVfu# z?gyaM$n_($8q#+rHX!#qq&Cd%Nf|JXXsOM|VUO4aU+T2kqtBSI(3WJaU7p(a$4}S`*bC0I0Q=IiUGx3U1^cbAkS` z*uf&r)(&@e8zhjv^0F9xGW;d$EC0VBUJ9YBS*Dlz+C}e{Z@AnsLVf^NdWMV-L6lq! q!vL|&$visozpI7%^7Yk1NAvsDlIIPEff?>Ezj50C{Vi69jsF6|SnJII literal 3523 zcmb7H|8LvI6+eoiELoDj$(H5V>6j*Nqu7x%BwL)sO`6(HV<&O!cqvdd0YU31hZ#j` zN6ML188Eg0bB6xl1|95nDXIot>LEp8VEbvnfC2djBtw`90tNyM1G?W7xC09Rw0C4I zOLhu)r{m+hyZ7GRz2p1b)7QaZJ%aN1^MA^HK7{^78{5VGlG%J4m?eY}W)zfS;!MgF zccs`k%g{cT;!bgKjzKQ;F2d}6gxybE2t9?zDRB?xCQ;P$AKHwvmiwx#8?W7ww-*-QO@uwA;xDWJNq_y&T&hklFF(kr^D&|u#CacrDrYf_%DkUUcB(ppg4Y8M> z5_Ji~2ru$_;*OLOt$_0ulCq>=t1+FKQsjh~kTq4%MZ@x_nUq8{D?r37?a~sms8~K- zHZr!5)$9leSW3#OY{)a3)s&PHcf{(Rh8-aRYe_3uk+7RJv(oWx`J^~QR}uAFjR}25 zNN7qXr3$K;lCtcW7PHtJmj<(uF)cMkBpyo&Kd7k&k*6}Vl4?kqE^zRqv(jgOQEh=l zjX)m=j6M4s-622A+lvAzUQ5d00DfB0rbGogws*!)Ui?MoFzAeLdx@G&Kl^f8Rz*5k zC7bzwZ4X_)loC$Z9Pxdv8U8(kIA2oMtuM1X+}A)Kanm9&j9TmXCUg7y7N>rIF#QL&CL-Bjo~4|*04Nu&nmXKG?7&!>LFoj&g<+z!WIkB zVEO2#?I{WUcE~{>Iz4%wmbb}kRHte8uCoVz+#~JO!Um8Z=26*$TEh$8=b_Frs`s6> z$--PY=rGh3c4g$7 z24FV&`q%r0*ZPLdNaym%qlT5h3)Xca{5=A)IJ%rGw)XxPCT*|w@85l;{&DCih9^ou zLldRYL>aNoXU*2GrRxu`=SPdJeWljEf?faJ+EF;rzcLC0YO(c9sr5|3uH|OBz!uaL zS!g6YbqwdYzutDBV9chaL^UE*P~}^QEc(HcNDk#eac`-`49xt}wnJRlH3`{5jset& z(R*BBQp}u-K{?l}v1gz5J+-+9%uz?@(B0T;&-F8-ZfA}Avzgz{gLV{wqZvo-T(!sm zGy4H+D!5dwTlPZb z+>oD0gdP;+W?tGbWVq$M-R~doP95*Y5P>NzqZ)PuCZsKipPquDR@LwVLWAKf7=tr+kS2=~c z!fJ<$CFVF-k4f;CB<9?OgyNm!lv+n^c?TTv^JEA_Xqur<11Yl%!^{lBiuaNiwD0>@4A`6cAJ!d?Qvd(} diff --git a/app/helpers/__pycache__/db_adapter.cpython-311.pyc b/app/helpers/__pycache__/db_adapter.cpython-311.pyc index 89113103b6d3e46ef54aa74acf34df7b52cf87c2..2ee5f47b5dbd8f751b9ee7b840cda9ea177b1247 100644 GIT binary patch delta 1464 zcmaJ=Z)jUp6uc z^SkGq-#c$Te6d%#F3Uc^*2Lex7@v#ZDxbipVP!Jqu9@VtmOXFjWHgg83a-sZ@4%yx zG6%pFU_lquS{k%C8bYzTb)cbZ+!xGs^Pw988~%*1uW;MAgK0ScSL28?3t4+TiKSsK?)Iu4SRO(f)3m|4M3y zZAdJzoBqKmGZn!WNXSN-XBjP{y%IHVSq31-<Vv`G;FYfK7${?isIIno0E+rrKeu)6n}*bhj$BDJOR z@1_dN@*WWz0s1bAyk_<9LG2DYHRb9fpYt@(f)(j4=@MK!+R~Qb5?JO=gC*FORM}11 z#UBTW5dDTfkSs@4Rek;8G@emk(R0SUp=C|AEUF7eK5Ler=m>d<7>hc|HofC2{h6P9 zTP&2AshEHy!fGcyO#JQ+X?d^%JFRKDrWys)$`*3Ex=mc`=CQZA>P z=0Yi-Um;=lWtY_is&k~cR2OtYisaarATb)O3+IVZuZ+*?1An9epmzq|4}_eq zg!A!?pNf z2M^Lku@Ci+P%6H9>H+TlNV?*`?Em;k6~}8h?%?=++~Y)!Z9R8;?7Oi$&sQhrY7=wJ zz;+d$n^Y~9eZ+g!49%3)JlV+BfXmAih b#u^-;!`_KQ&%oXWcvNh=wm%=^XcqPl&;VBl delta 945 zcmaKpUr19?9LLW&_ny1EKX*>&Cfm(9Ybq<+OGtxMnhvT5=YtYR)R5hq+>&eEUD=Q= zdeqPiPhh2#NMuohLJvJ=_MVGTV-WPxLr=~qh@Lt}Nm>7X2R^@Zf4_UazwbSFx?$e! zT$Usopn;d(X$SF|^Dgw6S~59a7w?WKLsv}Ih$NGm&T1lg3D1a&2!J_YLehl%tqL3l z3RKVnpGP?u2e;t)_B>|=IdBOwRThhgY$1;<>^?C*qP?g4eV%ObUbKkG7U!wus+-Sv zQp1JF8{XNI=X1PKn?nlDqAc?5$tl9P1VD;Qp$=M`8Z|9@Euh#nkmDZ0n=p^5C(^++ zd2_gW1l(~;R=5Bi>tn1kAd9{SYt`d7UO>2wl)Z z4p@A$sAjdasp-RIQPb1rke*V@P))OIC90)cveQlaKGnYtz6s=qrLEm-mA3)KL{9~| zCHF_UY2o;T&U>9J^#wUnlq2uu=vz5jY7Q+oEHyk16q=*O<|w%!o^36;>Tg@-?6dYe zrwgu@qN`=o)v{Z2&N>UOV9^!abOnjSI!$J*c;GvY3dWL~We1RIr%uiE-sqj|o9dhB zE4lp>3AQRnsh0{flnzp_^RdG{Ui3TQq)0-q^N_ssxo$zIJVSOmenYfVym&`Cdy?tDY50QEILarGrP1b zf(+Qe2oyjcOku!G;vj5MI1b_u7ia;0^rJ@p3o<3(#R39^76F=%0(6}MuF#LZSsoJQ zBrwo7yKnB9_h#NTpE{j(1n|$l-i@7y(%(s?0c^c+|4&f3ibNz*G78snDojD0mgzVX zW+>9e$ZVVob8$Y*$1P!t*+$9Muoe1P*%r5l?e(^{u)|!-8Ftc0vR{YQKX2@bC`6-^ zNaQXfk-ttu5BxO%&{Gq(Evo{p)S!7jw2n!(H`HlVvrRs=C3Dw>qI7_IMXW_-PJKMjSeNJ51FFfCGH2B>1; z=PoqHz`2M_D;?J<-~|2}ge~7PhZVW+onsYw;KTwPNSX#LsD?ADBvxbRQ&`YqO2Tl5 zR$hrLoPBlq^x64^$V*FS7SF!?($|BqrQsr7XXamvtSp5WBWIrdnq2yF&}LX8k%SPJ zA`!zLiNqB#C6lrv5_u~n$n_pe1P&w$%}gwz8MLAr{CSBS5N?B2uoKW{=ssYyJNSf< zOq&0iBqt@TJ|SJwun^Tk>cz?A2DXt21a-l$z5poKOgif+u^!z&SU6hPDBx=wrQlEg z+*KZ0s0=NX{fiaw8+*~Pj5}HAlwTbplXb&VS@t{ zOB!4%38L{CZP1fX8f;XOWtFTK2w*Q{yY^Wb)H15XlXK~@$Z9MhM%E-jlyC&tot%}G zs35CzP-u)Af?4V@Ksi+MhHBp}?c?skOCR{I`92x?aH#Bitm1p@uJ6e&eNUEsPgQ(R zH3ypF(sE|xW{<*upJ>kWV2`enQ?q74-!s%j^a{F4QE1C%+7;Sh;OLi4E-@M3!ijD6 zyp)hGCGlK(^m`a-3>qf$e*owokboE89DFwx3gg9Ng~w~P{rLZ-oL@eB=Ghk)zk_%@ z2828k*&~GPi9NFYp^`sz%Uw8igDv$RhVp^Wq7fl zSG$8-OOTg+a}~h7bCuq?&6C?+e~s1KKFG`dPz7*p-cNU2y;Vdx{e>Ab(>;B8J4p|8 zSNEn?yGkIZ2}E%T+&=}V)!j&}NK2f^NZcwbvX|LbwF~pm56<&ItEKKd9N-8RYbII2 zX-~Zvz`}Y!6f_~AtOj6&!KAQk@L1APIMJdU79htoqkl8$vZ4K^PQ zER~@i#c)udW*3^~HA#s~mWCDt$hsPu`H_*pHv5t)rfPd zLxycN$seJQ>DHEaD3QyuZ?o&vI=aN1L+ezKPrCyP%6fu?1>|JN0loa#A+P#Ref?HMD7Tl|9EQo@1M*bcg${WBg0U zc-b*gaZHpP6FZ)visy$V&(Rwvp}TkRN__LA-rZL?Rq7rqImRArySyOiwiA13yUSnd zKmNy~x8C~w^dF|72w%Btw$e3Qa?Cyes4_4fa+NsncTUVBbO+hzCz(6;q4^+lC&)p$ zMHb23o3iKy_W|K41&;eaS;_|Hx(HnlgLvS+vk<0bnf;DAb6^ClIAn$bY?c$*4EHGV zdl7-lX0r^164X{8!{n%~(;Cs6I_ydEep$_+R$0t|EM^#u$XOjkUgU3DuER>7H&@H@ z8U8}EwikdFi6MLlT5Upxk2Ggy_#%nQ9s<#N)7DBC*|E4!>-MII_5&j=X7FOE2QPmC zZ6(1#9Y9-$vR2L4s3Rn{WvGvk=*U<>GY{x0h0ehPBWr!zy7jMGzNe!a(T_Ta&Q==U zp@Qw{(LhK^T$He;ni{Mr0TMb8O%@^Y2MrVOMv&R14)0ROcPZO0WnadlupoXK&~rf^ zA0ib~->mAJv83sqI8ob3N`~uANlHdUX;nzc8idk@Eg^u%kxjQH$THXnW{r3&EJYP8 zs_EXEg)pPGriTkmq?w7CK)P>1Nla(~4B?JRsyQktBt$_CwnO-olw%s6Ak+*1GU%j8 z3<5k@K*cdp1=Cg#t01T12^IH~QM`ziBv5A2ln7xM&yiLx3C|$Su)iY6DaoWMXv5Et zB?z|$C&v3I#C4B}ht`ZNS0^b;P6{$pk>aMpQ%3%`Vr+ZpVPlMd! zWXfycFo@YfzF-vvY(5ihE>+nd-xOGo|k^R*ETl;SuzjKs4hvCzMhYRhxcc8#kIplr3YDMmmQg={y z_kY#nx$^UEcYop4vU|MZ9@hsa^no#b;Lvv8fIb$~2S=+eOYitS1n3)r@^rNe4F;}- zul?fgz>zNpj+6(ERtApd=_~e~9?#Wtzdl!-%AYIu1S>tkJgxiupY(m$S6nLlCM&+l zJg;{Hb5!Su?jO#dAqlKq=g0#HKHLLUL_49kfwnt(O1;nA*Z>6I?H6i!xnsW4F<*knhKa{KVDEFf(0@UnkG?=Ys zxK!J-%V3D?{}{G4=$K+Efx&}nf+xi+A&t02%oF4$gZSFp9uw1N@HO4SKO+@3E@*4! zgQeDQ2X&VkpN`scW+Hn!a&5nIUUuf3Ri2?{K{Xyi_ek3}dmVP< laBn{S-ptla&RXRil)H+WXmFZ>OL!RFBW>S&ONV(n{{tXsVe9|^ delta 2203 zcmbtVT}%{L6uvXFyEC&tu)r>?6zw>y(9r@KN}~*n7B43T?hdaZA;S^64SJO^Qqk-ld_q_grsTHmqv_jVoB5V-T}d-zV&A3 zn{&@O_nhyZd+w|e{)a1>@%h{UjdMT#Bu}uJk`OVSsqe~MY-sV2K=Mo@(;aNL$8j$S zI)O-E1|l;}A`5y}Ek2Rx1p@okZ)t+?>gP$Z@~H{GnbpbTgg!~Pv`J2pl4yyMoDw@o zZd&tPmj)f8>#}1*cDG2Q9L(1b>30a?f_{KK5wa*ng@O~lfMQq-l$6ntgd#y+zsGhG zRXXLqTMO%vElR8<0lpXisC?gt85TRi)`F!QE=p}f&+3K#AKJ`UqSCB=8 z@~A~aX*8CKL?w%r5@T@*MpVgi%Zi*()JP&K!EGpk;J2@>#h};&gK|u=TuLN98k67- zoZf}gqsR`teT$98q)1`_$rd#%owuB4BC#FC z7TfHViXZuM8-`t8{E1m9;)URb;;yLc&`RYm^x?P&{&ayrO}5jh2%8n*ab~e>(~Zra zY|g4iS=cN~g~we5#(&Ls#g~p4Ovq$Hc_y?(I@!htm7A^)XX4l63zforrEt5>sN7{% z?i%lW5DZ;El{+vyln=HU!8SA4Hokv+|C1$>WE%_Kz_sox-Px^%x5o6=g-M3g#nciFAYG>FT%Ix#rpSxpISRHM!OWu4A6-_^$UpZ*biv z*S)~?&2xSEqo?wx-!-_X$wkv$>8?dC#UreLIn8DT!ynH3!@pG5<~nXR-D+9_B|hGU zj4Mr~n+w%7X}VA+%*d1S#PHPc67ZBEMhG&8)1BGALe-Xh)%F4}%rs0kOf*e3E%1Bh z`8{*BbI{=Tnf$&5{@^@+@P2dt?IQ-?Yx2Dd{E7J#eNZ|nx&@7}R*8+BOlz*3}wxUpav0^RQqwBkmG0tWP10MLm6|3|<> z80@Q#FQMIeuo_Dk0_agL={^4aJ=>juZ0!<2uUt5%-fo|*xP4&iLz92i)~&-M3b@KI z_b-vib6soSa66<>0$XtS7xnx8mUhS@* List[Dict]: - """ Filter only fields defined on schema""" - filtered = [] + def map_rows(csv_rows: List[Dict]) -> List[Dict]: + mapped_rows = [] - for row in raw_rows: - mapped_row = {} - for csv_field, schema_field in CSVMapper.FIELD_MAPPING.items(): - if csv_field in row and row[csv_field]: - mapped_row[schema_field] = CSVMapper._clean_value( - row[csv_field], schema_field - ) - if all(field in mapped_row for field in CSVMapper.REQUIRED_FIELD): - filtered.append(mapped_row) - else: - print(f"omited fields: {mapped_row}") - return filtered - + for row in csv_rows: + mapped = {} + + for csv_field, bd_field in CSVMapper.FIELD_MAPPING.items(): + value = row.get(csv_field, '') + + if bd_field == 'razon_social': + value = value.replace('\n', ' ').replace('\r', ' ') + + if bd_field == 'situacion': + value = CSVMapper._normalize_situacion(value) + + if 'fecha' in bd_field and value: + value = CSVMapper._parse_date(value) + + mapped[bd_field] = value if value else None + + missing = [f for f in CSVMapper.REQUIRED_FIELD if not mapped.get(f)] + if missing: + print(f"Fila omitida - faltan: {missing}") + continue + + mapped_rows.append(mapped) + + return mapped_rows + @staticmethod - def _clean_value(value: str, field_type: str) -> Any: - """ Cleanen and typified values""" - if not value or value.strip() == '': + def _normalize_situacion(value: str) -> str: + """ converts text of CVS to format ENUM""" + if not value: + return None + if 'Sentencia Favorable' in value: + return 'sentencia_favorable' + if 'Definitivo' in value: + return 'definitivo' + return value.lower() + @staticmethod + def _parse_date(value: str) -> Optional[date]: + """Converts DD/MM/YYYY to date""" + if not value: + return None + try: + return datetime.strptime(value.strip(), '%dd%m%Y').date() + except: return None - - if 'date' in field_type: - from datetime import datetime - return datetime.strptime(value.strip(), '%Y-%m-%d') - elif 'amount' in field_type or 'price' in field_type: - return float(value.strip().replace(',','.')) - elif 'int' in field_type.lower(): - return int(value.strip()) - else: - return value.strip() - \ No newline at end of file diff --git a/app/helpers/db_adapter.py b/app/helpers/db_adapter.py index e81fbba..984e415 100644 --- a/app/helpers/db_adapter.py +++ b/app/helpers/db_adapter.py @@ -1,15 +1,24 @@ +#helper/db_adapter + from typing import List, Dict from database import sessionLocal from app.modules.edos.models import EDOS class DBAdapter: - """ Adapter: insert cleen data on a db""" + """ Adapter: insert cleen data on a db""" - def __inti__(self): + def __init__(self): self.db = sessionLocal() def insert_filtered_data(self, mapped_rows: List[Dict]) -> Dict: """ Insert only data filtered and mapped""" + print(f" [DBAdapter] Recibidas {len(mapped_rows)} filas") + + if mapped_rows: + print(f" [DBAdapter] Primera fila: {mapped_rows[0]}") + + + results = { 'total_processed': len(mapped_rows), 'inserted': 0, @@ -19,8 +28,11 @@ class DBAdapter: for row in mapped_rows: try: #verified dupled data + + unique_field_value = row.get('numero') + existing = self.db.query(EDOS).filter_by( - unique_field=row.get('unique_field') + numero=unique_field_value ).first() if existing: @@ -28,8 +40,11 @@ class DBAdapter: continue instance = EDOS(**row) + self.db.add(instance) + print(f" [DBAdapter] Added instance for {row.get('numero')}") self.db.commit() + print(f" [DBAdapter] Committed successfully") results['inserted'] += 1 except Exception as e: self.db.rollback() @@ -37,6 +52,6 @@ class DBAdapter: 'row': row, 'error': str(e) }) - self.db.close() - return results + self.db.close() + return results \ No newline at end of file diff --git a/app/helpers/extractCsv.py b/app/helpers/extractCsv.py index 04b3007..afa5931 100644 --- a/app/helpers/extractCsv.py +++ b/app/helpers/extractCsv.py @@ -1,3 +1,5 @@ +#helper/csv_mapper + import csv import requests from io import StringIO @@ -5,6 +7,10 @@ from io import StringIO from typing import List, Dict, Any import os from dotenv import load_dotenv +# +import pandas as pd +import math + load_dotenv() @@ -17,6 +23,21 @@ class Settings: settings = Settings() +def _find_header_row(self, rows, delimiter): + """ Search row with RFC an name""" + + for idx, row in enumerate(rows): + #Normalice: Convert to String, clean space, mayus + clean_row = [str(cell).strip().upper() for cell in row] + + has_rfc = any('RFC' in cell for cell in clean_row) + has_nombre =any('NOMBRE' in cell for cell in clean_row) + + if has_rfc and has_nombre: + return idx + return None + + class CSVExtractor: """Extract raw data of CSV""" @@ -36,46 +57,40 @@ class CSVExtractor: @staticmethod - def read_csv(content: str) -> List[Dict[str, Any]]: + def read_csv(content): """ Converts CSV to list of dictionary""" - csv_file = StringIO(content) - #detect delimiter auto - sample = csv_file.read(1024) - csv_file.seek(0) - try: - dialect = csv.Sniffer().sniff(sample) - reader =csv.DictReader(csv_file, dialect=dialect) - except: - csv_file.seek(0) - reader = csv.DictReader(csv_file) - - rows = [row for row in reader] + for delimiter in[',', ';', '\t']: - if rows: - clean_row = {} + try: + lines = content.split('\n') - for key, value in rows[0].items(): + header_row_idx = None + for idx, line in enumerate(lines): + if 'RFC' in line.upper(): + header_row_idx = idx + break + + if header_row_idx is None: + continue - if key is not None: - clean_key = key.strip().replace('"', '').replace("'", "").replace('\ufeff', '') - else: - clean_key = '' - - clean_row[clean_key] = value + df = pd.read_csv( + StringIO(content), + delimiter=delimiter, + encoding='utf-8', + skiprows=header_row_idx, + dtype=str, + keep_default_na=False, + na_filter=False) - for i, row in enumerate(rows): - new_row = {} - for old_key, value in row.items(): - if old_key is not None: - new_key = old_key.strip().replace('"', '').replace("'", "").replace('\ufeff', '') - else: - new_key = '' - #FIX: manage value None for avoid error strip - if value is not None and isinstance(value, str): - new_value = value.strip() - else: - new_value = value + df.columns = df.columns.str.strip() - new_row[new_key] = value - rows[i] = new_row - return rows + df = df.dropna(how='all') + result = df.to_dict('records') + if result: + return result + + except Exception as e: + print(f"Error with delimiter '{delimiter}': {e}") + continue + raise ValueError("Don't read CSV with pandas") + \ No newline at end of file diff --git a/app/modules/edos/__pycache__/models.cpython-311.pyc b/app/modules/edos/__pycache__/models.cpython-311.pyc index 509dd083e522526f5b21b433326cdaf81f31da5e..8ce73e4e5b69e94c6e3d41efd9865aed37ea75e2 100644 GIT binary patch delta 439 zcmdlaa9)6SIWI340}#CCy^&eBkvD>gamnN~CPNn{hE(%_lm9ZSv%ZuBdUdi2OFVB8 zE0A@IEw3~;wJ3k`1ePEU{({n^oXq6J$*)<`7moGG|_8{*9y@Ky!sb; i^{?>iPwrtYpX|Y=KiQaVGNbn7<80ZCR+Ej{{{sNci&Dz~ delta 415 zcmX>vut|V-IWI340}$+-do8nOBX0x~Acg>t+R3+>q{Z2RERJQ&Ksg{rlH+9-_vA?7#MFgm124#iRK64@47>Re zGLv(`rlGq5Q;#4*51Ktf2${)enZ>m+eS~S6ND5algC_T6eirq~)-3V7x7hMZb5o1* zi&%jkoxGeSh*O}TG$|)DIWc)MBWoI?+T;vYJs$T3sTX*4F7oPJ;nkVEinUxsTy{d) qjFc527sQP&iW^-KH@d(gGP#0HW^w@AWJayYPuQ{S diff --git a/app/modules/edos/__pycache__/route.cpython-311.pyc b/app/modules/edos/__pycache__/route.cpython-311.pyc index 4abc7574baf76dcbea4fb03776a12c83320433e1..4694fad0901edf480cad667a2107cf3b63da196f 100644 GIT binary patch delta 111 zcmew^^<9d0IWI340}z~HyOH^NBX0={qub^tmP?H6su%e*uJCD0j%JHx5o6_gx_J-V zMP^3R&3#;DOa|(VTpt*K#0--K%nO1SaLlm-a}mTxW(Fq3D~t*sm_SNHCY$g&0|0OT BAh`el delta 111 zcmew^^<9d0IWI340}w=>y_Wf4BX0={W60(vmP?H6Di`_Gukfi)j%JHx5oP7NyLk`W zMP^2m&3#;DOa^L7&-6`tkK@^49z6fIe#Ev;->3L`~QB8gIDB~5HgPHb6DCA&$86a>Xt%Z!&l zo?S(@Ko&vZ0zTxR76t4yGRhQo!XPOS^wJyy^wJ_d>>7)Dv48;sMGiidfomYhsc&{g z(aXr_rOU_JH}B27_r15XGjBf&2K@-Gf3kYMJB-l3Xs3C(c9=)M1Lg+82s0uoFlmON zyhC&p9BGHiJ4I)~m3EoDOLQ05G;8v1(NplIy(Z6!zJfpPF9gzo0+;4YnMVv3Lg|pn zd&Q1II2|^5pBO1b(@_RF&`E^-*ANbT>OklV_*+-H6LTvl8GJ~)NyhN32$Ga96@m5Q ztSrd+f7lP#A8LOUmpy^6DC+3awjnZF{FY)&qb+zw9jW@Z?;N3TG0 z0|_Y2V3c-XChZhlKIB8#hh2gjy9E}rf(Ls9?*%XRUUPp6&JcHv|Ua&BX@P%x7gwIlZd^ukh#C%a$STa00>AlQ45e*L} zS2JY2Xm|_RD;ZMSkPMax@;WKzoK2T-55S-HA01#Rcps9PmKHM=^f9{S_!MMc?8+G= zKVjkB-G^2|_Hplkrs}M^Y!)4|r)9t5w9oh$VQ0k!9$j`hc--}l5dMj_;;c0nbIUkz z*lo8{b;~r~ovY%;EcX1`(bE*P;Xi?XIcCPdRy{WV-UY6 zo)vT$$-S*7J%Gqd)sQXrKDoc8uY~Ne?v;Ib_gOJg?WlCrqOF-N&b~ew^Z~+bHT0{{ z2D5>#INwGaOp@Pxc|j~~%<{OjQKaW54~NtUU(OOqFi#_1F7Xl^RfX&bpNF%S$Z!K0 z9N5{_tR!qkPVCIg7uV?IS$;Fb^L$e(V16^ouU?e}X@nPwxe|tPFs5rEPb8Utew?Rg zoWz$dn3I2G{7Vn%*}Lgp1UbLynB^b-jrRRu^bHCN47z!G^bvYUZN`=?F#LtmdJzLS zYA5M7I^>d^6*D_0tAGv862J*thMw07M!nT3fN$!Y@sX)7A}uzd6rLh8tyUy^n~th2G?9D8K_ANrzjK+w*+uy!zUEi z3j!20g5e@UnHXLoWU)ESCFUjBa9t9vN`#&gw0tR>jU}Qc@LSF@_m`X)iD9VPnx#kSFxzG$I zZvdR40PwOKRtXib)`ZmwHB*+!!Ge;_WEdW^Bohd&wxo4YHW-17f**;xg~Z#gANes@ zW9Gk5!t0_SjR{bnj={@AYroO*RU=j=r5x0$G&`9Kq*S2Q-R2=%jX1j37*|9DA6-_ zCTbg>%-x;?7U-K9>#lN38n>iyOOIL8}4JH_rU_Ol?4o4Qa8V>+|;$ z1GV$@iTbhn#O<`27}pZx4a5Ye&E_|rEoN)-et4hKHFYx!4bYv*x~qP!o~xhx(ytz# z&<;;nay6XN!YRchaJzp%>mI4Usdi6k-BZ_JGto}ml4>&_Rky23biyK)o7A{Tg`0fP znP?zSF#2HV8GYc0&iCtmgL>bT(wk}o*!|H4vYJTraoCLL=$%;YV*T>%OZNs-cL!7I z;EXmnqxQe3^}nb@j{>ROoW{*5+}vY7N(^Z6k@{O|Jf+1`*H4>RUI#2k)xI&U4^sWQ zMJjhn<4!5usRte14de*U=)K8$vOZH!YQ2+N0W%M&+7+cMWsw@5*22?Dc)H<7y?xi0 z^nKme7anxR8z>N*F-M=ildBJ_gDGtgrku7&4KHfpMJ2p=KYsY5;qBoqkKV)I>zTUS zGxgOawda`Db8L&%`-eV@|1n-a^);jR&uaa%TOqxt?+@|c$7`?ESJj^9wVvm<*d0X0 zHaMpE+*j;h{D1ZDieowY-?&h6ow!(S~oI=rz^16fTl^uyyA3Ikr34;DkSF`wkXt>ZQ3qlN7S zB{~44az`}oh{7GwxdTd*bZ*~0uJ10_r*i!o*ROE>Iv4o?u7(e6!CmTuxABAHlc!zi zuhSm_%=JI-0+|hV4 z8C=Ryu_R~Ze6AqK7fYBt2TlyvYN;d|PS}PG7tZHoG6&Mn(IaRPIZB(3Wr*0dV2N!2O6Fg#tk{4tsaRroRoPG+1GHJau#)V?F{iEf2iSWjke}| z??i0$+B*$LyMin(TpcZu0-;}+gnrZ+tbDZ$rz3IGj^{!aA};3% zOMBh0d5-REoTGH(q#Z$&2(1M7&|1|A@q(7nj~YVD2cwHxxhRP6(JzhCUDn*=iFN$g zlz26>v%6b|kxjW%J{*_cGmk$j%k$KW87R&CkKQ>8?aSmWbZK=h{TrHwlVKPg9a8M1 zql1c_bd*r+q@#YtPC9xSG7Kz>z1#0i`G20GH52k* D351w3 delta 1677 zcma)6Z)h837=JIxT`sxHziYa7Yg*qfEot;R>$>bumvl7QW?jd0E$W2HID2Oa=AU;L zvz4nI;$Wa3#$Lg}T9-nrk!>=+$o55sFh42W$)TZ8s1p%E@Ww2Ne(}A;cB>Qd?#b_c z-{<$d&+~iU_sHemPqz6#^7*^~KMP-dn;r0e>i>*jyzp81^|GX>l8k;O1{oUDPW-!H za-x36JUZcE2^YHV-1;P`687HaBf3=g`Wyg_{19_jH~7myB+>V-mK3SbBGq`^Q-ny* zxhBA-EFP74IEzSa-B?yBbesyKm)#L`iwX~D zF9l)>h=Dn30z`r8-#81<1LV|<8+E!m2sgS)ZD;YBdUKu`TBEDpzt71(OMth4=$&!D z>8=nJaN7A2s1WhcG!I|R6)R9IRtmYIAVQ%a!jd2>*}@d2vN?S94WA$iCAGW-SGBI7 zC3sL$CIxv4M;Bpc;)Nkt<}d{zygPSpgT&l42VXg(N(zr|(adN$^dRUtTtvcI6*x`V zg#<-*V0OCuSP?1`lnRr@f+`Dgu>eIuEh^XprJP+X?n1qu`0O$z$#PLv)+=LO@4!qk zSI!p%We6TCw=|1cSKhk_k7n~(RltG-_`rYhbNEa_kz}>e#x&cMVHZjt{Mi;0r{q>V z$adRun+^R=r-eb1$wHJ-HgxQujKib%~e`&JTH@lEu%y`j%vE z6hqUDV9E@p^kB-OKo|=md`F2!(2?tz8;rpxO&&)cX_^rnGlOG#aBPhRJ+bQPh16nd z1xBl51{^TqKz-MKJ(8Z!R2d_}n-N}*@V7(t4)nD@*8BKQecRBbzUt2ENOk8Ne~G{8 zuzc>Y-vUkW`F~&IP=BDUy>H1`9a*3jDV^)XX0UOSjq7Y2oeT_irs+dFz!ywrJ09-r z@E%S$@9f=sc)$sTqGU0g-FWM+udOW19Lt>_OrnJ2GNgVTq2Pou!=|Fp6&bX JUx-HbzW^fH!S4V7 diff --git a/app/modules/edos/models.py b/app/modules/edos/models.py index 7079d9e..d384f81 100644 --- a/app/modules/edos/models.py +++ b/app/modules/edos/models.py @@ -16,12 +16,12 @@ class EDOS(Base): __tablename__="edos" id = Column(Integer, primary_key=True, nullable=False) - numero = Column(Integer, nullable=False) + numero = Column(String(25), nullable=False) razon_social = Column(String(100), nullable=False) situacion = Column(SQLEnum(Situacion, name="situcion", create_type=False), nullable=False) numero_definitivo = Column(String(60), nullable=False) fecha_definitivo = Column(Date, nullable=True ) - publicaccion_sat = Column(Date, nullable=True) + publicacion_sat = Column(Date, nullable=True) numero_def_dof = Column(String(100), nullable=True) fecha_def_dof = Column(Date, nullable=True) publicacion_dof = Column(Date, nullable=True) diff --git a/app/modules/edos/route.py b/app/modules/edos/route.py index 1acb973..7849f43 100644 --- a/app/modules/edos/route.py +++ b/app/modules/edos/route.py @@ -13,7 +13,7 @@ async def import_csv(data: Dict[str, Any] = Body(..., example={"url": "http://ex csv_url = data.get('url') - dry_run = data.get('dry_run', True) + dry_run = data.get('dry_run', False) max_rows = data.get('max_rows') if not csv_url: @@ -26,6 +26,7 @@ async def import_csv(data: Dict[str, Any] = Body(..., example={"url": "http://ex ) try: + result = XMACSVService.process_csv_from_url( csv_url= csv_url, dry_run=dry_run, diff --git a/app/modules/edos/schema.py b/app/modules/edos/schema.py index b751146..07060ca 100644 --- a/app/modules/edos/schema.py +++ b/app/modules/edos/schema.py @@ -16,7 +16,7 @@ class EdosUpload(BaseModel): situacion : Situacion numero_definitivo : str fecha_definitivo : date - publicaccion_sat : date + publicacion_sat : date numero_def_dof : str fecha_def_dof : date publicacion_dof : date diff --git a/app/modules/edos/service.py b/app/modules/edos/service.py index a3ca75d..ad03f16 100644 --- a/app/modules/edos/service.py +++ b/app/modules/edos/service.py @@ -4,71 +4,95 @@ from typing import Optional from fastapi import HTTPException from app.modules.edos.models import EDOS from app.modules.edos.schema import EdosResponse, EdosUpload, messageResponse - - from app.helpers.csv_mapper import CSVMapper from app.helpers.db_adapter import DBAdapter from app.helpers.extractCsv import CSVExtractor class XMACSVService: - """ Orquester""" + """Orchestrator: extract, map, adapt pipeline""" @staticmethod def process_csv_from_url( csv_url: str, dry_run: bool = False, max_rows: int = None - ) -> dict: - """ flow download and parsing, filter and adapt""" - + ) -> dict: + """Flow: download CSV, parse, map to schema, insert into database""" try: - # X - EXTRACT - print(f"📥 Descargando CSV: {csv_url}") + # ========================================================= + # X - EXTRACT: Download and parse CSV + # ========================================================= + print(f"Downloading CSV: {csv_url}") - # ✅ Ahora download_csv retorna (content, encoding) content, encoding = CSVExtractor.download_csv(csv_url) - print(f" ✅ Descargado {len(content)} bytes, encoding: {encoding}") + print(f" Downloaded {len(content)} bytes, encoding: {encoding}") + + # Debug: show first lines + print("First 10 lines of CSV:") + lines = content.split('\n') + for i, line in enumerate(lines[:10]): + print(f"Line {i}: {repr(line[:200])}") - # Parsear CSV + # Parse CSV to raw rows raw_rows = CSVExtractor.read_csv(content) - print(f" ✅ Parseadas {len(raw_rows)} filas") + print(f" Parsed {len(raw_rows)} rows") if not raw_rows: return { 'total_extracted': 0, 'total_mapped': 0, 'inserted': 0, - 'errors': ['No se encontraron datos en el CSV'] + 'dry_run': dry_run, + 'errors': ['No data found in CSV'] } - # Mostrar columnas encontradas - print(f" 📋 Columnas: {list(raw_rows[0].keys())}") + print(f" Columns: {list(raw_rows[0].keys())}") - # M - MAP (si tienes mapper) - # Por ahora, usar datos crudos - mapped_rows = raw_rows + # ========================================================= + # M - MAP: Transform CSV rows to database schema + # ========================================================= + mapped_rows = CSVMapper.map_rows(raw_rows) + print(f" Mapped {len(mapped_rows)} rows to database schema") if max_rows: mapped_rows = mapped_rows[:max_rows] - print(f" 🔒 Limitado a {max_rows} filas") + print(f" Limited to {max_rows} rows") + + # ========================================================= + # A - ADAPT: Insert into database + # ========================================================= + inserted_count = 0 + errors = [] - # A - ADAPT if not dry_run: - print(f" 💾 Insertando {len(mapped_rows)} filas en BD") - # Aquí iría la inserción en BD + print(f" Inserting {len(mapped_rows)} rows into database") + adapter = DBAdapter() + result = adapter.insert_filtered_data(mapped_rows) + inserted_count = result.get('inserted', 0) + errors = result.get('errors', []) + print(f" Inserted: {inserted_count}") + print(f" Duplicates skipped: {result.get('duplicates_skiped', 0)}") + if errors: + print(f" Errors: {len(errors)}") + else: + print(f" DRY RUN - No data inserted") + # ========================================================= + # Return response + # ========================================================= return { 'total_extracted': len(raw_rows), 'total_mapped': len(mapped_rows), - 'inserted': len(mapped_rows) if not dry_run else 0, + 'inserted': inserted_count, 'dry_run': dry_run, 'columns': list(raw_rows[0].keys()) if raw_rows else [], - 'sample': raw_rows[:2] if raw_rows else [] + 'sample': mapped_rows[:2] if mapped_rows else [], + 'errors': errors[:10] if errors else [] # Limit errors in response } except Exception as e: - print(f"❌ Error: {str(e)}") + print(f"Error: {str(e)}") import traceback traceback.print_exc() - raise + raise \ No newline at end of file diff --git a/migrations/versions/876421e4eea2_adjustmen_of_bds.py b/migrations/versions/876421e4eea2_adjustmen_of_bds.py new file mode 100644 index 0000000..69c7736 --- /dev/null +++ b/migrations/versions/876421e4eea2_adjustmen_of_bds.py @@ -0,0 +1,54 @@ +"""adjustmen of BDs + +Revision ID: 876421e4eea2 +Revises: c01f58ebc7d4 +Create Date: 2026-04-09 20:28:03.853032 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '876421e4eea2' +down_revision: Union[str, None] = 'c01f58ebc7d4' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('edos', 'numero', + existing_type=sa.INTEGER(), + type_=sa.String(length=25), + existing_nullable=False) + op.alter_column('edos', 'fecha_definitivo', + existing_type=sa.DATE(), + nullable=True) + op.alter_column('license', 'deleted_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True) + op.drop_column('location', 'country_id') + op.drop_column('location', 'city_id') + op.drop_column('location', 'state_id') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('location', sa.Column('state_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('location', sa.Column('city_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.add_column('location', sa.Column('country_id', sa.INTEGER(), autoincrement=False, nullable=False)) + op.alter_column('license', 'deleted_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False) + op.alter_column('edos', 'fecha_definitivo', + existing_type=sa.DATE(), + nullable=False) + op.alter_column('edos', 'numero', + existing_type=sa.String(length=25), + type_=sa.INTEGER(), + existing_nullable=False) + # ### end Alembic commands ### diff --git a/migrations/versions/__pycache__/876421e4eea2_adjustmen_of_bds.cpython-311.pyc b/migrations/versions/__pycache__/876421e4eea2_adjustmen_of_bds.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e21fabdbca765f869ea99b885ca129e3c8ec671e GIT binary patch literal 3140 zcmcgu&2JM&6d&8`ueBW~DJitYj`=9ykPpX6AXrgFA)!qL0g)gSE7I2XjGax_yJmJx z38GRC95}iXQiGIAoKm9Xf)u3w2|cb=AFNR$RjPX6W&~X5rEl!miQSsgLpz%N?d+R3 z@6GPK_j~(;&nF^iKYPmZFHVGh;}7*F><(Un2ZvVh}LF^9t))+!|Abcl1!b&qvAzM5(WuP0>nbmNc7A= zWON{M4u-Mlcq}q9I6gKK8HtMGbN;enD|5|~TX~|U2(#Z%!BN=4tXggcZemW;jSMCF zErogayJjxByDR41@0|It_O&WCCVfeoP*aPNl1LIoXCfv!qGDarROX<>$Wt}d_~I~d z!DWN%<+_ERt^rtYON}u=I$7wCY^fRAZ*(bxb@9Fd-g5II@sH6 z*tE3w>};~ySkdemySCix;Bm9zwXH=FdkkM=%V-JWz1F~nw`HIIWR-utYQsLA|)x-#5wrhq71Zz5TsQE0R+H;;48y4nJh?+4LFKyVEkKB=ULsm1A zv7Cb_<0!GNhE&Zf5FiR+ooSL%c}-mmSoJFDcPtFT)F|70iv**^7CWfgklgXr8Fg< zRhd;!Fz58l+|*}N*O_D1ps;NwOny2yMIBt*JvV)2YIg3^E7zC@Q!TfZsTT;~6^9Ck*Dhpt@}=hC zLB96pqxrS@@}a{OyVV|ifdF1{sBn1gsOe6x-rEVBECq&t95n;yi-Gf{!1=c>nJL6p z?-hmqlF)Ao{X4K%E?F)L;gS$Gg>YSRw1lYmrsQ{npql&z2;26_-;{uj z)Ni4LM&h+3rmdR(O{v)$)a^*8jbD3{{QvXo_)q*c2n{>2s|_)^x5O4=vTb?ZN_Tr! zz1Y{Z?*M^Uw5|5j)7KAZH~p43)XYcZ(OT=xv^1-tYpz$GY}zC_;8 zWHm(z)M|r%$mfo52yys`!(o6q8sJ0DiPX>G5Qk0E-Rq=CvEoj45Pz^468z&!XG0rxV)~Kjlpcn?X_xk`kbh zE(vK@T4Yzi`0zFPQuvI%g0>gD{@%lWC7 z^HXn~pIW%=?YcF}oPUxu1H;9@Ft8eD&iJ7%Q6rSB3ukDS0($XI=D7DgpFkr{o*l`*vBxr)v15rcLO8Bx`?~GGN>s@zt zonj=U9D2yn2c&W+Qcj^Ly&#nXQjs`t;JB82utq{cLL9hR%B?5f_%A>7&QE0P#1Hm--t zp@EKVwHzj)a)d^yM${Xyl7!1sRH0FVtMVJ;y%AU=11m1agq6xPdTa$><#+_X(UEWb z6@0aUZ-T@g;&Or{NF2UNf;Uhm`BmK{+fC1}Qzz|K)1Rz(cXdtsobJ@Tn(L%*tt_W4 zy`WoqX<@-ERfiVpEvJ`Civ@k27AQ4!?Ixq9Ptz*^%V|BYFJ|+FZ2mf^%L}Dty_B0@ zS}GQFdku}|KPEc;dq?54;*7+j8~ZH zSoaLutWewI8les95YKQOt}^O3nPW||3`zt#FpG{303IP8O)PP&=Wso&*6;>KezCxE@b7niacd znHD6?@JxRbG8b{0Hk{q2e0pphn=>jCn{&-1#IRhuS$DYXnOwa&)Ulw9PZ8!eM&=*T zk@&|KAI_PL#$3I&#Y`V!d2>6IiIVWg7vMum7gdalvp;bsHY5z11q)kWyu zD!?xK8>s`Da;PpSibFIA?yqWFb zTo>npv9Q-+Cky(dML`pJK6(Q1fALB1)6fC1eR$0Ht1_y-l+o}SOThoKa{!-i!j*`d zP2a6K7Nc+%eKswONq`ZB?7WzGO~3^Jj#ob3TxFNU__6?z7A9T@9u>D^nEn|UzXjd9 z1#l+)i@(Uv!#_u#M}zgdfk}f@wUerL0r6HBZv}X3Lcs?eobBRlFqT2AjLc%N=~`x* zhYf?cmSJ$!Z#8NVTCl-HqIlS}X}wajMpfwDw@ur+N9(QONy5TM@qEUUhG9C6>ko^~ zFqmMKiKm!N2@r+E#Bt7?_C@4I-6c(%e#G7etGK_Ocx3vDBuRZ)m2e+TAc3T-0h;}< z{E03Fr=^E32V?0+(77wSkv&*D1JXe=z1jS3e6QKw`1V@+i*E8tfbt+6l<&G<%_0efa NC{j+$y%Yvy{0rLOjRpV! literal 0 HcmV?d00001 diff --git a/migrations/versions/__pycache__/d91de687dac7_full_bd.cpython-311.pyc b/migrations/versions/__pycache__/d91de687dac7_full_bd.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75fb94dbfa4dbf9c749602143b02658a69e060e7 GIT binary patch literal 1074 zcma)5J8u&~5Z-(Ee7^XR2LT~OVkHz9Wcz%1m{<`Fp`b~Dm4a(d=i4>0bPs3G4g?9w z1w~31NTHxe2kH0`{D8BR!rB5wqN1~uE)}!3^Dq>!n*DC(zM0vX*}E^Pl!jpanCmyb zi3t7V#^sX7%K3Ltj?fOm$V6DcqAPf!DSDDA3EY-k*^8O6p)R|M8OJeGB?+QoWfLNC z+?*m3Nnp7y-o4C?fK`XAY$iENM3T5>!B@Jrz&A1WP5pTTD}*083%B>Ro95%}2`Ta_%{C}Fbxq>OE+JYq;#$vVcW;TYwW6}w#5o>5}A zNOlXLk~MT=J+H6k^^L4is^~_=FbXAz-_SMf7w<7AGPO$f+QfGVi|zPuP)=ZC(A;Lp zn};!#j_Y8_o5!&f4&M0I6`870+jn*?*RByaWEv(-;^WW?e5O#+YE$328Z9USYzg+c z0&oNuS4Ahn_6edH{t`C2(2I7nS@asOsojF=!s0%md>&!(W_=c)msP`s<^hYho8XKI zorU;$o+u8`L1(hs*`m{+|EmU?2B%>SBj7P~27o1ayQAzp^F2ottKH z%ff+USxjjiG#e1XyQKU}WpUdjUajHK7`L==&$V4=mv{$E3~kCwWm4FpEMr-=?+2|> z+m=NWJO%#^=oAOu9py#*QR9WW=>@p$lBe`OuzbWJzw3b{2*N;A1bKk2AdQ6O2rd0B zXXth`DSdP&x|D&6W)?f@A;d0#^w2_oN$;c%+uiD?2i;evnZ*d{AU&k_OKW4|;ql7n z{MTF2eEu|3h|n5H53ThJMn^k*-d*m#J(fPcKgQAQqtoQ$2pJ$fWDFEhNDR; None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('edos', sa.Column('publicacion_sat', sa.Date(), nullable=True)) + op.drop_column('edos', 'publicaccion_sat') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('edos', sa.Column('publicaccion_sat', sa.DATE(), autoincrement=False, nullable=True)) + op.drop_column('edos', 'publicacion_sat') + # ### end Alembic commands ### diff --git a/migrations/versions/d91de687dac7_full_bd.py b/migrations/versions/d91de687dac7_full_bd.py new file mode 100644 index 0000000..0409931 --- /dev/null +++ b/migrations/versions/d91de687dac7_full_bd.py @@ -0,0 +1,30 @@ +"""full bd + +Revision ID: d91de687dac7 +Revises: c242c2733a7f +Create Date: 2026-04-09 21:02:22.168790 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd91de687dac7' +down_revision: Union[str, None] = 'c242c2733a7f' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ###