From 9255dbc6b24c16fe06f18dd580080dd2e2878391 Mon Sep 17 00:00:00 2001 From: Kevin_Ramirez Date: Thu, 26 Mar 2026 09:07:28 -0500 Subject: [PATCH 01/32] Se arreglo los filtros de facturas --- backend/api/v1/modules/a76/invoices/routes.py | 7 +- .../api/v1/modules/a76/invoices/services.py | 64 +++++++++++++------ 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/backend/api/v1/modules/a76/invoices/routes.py b/backend/api/v1/modules/a76/invoices/routes.py index fa6c888e..5fd67f89 100644 --- a/backend/api/v1/modules/a76/invoices/routes.py +++ b/backend/api/v1/modules/a76/invoices/routes.py @@ -95,6 +95,9 @@ def list_invoices( invoice_type: str = Query(None, description="Filter by invoice type"), manifest_number: str = Query(None, description="Filter by manifest number"), pedimento: str = Query(None, description="Filter by pedimento"), + invoice_number: str = Query(None, description="Filter by invoice number"), + project_number: str = Query(None, description="Filter by project number"), + year: str = Query(None, description="Filter by year"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -107,12 +110,14 @@ def list_invoices( skip = (page - 1) * page_size filters = { - "invoice_number": search, + "invoice_number": invoice_number or search, "status": status, "operation_type": operation_type, "invoice_type": invoice_type, "manifest_number": manifest_number, "pedimento": pedimento, + "project_number": project_number, + "year": year, } # Remove None values diff --git a/backend/api/v1/modules/a76/invoices/services.py b/backend/api/v1/modules/a76/invoices/services.py index a9871ad6..f9787d92 100644 --- a/backend/api/v1/modules/a76/invoices/services.py +++ b/backend/api/v1/modules/a76/invoices/services.py @@ -124,46 +124,70 @@ class InvoiceService: # Apply filters if provided if filters: + # Join compliance_mx if needed for filters + needs_compliance_join = any(k in filters for k in ["pedimento", "manifest_number"]) + if needs_compliance_join: + query = query.join(models.InvoiceComplianceMx) + if filters.get("status") is not None: - status = models.InvoiceStatus.PROCESSED if filters["status"] == True else models.InvoiceStatus.PENDING - + status = ( + models.InvoiceStatus.PROCESSED + if filters["status"] == True + else models.InvoiceStatus.PENDING + ) query = query.filter(models.InvoiceHeader.status == status) + if filters.get("operation_type"): ot = filters["operation_type"] ot_val = ot.value if hasattr(ot, "value") else ot - query = query.filter( - models.InvoiceHeader.operation_type == ot_val - ) + query = query.filter(models.InvoiceHeader.operation_type == ot_val) + if filters.get("invoice_type"): query = query.filter( models.InvoiceHeader.invoice_type == filters["invoice_type"] ) + if filters.get("invoice_number"): query = query.filter( models.InvoiceHeader.invoice_number.ilike( f"%{filters['invoice_number']}%" ) ) - if filters.get("pedimento"): - query = query.join(models.InvoiceComplianceMx).filter( - models.InvoiceComplianceMx.pedimento.ilike( - f"%{filters['pedimento']}%" + + if filters.get("project_number"): + query = query.filter( + models.InvoiceHeader.project_number.ilike( + f"%{filters['project_number']}%" ) ) + + if filters.get("year"): + try: + year_val = int(filters["year"]) + query = query.filter( + func.extract("year", models.InvoiceHeader.invoice_date) == year_val + ) + except (ValueError, TypeError): + pass + + if filters.get("pedimento"): + from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos + query = query.join(models.InvoiceComplianceMx.pedimento).filter( + Pedimentos.pedimento_number.ilike(f"%{filters['pedimento']}%") + ) + + if filters.get("manifest_number"): + query = query.filter( + models.InvoiceComplianceMx.manifest_number.ilike( + f"%{filters['manifest_number']}%" + ) + ) + + # Special case for exports: exclude REPAR if no invoice_type specified ot_exp = filters.get("operation_type") ot_exp_val = ot_exp.value if hasattr(ot_exp, "value") else ot_exp if not filters.get("invoice_type") and ot_exp_val == "exp": - query = query.filter(models.InvoiceHeader.operation_type != "REPAR") - - if filters.get("manifest_number"): - # Avoid duplicate joins if pedimento filter was also applied (though rare in this context) - # For safety, we can just use the relationship attribute directly if mapped, - # but explicit join is clearer given the previous pattern. - # Assuming SQLAlchemy handles the join overlap or we just accept it for now. - # To be safe and consistent with previous 'pedimento' block: - query = query.join(models.InvoiceComplianceMx).filter( - models.InvoiceComplianceMx.manifest_number.ilike(f"%{filters['manifest_number']}%") - ) + query = query.filter(models.InvoiceHeader.invoice_type != "REPAR") total = query.count() items = query.offset(skip).limit(limit).all() From b4772332450ca9b7a1390fadc9a4b7f033f68c2a Mon Sep 17 00:00:00 2001 From: hreyes Date: Wed, 25 Mar 2026 10:38:24 -0600 Subject: [PATCH 02/32] feature/migraciones-id-transportes --- ...2a_invoice_logistics_int_fk_constraints.py | 266 ++++++++++++++++++ .../a76/invoices/common/common_validators.py | 30 +- backend/api/v1/modules/a76/invoices/models.py | 9 + .../api/v1/modules/a76/invoices/schemas.py | 12 +- .../api/v1/modules/a76/invoices/services.py | 99 +++++++ .../modules/a76/layouts_csv/facturas/tasks.py | 24 +- .../importacion/consolidados/mex/service.py | 42 ++- .../consolidados/temporary/mex/service.py | 34 ++- .../importacion/facturas/mex/service.py | 46 ++- .../facturas/temporary/mex/service.py | 34 ++- .../importacion/facturas/usa/service.py | 50 +++- .../importacion/packing_list/service.py | 40 ++- .../modules/a76/transportation/drivers/dto.py | 1 + .../a76/transportation/drivers/models.py | 4 +- .../a76/transportation/trailers/dto.py | 1 + .../a76/transportation/trailers/models.py | 4 +- .../a76/transportation/transporters/dto.py | 1 + .../a76/transportation/transporters/models.py | 4 +- .../a76/transportation/vehicles/dto.py | 1 + .../a76/transportation/vehicles/models.py | 4 +- .../src/lib/api/dashboard/a76/invoices.ts | 3 + .../src/lib/api/dashboard/a76/trailers.ts | 1 + .../src/lib/api/dashboard/a76/transporters.ts | 1 + .../src/lib/api/dashboard/a76/vehicles.ts | 1 + 24 files changed, 649 insertions(+), 63 deletions(-) create mode 100644 backend/alembic/versions/ca7d3c4e8b2a_invoice_logistics_int_fk_constraints.py diff --git a/backend/alembic/versions/ca7d3c4e8b2a_invoice_logistics_int_fk_constraints.py b/backend/alembic/versions/ca7d3c4e8b2a_invoice_logistics_int_fk_constraints.py new file mode 100644 index 00000000..394da688 --- /dev/null +++ b/backend/alembic/versions/ca7d3c4e8b2a_invoice_logistics_int_fk_constraints.py @@ -0,0 +1,266 @@ +"""Add surrogate int IDs for transport catalogs and invoice logistics. + +Single consolidated migration for: +- transporter_id, trailer_id, vehicle_id, driver_id +- carrier_int_id, transport_int_id, trailer_int_id in invoice_logistics +- backfill mappings from existing string keys/codes +- FK constraints for the new invoice_logistics int references +""" + +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "ca7d3c4e8b2a" +down_revision: Union[str, Sequence[str], None] = "c1a2b3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # transporter + op.add_column("transporter", sa.Column("transporter_id", sa.BigInteger(), nullable=True), schema="a76") + op.execute( + """ + WITH s AS ( + SELECT transporter_key, ROW_NUMBER() OVER (ORDER BY transporter_key) AS new_id + FROM a76.transporter + ) + UPDATE a76.transporter t + SET transporter_id = s.new_id + FROM s + WHERE t.transporter_key = s.transporter_key + """ + ) + op.execute("CREATE SEQUENCE IF NOT EXISTS a76.transporter_transporter_id_seq") + op.execute( + """ + SELECT setval( + 'a76.transporter_transporter_id_seq', + COALESCE((SELECT MAX(transporter_id) FROM a76.transporter), 0) + 1, + false + ) + """ + ) + op.execute( + """ + ALTER TABLE a76.transporter + ALTER COLUMN transporter_id + SET DEFAULT nextval('a76.transporter_transporter_id_seq') + """ + ) + op.execute("ALTER TABLE a76.transporter ALTER COLUMN transporter_id SET NOT NULL") + op.create_unique_constraint("uq_a76_transporter_transporter_id", "transporter", ["transporter_id"], schema="a76") + + # trailer + op.add_column("trailer", sa.Column("trailer_id", sa.BigInteger(), nullable=True), schema="a76") + op.execute( + """ + WITH s AS ( + SELECT trailer_number, ROW_NUMBER() OVER (ORDER BY trailer_number) AS new_id + FROM a76.trailer + ) + UPDATE a76.trailer t + SET trailer_id = s.new_id + FROM s + WHERE t.trailer_number = s.trailer_number + """ + ) + op.execute("CREATE SEQUENCE IF NOT EXISTS a76.trailer_trailer_id_seq") + op.execute( + """ + SELECT setval( + 'a76.trailer_trailer_id_seq', + COALESCE((SELECT MAX(trailer_id) FROM a76.trailer), 0) + 1, + false + ) + """ + ) + op.execute( + """ + ALTER TABLE a76.trailer + ALTER COLUMN trailer_id + SET DEFAULT nextval('a76.trailer_trailer_id_seq') + """ + ) + op.execute("ALTER TABLE a76.trailer ALTER COLUMN trailer_id SET NOT NULL") + op.create_unique_constraint("uq_a76_trailer_trailer_id", "trailer", ["trailer_id"], schema="a76") + + # vehicle + op.add_column("vehicle", sa.Column("vehicle_id", sa.BigInteger(), nullable=True), schema="a76") + op.execute( + """ + WITH s AS ( + SELECT vehicle_key, ROW_NUMBER() OVER (ORDER BY vehicle_key) AS new_id + FROM a76.vehicle + ) + UPDATE a76.vehicle v + SET vehicle_id = s.new_id + FROM s + WHERE v.vehicle_key = s.vehicle_key + """ + ) + op.execute("CREATE SEQUENCE IF NOT EXISTS a76.vehicle_vehicle_id_seq") + op.execute( + """ + SELECT setval( + 'a76.vehicle_vehicle_id_seq', + COALESCE((SELECT MAX(vehicle_id) FROM a76.vehicle), 0) + 1, + false + ) + """ + ) + op.execute( + """ + ALTER TABLE a76.vehicle + ALTER COLUMN vehicle_id + SET DEFAULT nextval('a76.vehicle_vehicle_id_seq') + """ + ) + op.execute("ALTER TABLE a76.vehicle ALTER COLUMN vehicle_id SET NOT NULL") + op.create_unique_constraint("uq_a76_vehicle_vehicle_id", "vehicle", ["vehicle_id"], schema="a76") + + # driver + op.add_column("driver", sa.Column("driver_id", sa.BigInteger(), nullable=True), schema="a76") + op.execute( + """ + WITH s AS ( + SELECT transporter_key, line, ROW_NUMBER() OVER (ORDER BY transporter_key, line) AS new_id + FROM a76.driver + ) + UPDATE a76.driver d + SET driver_id = s.new_id + FROM s + WHERE d.transporter_key = s.transporter_key + AND d.line = s.line + """ + ) + op.execute("CREATE SEQUENCE IF NOT EXISTS a76.driver_driver_id_seq") + op.execute( + """ + SELECT setval( + 'a76.driver_driver_id_seq', + COALESCE((SELECT MAX(driver_id) FROM a76.driver), 0) + 1, + false + ) + """ + ) + op.execute( + """ + ALTER TABLE a76.driver + ALTER COLUMN driver_id + SET DEFAULT nextval('a76.driver_driver_id_seq') + """ + ) + op.execute("ALTER TABLE a76.driver ALTER COLUMN driver_id SET NOT NULL") + op.create_unique_constraint("uq_a76_driver_driver_id", "driver", ["driver_id"], schema="a76") + + # invoice_logistics int refs + op.add_column("invoice_logistics", sa.Column("carrier_int_id", sa.BigInteger(), nullable=True), schema="a76") + op.add_column("invoice_logistics", sa.Column("transport_int_id", sa.BigInteger(), nullable=True), schema="a76") + op.add_column("invoice_logistics", sa.Column("trailer_int_id", sa.BigInteger(), nullable=True), schema="a76") + + # ensure referenced codes exist + op.execute( + """ + INSERT INTO a76.transporter (transporter_key, tenant_id, company_id) + SELECT DISTINCT ON (il.carrier_id) il.carrier_id, il.tenant_id, il.company_id + FROM a76.invoice_logistics il + WHERE il.carrier_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM a76.transporter t WHERE t.transporter_key = il.carrier_id + ) + """ + ) + op.execute( + """ + INSERT INTO a76.vehicle (vehicle_key, tenant_id, company_id) + SELECT DISTINCT ON (il.transport_id) il.transport_id, il.tenant_id, il.company_id + FROM a76.invoice_logistics il + WHERE il.transport_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM a76.vehicle v WHERE v.vehicle_key = il.transport_id + ) + """ + ) + op.execute( + """ + INSERT INTO a76.trailer (trailer_number, tenant_id, company_id) + SELECT DISTINCT ON (il.trailer_num) il.trailer_num, il.tenant_id, il.company_id + FROM a76.invoice_logistics il + WHERE il.trailer_num IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM a76.trailer tr WHERE tr.trailer_number = il.trailer_num + ) + """ + ) + + # backfill int refs + op.execute( + """ + UPDATE a76.invoice_logistics il + SET carrier_int_id = t.transporter_id + FROM a76.transporter t + WHERE il.carrier_id IS NOT NULL + AND il.carrier_id = t.transporter_key + """ + ) + op.execute( + """ + UPDATE a76.invoice_logistics il + SET transport_int_id = v.vehicle_id + FROM a76.vehicle v + WHERE il.transport_id IS NOT NULL + AND il.transport_id = v.vehicle_key + """ + ) + op.execute( + """ + UPDATE a76.invoice_logistics il + SET trailer_int_id = tr.trailer_id + FROM a76.trailer tr + WHERE il.trailer_num IS NOT NULL + AND il.trailer_num = tr.trailer_number + """ + ) + + # FK constraints + op.create_foreign_key( + "fk_a76_invoice_logistics_carrier_int_id_transporter_id", + "invoice_logistics", + "transporter", + ["carrier_int_id"], + ["transporter_id"], + source_schema="a76", + referent_schema="a76", + ondelete="SET NULL", + ) + op.create_foreign_key( + "fk_a76_invoice_logistics_transport_int_id_vehicle_id", + "invoice_logistics", + "vehicle", + ["transport_int_id"], + ["vehicle_id"], + source_schema="a76", + referent_schema="a76", + ondelete="SET NULL", + ) + op.create_foreign_key( + "fk_a76_invoice_logistics_trailer_int_id_trailer_id", + "invoice_logistics", + "trailer", + ["trailer_int_id"], + ["trailer_id"], + source_schema="a76", + referent_schema="a76", + ondelete="SET NULL", + ) + + +def downgrade() -> None: + pass + diff --git a/backend/api/v1/modules/a76/invoices/common/common_validators.py b/backend/api/v1/modules/a76/invoices/common/common_validators.py index 3f7dc48b..558f6940 100644 --- a/backend/api/v1/modules/a76/invoices/common/common_validators.py +++ b/backend/api/v1/modules/a76/invoices/common/common_validators.py @@ -512,15 +512,29 @@ def validate_common( if invoice.logistics: if invoice.logistics.carrier_id: - carrier_exists = ( - db.query(Transporter) - .filter( - Transporter.id == invoice.logistics.carrier_id, - Transporter.tenant_id == tenant_id, - Transporter.company_id == company_id, + # `carrier_id` is the frontend/export key (string). Prefer validating by the internal int + # when present, otherwise validate by `transporter_key`. + if getattr(invoice.logistics, "carrier_int_id", None) is not None: + carrier_exists = ( + db.query(Transporter) + .filter( + Transporter.transporter_id + == invoice.logistics.carrier_int_id, + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) + .first() + ) + else: + carrier_exists = ( + db.query(Transporter) + .filter( + Transporter.transporter_key == invoice.logistics.carrier_id, + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) + .first() ) - .first() - ) if not carrier_exists: errors.add_error( field="logistics.carrier_id", diff --git a/backend/api/v1/modules/a76/invoices/models.py b/backend/api/v1/modules/a76/invoices/models.py index a10be8c7..f6723b3e 100644 --- a/backend/api/v1/modules/a76/invoices/models.py +++ b/backend/api/v1/modules/a76/invoices/models.py @@ -563,9 +563,15 @@ class InvoiceLogistics(Base, TenantScopedMixin, TimestampMixin): carrier_id: Mapped[Optional[str]] = mapped_column( String(10) ) # TRANSPORTISTA / Transportista + carrier_int_id: Mapped[Optional[int]] = mapped_column( + BigInteger, ForeignKey("a76.transporter.transporter_id"), nullable=True + ) # Internal surrogate ID for Transporter (by transporter_key) transport_id: Mapped[Optional[str]] = mapped_column( String(10) ) # NUMTRAILER / Transportista + transport_int_id: Mapped[Optional[int]] = mapped_column( + BigInteger, ForeignKey("a76.vehicle.vehicle_id"), nullable=True + ) # Internal surrogate ID for Vehicle (by vehicle_key) transport_us_id: Mapped[Optional[str]] = mapped_column( String(10) ) # TRANSPORTISTAAME / Transportista americano @@ -601,6 +607,9 @@ class InvoiceLogistics(Base, TenantScopedMixin, TimestampMixin): trailer_num: Mapped[Optional[str]] = mapped_column( String(20) ) # NUMTRAILER / Número de trailer + trailer_int_id: Mapped[Optional[int]] = mapped_column( + BigInteger, ForeignKey("a76.trailer.trailer_id"), nullable=True + ) # Internal surrogate ID for Trailer (by trailer_number) seal_number: Mapped[Optional[str]] = mapped_column( String(15) ) # PRECINTO / Precinto diff --git a/backend/api/v1/modules/a76/invoices/schemas.py b/backend/api/v1/modules/a76/invoices/schemas.py index 46d0997a..e8df1724 100644 --- a/backend/api/v1/modules/a76/invoices/schemas.py +++ b/backend/api/v1/modules/a76/invoices/schemas.py @@ -299,9 +299,14 @@ class InvoiceFinancialsBase(BaseModel): class InvoiceLogisticsBase(BaseModel): """Base fields for Logistics""" - - carrier_id: Optional[str] = Field(None, max_length=10, description="Carrier ID") + carrier_id: Optional[str] = Field(None, max_length=10, description="Carrier code (front-end key)") + carrier_int_id: Optional[int] = Field( + None, description="Internal carrier integer ID (mapped from carrier_id key)" + ) transport_id: Optional[str] = Field(None, max_length=10, description="Transport ID") + transport_int_id: Optional[int] = Field( + None, description="Internal vehicle integer ID (mapped from transport_id key)" + ) transport_us_id: Optional[str] = Field( None, max_length=10, description="US transport ID" ) @@ -329,6 +334,9 @@ class InvoiceLogisticsBase(BaseModel): trailer_num: Optional[str] = Field( None, max_length=20, description="Trailer number" ) + trailer_int_id: Optional[int] = Field( + None, description="Internal trailer integer ID (mapped from trailer_num key)" + ) seal_number: Optional[str] = Field(None, max_length=15, description="Seal number") guide_number: Optional[str] = Field(None, max_length=20, description="Guide number") bill_number: Optional[str] = Field(None, max_length=15, description="Bill number") diff --git a/backend/api/v1/modules/a76/invoices/services.py b/backend/api/v1/modules/a76/invoices/services.py index a9871ad6..5bcd6fd0 100644 --- a/backend/api/v1/modules/a76/invoices/services.py +++ b/backend/api/v1/modules/a76/invoices/services.py @@ -15,6 +15,87 @@ from api.v1.modules.a76.items.models import LineItem from . import models, schemas +def _autofill_transport_int_ids( + db: Session, + logistics_target, + tenant_id: int, + company_id: int, +) -> None: + """ + Ensures invoice logistics internal int IDs are populated when their string codes exist. + + The frontend and exports/imports keep using the string keys/codes (carrier_id, transport_id, trailer_num), + but internally we also persist integer surrogate IDs for fast/consistent joins. + """ + + # Local imports to avoid circular dependencies. + from api.v1.modules.a76.transportation.transporters.models import Transporter + from api.v1.modules.a76.transportation.vehicles.models import Vehicle + from api.v1.modules.a76.transportation.trailers.models import Trailer + + def _get(field: str): + if isinstance(logistics_target, dict): + return logistics_target.get(field) + return getattr(logistics_target, field, None) + + def _set(field: str, value): + if isinstance(logistics_target, dict): + logistics_target[field] = value + else: + setattr(logistics_target, field, value) + + carrier_code = _get("carrier_id") + carrier_int_id = _get("carrier_int_id") + if carrier_code and carrier_int_id is None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_key == carrier_code) + .first() + ) + if not transporter_obj: + # Minimal insert so mapping doesn't end up NULL. + transporter_obj = Transporter( + transporter_key=carrier_code, + tenant_id=tenant_id, + company_id=company_id, + ) + db.add(transporter_obj) + db.flush() + _set("carrier_int_id", transporter_obj.transporter_id) + + transport_code = _get("transport_id") + transport_int_id = _get("transport_int_id") + if transport_code and transport_int_id is None: + vehicle_obj = ( + db.query(Vehicle).filter(Vehicle.vehicle_key == transport_code).first() + ) + if not vehicle_obj: + vehicle_obj = Vehicle( + vehicle_key=transport_code, + tenant_id=tenant_id, + company_id=company_id, + ) + db.add(vehicle_obj) + db.flush() + _set("transport_int_id", vehicle_obj.vehicle_id) + + trailer_code = _get("trailer_num") + trailer_int_id = _get("trailer_int_id") + if trailer_code and trailer_int_id is None: + trailer_obj = ( + db.query(Trailer).filter(Trailer.trailer_number == trailer_code).first() + ) + if not trailer_obj: + trailer_obj = Trailer( + trailer_number=trailer_code, + tenant_id=tenant_id, + company_id=company_id, + ) + db.add(trailer_obj) + db.flush() + _set("trailer_int_id", trailer_obj.trailer_id) + + def _get_current_username() -> str: """Helper to get current username from context or fallback to System""" try: @@ -282,6 +363,12 @@ class InvoiceService: logistics_dict["invoice_id"] = new_invoice.id logistics_dict["tenant_id"] = tenant_id logistics_dict["company_id"] = company_id + _autofill_transport_int_ids( + db, + logistics_dict, + tenant_id=tenant_id, + company_id=company_id, + ) new_logistics = models.InvoiceLogistics(**logistics_dict) db.add(new_logistics) @@ -444,11 +531,23 @@ class InvoiceService: if value == "": value = None setattr(invoice.logistics, key, value) + _autofill_transport_int_ids( + db, + invoice.logistics, + tenant_id=tenant_id, + company_id=company_id, + ) else: logistics_dict = invoice_data.logistics.model_dump() logistics_dict["invoice_id"] = invoice.id logistics_dict["tenant_id"] = tenant_id logistics_dict["company_id"] = company_id + _autofill_transport_int_ids( + db, + logistics_dict, + tenant_id=tenant_id, + company_id=company_id, + ) new_logistics = models.InvoiceLogistics(**logistics_dict) db.add(new_logistics) diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index f1ad7d80..7717a874 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -6071,8 +6071,30 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt transport_type = TransportType(transport_str) except ValueError: transport_type = TransportType.NONE + + carrier_code = row_norm.get('CLAVE TRANSPORTISTA') or None + carrier_int_id = None + if carrier_code: + # Internal int mapping for carrier_code -> transporter_id + from api.v1.modules.a76.transportation.transporters.models import Transporter + + carrier_obj = ( + session.query(Transporter) + .filter(Transporter.transporter_key == carrier_code) + .first() + ) + if not carrier_obj: + carrier_obj = Transporter( + transporter_key=carrier_code, + tenant_id=tenant_id, + company_id=company_id, + ) + session.add(carrier_obj) + session.flush() + carrier_int_id = carrier_obj.transporter_id logistics = InvoiceLogistics( - carrier_id=(row_norm.get('CLAVE TRANSPORTISTA') or None), + carrier_id=carrier_code, + carrier_int_id=carrier_int_id, driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), transport_type=transport_type, transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), diff --git a/backend/api/v1/modules/a76/reports/importacion/consolidados/mex/service.py b/backend/api/v1/modules/a76/reports/importacion/consolidados/mex/service.py index b35c8bc1..c90b81b1 100644 --- a/backend/api/v1/modules/a76/reports/importacion/consolidados/mex/service.py +++ b/backend/api/v1/modules/a76/reports/importacion/consolidados/mex/service.py @@ -298,7 +298,13 @@ class ConsolidadoImportacionMexService: if logistics: # 1. Transporter (CAAT / SCAC) - if logistics.carrier_id: + if logistics.carrier_int_id is not None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_id == logistics.carrier_int_id) + .first() + ) + elif logistics.carrier_id: transporter_obj = ( db.query(Transporter) .filter(Transporter.transporter_key == logistics.carrier_id) @@ -342,11 +348,18 @@ class ConsolidadoImportacionMexService: # 2. Vehicle (Placas Tracto) - Try transport_id first if logistics.transport_id: - veh_obj = ( - db.query(Vehicle) - .filter(Vehicle.vehicle_key == logistics.transport_id) - .first() - ) + if logistics.transport_int_id is not None: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_id == logistics.transport_int_id) + .first() + ) + else: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_key == logistics.transport_id) + .first() + ) if veh_obj: placas_val = veh_obj.plate_number or placas_val elif ( @@ -362,11 +375,18 @@ class ConsolidadoImportacionMexService: # 3. Trailer (Placas Remolque) if logistics.trailer_num: - trl_obj = ( - db.query(Trailer) - .filter(Trailer.trailer_number == logistics.trailer_num) - .first() - ) + if logistics.trailer_int_id is not None: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_id == logistics.trailer_int_id) + .first() + ) + else: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_number == logistics.trailer_num) + .first() + ) if trl_obj: placas_remolque_val = trl_obj.plate_number or "" diff --git a/backend/api/v1/modules/a76/reports/importacion/consolidados/temporary/mex/service.py b/backend/api/v1/modules/a76/reports/importacion/consolidados/temporary/mex/service.py index a3949d07..d942d1ce 100644 --- a/backend/api/v1/modules/a76/reports/importacion/consolidados/temporary/mex/service.py +++ b/backend/api/v1/modules/a76/reports/importacion/consolidados/temporary/mex/service.py @@ -175,8 +175,16 @@ class ConsolidadoImportacionMexService: if logistics: # 1. Transporter (CAAT / SCAC) - if logistics.carrier_id: - transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first() + if logistics.carrier_int_id is not None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_id == logistics.carrier_int_id) + .first() + ) + elif logistics.carrier_id: + transporter_obj = db.query(Transporter).filter( + Transporter.transporter_key == logistics.carrier_id + ).first() if transporter_obj: caat_val = transporter_obj.caat_code or "" scac_val = transporter_obj.transport_code or "" # Mapping transport_code to SCAC @@ -213,7 +221,16 @@ class ConsolidadoImportacionMexService: # 2. Vehicle (Placas Tracto) - Try transport_id first if logistics.transport_id: - veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first() + if logistics.transport_int_id is not None: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_id == logistics.transport_int_id) + .first() + ) + else: + veh_obj = db.query(Vehicle).filter( + Vehicle.vehicle_key == logistics.transport_id + ).first() if veh_obj: placas_val = veh_obj.plate_number or placas_val elif logistics.vehicle_num: # Fallback to vehicle_num if populated and transport_id failed/empty @@ -223,7 +240,16 @@ class ConsolidadoImportacionMexService: # 3. Trailer (Placas Remolque) if logistics.trailer_num: - trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first() + if logistics.trailer_int_id is not None: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_id == logistics.trailer_int_id) + .first() + ) + else: + trl_obj = db.query(Trailer).filter( + Trailer.trailer_number == logistics.trailer_num + ).first() if trl_obj: placas_remolque_val = trl_obj.plate_number or "" diff --git a/backend/api/v1/modules/a76/reports/importacion/facturas/mex/service.py b/backend/api/v1/modules/a76/reports/importacion/facturas/mex/service.py index 185589d0..300f462e 100644 --- a/backend/api/v1/modules/a76/reports/importacion/facturas/mex/service.py +++ b/backend/api/v1/modules/a76/reports/importacion/facturas/mex/service.py @@ -317,7 +317,17 @@ class FacturaImportacionMexService: if logistics: # 1. Transporter (CAAT / SCAC) - if logistics.carrier_id: + if logistics.carrier_int_id is not None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_id == logistics.carrier_int_id) + .first() + ) + if transporter_obj: + caat_val = transporter_obj.caat_code or "" + scac_val = transporter_obj.transport_code or "" + transportista_val = transporter_obj.name or logistics.carrier_id + elif logistics.carrier_id: transporter_obj = ( db.query(Transporter) .filter(Transporter.transporter_key == logistics.carrier_id) @@ -332,11 +342,18 @@ class FacturaImportacionMexService: # 2. Vehicle (Placas Tracto) - Try transport_id first if logistics.transport_id: - veh_obj = ( - db.query(Vehicle) - .filter(Vehicle.vehicle_key == logistics.transport_id) - .first() - ) + if logistics.transport_int_id is not None: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_id == logistics.transport_int_id) + .first() + ) + else: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_key == logistics.transport_id) + .first() + ) if veh_obj: placas_val = veh_obj.plate_number or placas_val elif ( @@ -352,11 +369,18 @@ class FacturaImportacionMexService: # 3. Trailer (Placas Remolque) if logistics.trailer_num: - trl_obj = ( - db.query(Trailer) - .filter(Trailer.trailer_number == logistics.trailer_num) - .first() - ) + if logistics.trailer_int_id is not None: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_id == logistics.trailer_int_id) + .first() + ) + else: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_number == logistics.trailer_num) + .first() + ) if trl_obj: placas_remolque_val = trl_obj.plate_number or "" diff --git a/backend/api/v1/modules/a76/reports/importacion/facturas/temporary/mex/service.py b/backend/api/v1/modules/a76/reports/importacion/facturas/temporary/mex/service.py index 192dab8a..3d0b86d5 100644 --- a/backend/api/v1/modules/a76/reports/importacion/facturas/temporary/mex/service.py +++ b/backend/api/v1/modules/a76/reports/importacion/facturas/temporary/mex/service.py @@ -166,8 +166,16 @@ class FacturaImportacionMexService: if logistics: # 1. Transporter (CAAT / SCAC) - if logistics.carrier_id: - transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first() + if logistics.carrier_int_id is not None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_id == logistics.carrier_int_id) + .first() + ) + elif logistics.carrier_id: + transporter_obj = db.query(Transporter).filter( + Transporter.transporter_key == logistics.carrier_id + ).first() if transporter_obj: caat_val = transporter_obj.caat_code or "" scac_val = transporter_obj.transport_code or "" # Mapping transport_code to SCAC @@ -175,7 +183,16 @@ class FacturaImportacionMexService: # 2. Vehicle (Placas Tracto) - Try transport_id first if logistics.transport_id: - veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first() + if logistics.transport_int_id is not None: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_id == logistics.transport_int_id) + .first() + ) + else: + veh_obj = db.query(Vehicle).filter( + Vehicle.vehicle_key == logistics.transport_id + ).first() if veh_obj: placas_val = veh_obj.plate_number or placas_val elif logistics.vehicle_num: # Fallback to vehicle_num if populated and transport_id failed/empty @@ -185,7 +202,16 @@ class FacturaImportacionMexService: # 3. Trailer (Placas Remolque) if logistics.trailer_num: - trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first() + if logistics.trailer_int_id is not None: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_id == logistics.trailer_int_id) + .first() + ) + else: + trl_obj = db.query(Trailer).filter( + Trailer.trailer_number == logistics.trailer_num + ).first() if trl_obj: placas_remolque_val = trl_obj.plate_number or "" diff --git a/backend/api/v1/modules/a76/reports/importacion/facturas/usa/service.py b/backend/api/v1/modules/a76/reports/importacion/facturas/usa/service.py index 9bd7c40d..8a4c1a91 100644 --- a/backend/api/v1/modules/a76/reports/importacion/facturas/usa/service.py +++ b/backend/api/v1/modules/a76/reports/importacion/facturas/usa/service.py @@ -317,7 +317,17 @@ class FacturaImportacionUsaService: if logistics: # 1. Transporter (CAAT / SCAC) - if logistics.carrier_id: + if logistics.carrier_int_id is not None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_id == logistics.carrier_int_id) + .first() + ) + if transporter_obj: + caat_val = transporter_obj.caat_code or "" + scac_val = transporter_obj.transport_code or "" + transportista_val = transporter_obj.name or logistics.carrier_id + elif logistics.carrier_id: transporter_obj = ( db.query(Transporter) .filter(Transporter.transporter_key == logistics.carrier_id) @@ -325,18 +335,23 @@ class FacturaImportacionUsaService: ) if transporter_obj: caat_val = transporter_obj.caat_code or "" - scac_val = ( - transporter_obj.transport_code or "" - ) # Mapping transport_code to SCAC + scac_val = transporter_obj.transport_code or "" transportista_val = transporter_obj.name or logistics.carrier_id # 2. Vehicle (Plates) if logistics.transport_id: - veh_obj = ( - db.query(Vehicle) - .filter(Vehicle.vehicle_key == logistics.transport_id) - .first() - ) + if logistics.transport_int_id is not None: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_id == logistics.transport_int_id) + .first() + ) + else: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_key == logistics.transport_id) + .first() + ) if veh_obj: placas_val = veh_obj.plate_number or placas_val elif logistics.vehicle_num: @@ -350,11 +365,18 @@ class FacturaImportacionUsaService: # 3. Trailer if logistics.trailer_num: - trl_obj = ( - db.query(Trailer) - .filter(Trailer.trailer_number == logistics.trailer_num) - .first() - ) + if logistics.trailer_int_id is not None: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_id == logistics.trailer_int_id) + .first() + ) + else: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_number == logistics.trailer_num) + .first() + ) if trl_obj: placas_remolque_val = trl_obj.plate_number or "" diff --git a/backend/api/v1/modules/a76/reports/importacion/packing_list/service.py b/backend/api/v1/modules/a76/reports/importacion/packing_list/service.py index 8a74bbb3..822294ae 100644 --- a/backend/api/v1/modules/a76/reports/importacion/packing_list/service.py +++ b/backend/api/v1/modules/a76/reports/importacion/packing_list/service.py @@ -186,8 +186,18 @@ class PackingListService: if logistics: # 1. Transporter (CAAT / SCAC) - if logistics.carrier_id: - transporter_obj = db.query(Transporter).filter(Transporter.transporter_key == logistics.carrier_id).first() + if logistics.carrier_int_id is not None: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_id == logistics.carrier_int_id) + .first() + ) + elif logistics.carrier_id: + transporter_obj = ( + db.query(Transporter) + .filter(Transporter.transporter_key == logistics.carrier_id) + .first() + ) if transporter_obj: caat_val = transporter_obj.caat_code or "" scac_val = transporter_obj.transport_code or "" @@ -195,7 +205,18 @@ class PackingListService: # 2. Vehicle (Placas Tracto) if logistics.transport_id: - veh_obj = db.query(Vehicle).filter(Vehicle.vehicle_key == logistics.transport_id).first() + if logistics.transport_int_id is not None: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_id == logistics.transport_int_id) + .first() + ) + else: + veh_obj = ( + db.query(Vehicle) + .filter(Vehicle.vehicle_key == logistics.transport_id) + .first() + ) if veh_obj: placas_val = veh_obj.plate_number or placas_val elif logistics.vehicle_num: @@ -205,7 +226,18 @@ class PackingListService: # 3. Trailer (Placas Remolque) if logistics.trailer_num: - trl_obj = db.query(Trailer).filter(Trailer.trailer_number == logistics.trailer_num).first() + if logistics.trailer_int_id is not None: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_id == logistics.trailer_int_id) + .first() + ) + else: + trl_obj = ( + db.query(Trailer) + .filter(Trailer.trailer_number == logistics.trailer_num) + .first() + ) if trl_obj: placas_remolque_val = trl_obj.plate_number or "" diff --git a/backend/api/v1/modules/a76/transportation/drivers/dto.py b/backend/api/v1/modules/a76/transportation/drivers/dto.py index 02420ea8..e67daee0 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/dto.py +++ b/backend/api/v1/modules/a76/transportation/drivers/dto.py @@ -5,6 +5,7 @@ from pydantic import BaseModel class DriverBaseDTO(BaseModel): transporter_key: str + driver_id: Optional[int] = None line: int driver_name: Optional[str] = None license_number: Optional[str] = None diff --git a/backend/api/v1/modules/a76/transportation/drivers/models.py b/backend/api/v1/modules/a76/transportation/drivers/models.py index 7c70a9c3..dc6d7ee9 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/models.py +++ b/backend/api/v1/modules/a76/transportation/drivers/models.py @@ -1,6 +1,6 @@ from api.v1.common.base_models import TenantScopedMixin, TimestampMixin from core.database import Base -from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String +from sqlalchemy import BigInteger, Column, ForeignKey, ForeignKeyConstraint, Integer, String class Driver(Base, TenantScopedMixin, TimestampMixin): @@ -16,6 +16,8 @@ class Driver(Base, TenantScopedMixin, TimestampMixin): nullable=False, ) line = Column(Integer, primary_key=True, nullable=False) + # Internal integer surrogate ID. The frontend continues to use transporter_key/line. + driver_id = Column(BigInteger, nullable=False, unique=True, index=True) driver_name = Column(String(80), nullable=True) license_number = Column(String(29), nullable=True) express_line_id = Column(String(17), nullable=True) diff --git a/backend/api/v1/modules/a76/transportation/trailers/dto.py b/backend/api/v1/modules/a76/transportation/trailers/dto.py index 548f1d8f..662fb5b4 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/dto.py +++ b/backend/api/v1/modules/a76/transportation/trailers/dto.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field class TrailerBaseDTO(BaseModel): trailer_number: str = Field(..., description="Trailer number (primary identifier)") + trailer_id: Optional[int] = Field(None, description="Internal trailer integer ID") ace_trailer_number: Optional[str] = None trailer_type_key: Optional[str] = None seal: Optional[str] = None diff --git a/backend/api/v1/modules/a76/transportation/trailers/models.py b/backend/api/v1/modules/a76/transportation/trailers/models.py index d3b19e8b..19e7b1c9 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/models.py +++ b/backend/api/v1/modules/a76/transportation/trailers/models.py @@ -1,6 +1,6 @@ from api.v1.common.base_models import TenantScopedMixin, TimestampMixin from core.database import Base -from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, String +from sqlalchemy import BigInteger, Column, ForeignKey, ForeignKeyConstraint, String class Trailer(Base, TenantScopedMixin, TimestampMixin): @@ -10,6 +10,8 @@ class Trailer(Base, TenantScopedMixin, TimestampMixin): ) trailer_number = Column(String(20), primary_key=True, nullable=False) + # Internal integer identifier (surrogate). The frontend continues to use trailer_number. + trailer_id = Column(BigInteger, nullable=False, unique=True, index=True) ace_trailer_number = Column(String(10), nullable=True) trailer_type_key = Column( String(2), ForeignKey("public.trailer_type.trailer_type_key"), nullable=True diff --git a/backend/api/v1/modules/a76/transportation/transporters/dto.py b/backend/api/v1/modules/a76/transportation/transporters/dto.py index 06391a6e..c4397d00 100644 --- a/backend/api/v1/modules/a76/transportation/transporters/dto.py +++ b/backend/api/v1/modules/a76/transportation/transporters/dto.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field class TransporterBaseDTO(BaseModel): transporter_key: str = Field(..., description="Transporter key (primary identifier)") + transporter_id: Optional[int] = Field(None, description="Internal transporter integer ID") name: Optional[str] = None short_name: Optional[str] = None responsible: Optional[str] = None diff --git a/backend/api/v1/modules/a76/transportation/transporters/models.py b/backend/api/v1/modules/a76/transportation/transporters/models.py index 9f555c79..58d03367 100644 --- a/backend/api/v1/modules/a76/transportation/transporters/models.py +++ b/backend/api/v1/modules/a76/transportation/transporters/models.py @@ -1,6 +1,6 @@ from api.v1.common.base_models import TenantScopedMixin, TimestampMixin from core.database import Base -from sqlalchemy import Column, ForeignKeyConstraint, String +from sqlalchemy import BigInteger, Column, ForeignKeyConstraint, String class Transporter(Base, TenantScopedMixin, TimestampMixin): @@ -10,6 +10,8 @@ class Transporter(Base, TenantScopedMixin, TimestampMixin): ) transporter_key = Column(String(23), primary_key=True, nullable=False) + # Internal integer identifier (surrogate). The frontend continues to use transporter_key. + transporter_id = Column(BigInteger, nullable=False, unique=True, index=True) name = Column(String(256), nullable=True) short_name = Column(String(10), nullable=True) responsible = Column(String(100), nullable=True) diff --git a/backend/api/v1/modules/a76/transportation/vehicles/dto.py b/backend/api/v1/modules/a76/transportation/vehicles/dto.py index 7c7d28b9..c4cb6c8a 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/dto.py +++ b/backend/api/v1/modules/a76/transportation/vehicles/dto.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field class VehicleBaseDTO(BaseModel): vehicle_key: str = Field(..., description="Vehicle key (primary identifier)") + vehicle_id: Optional[int] = Field(None, description="Internal vehicle integer ID") ace_vehicle_key: Optional[str] = None transporter_key: Optional[str] = None transport_identifier: Optional[str] = None diff --git a/backend/api/v1/modules/a76/transportation/vehicles/models.py b/backend/api/v1/modules/a76/transportation/vehicles/models.py index 9339daf8..0bbbb5d9 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/models.py +++ b/backend/api/v1/modules/a76/transportation/vehicles/models.py @@ -1,6 +1,6 @@ from api.v1.common.base_models import TenantScopedMixin, TimestampMixin from core.database import Base -from sqlalchemy import DECIMAL, Column, Integer, String +from sqlalchemy import DECIMAL, Column, Integer, BigInteger, String class Vehicle(Base, TenantScopedMixin, TimestampMixin): @@ -10,6 +10,8 @@ class Vehicle(Base, TenantScopedMixin, TimestampMixin): ) vehicle_key = Column(String(14), primary_key=True, nullable=False) + # Internal integer identifier (surrogate). The frontend continues to use vehicle_key. + vehicle_id = Column(BigInteger, nullable=False, unique=True, index=True) ace_vehicle_key = Column(String(10), nullable=True) transporter_key = Column(String(23), nullable=True) transport_identifier = Column(String(30), nullable=True) diff --git a/frontend/src/lib/api/dashboard/a76/invoices.ts b/frontend/src/lib/api/dashboard/a76/invoices.ts index d0849433..fb3c491c 100644 --- a/frontend/src/lib/api/dashboard/a76/invoices.ts +++ b/frontend/src/lib/api/dashboard/a76/invoices.ts @@ -124,7 +124,9 @@ export interface InvoiceLogistics { id?: number; invoice_id?: number; carrier_id?: string | null; + carrier_int_id?: number | null; transport_id?: string | null; + transport_int_id?: number | null; transport_us_id?: string | null; transport_type?: TransportType | null; transport_num?: string | null; @@ -136,6 +138,7 @@ export interface InvoiceLogistics { license_plate?: string | null; license_plate_complete?: string | null; trailer_num?: string | null; + trailer_int_id?: number | null; seal_number?: string | null; guide_number?: string | null; bill_number?: string | null; diff --git a/frontend/src/lib/api/dashboard/a76/trailers.ts b/frontend/src/lib/api/dashboard/a76/trailers.ts index 5d74aa2d..bf57509c 100644 --- a/frontend/src/lib/api/dashboard/a76/trailers.ts +++ b/frontend/src/lib/api/dashboard/a76/trailers.ts @@ -2,6 +2,7 @@ import { api, type ApiResponse } from '$lib/api'; export interface Trailer { trailer_number: string; + trailer_id?: number; ace_trailer_number?: string; trailer_type_key?: string; seal?: string; diff --git a/frontend/src/lib/api/dashboard/a76/transporters.ts b/frontend/src/lib/api/dashboard/a76/transporters.ts index fa585465..da855ae0 100644 --- a/frontend/src/lib/api/dashboard/a76/transporters.ts +++ b/frontend/src/lib/api/dashboard/a76/transporters.ts @@ -2,6 +2,7 @@ import { api, type ApiResponse } from '$lib/api'; export interface Transporter { transporter_key: string; + transporter_id?: number; name?: string; short_name?: string; responsible?: string; diff --git a/frontend/src/lib/api/dashboard/a76/vehicles.ts b/frontend/src/lib/api/dashboard/a76/vehicles.ts index 3fd7d67a..9c501dab 100644 --- a/frontend/src/lib/api/dashboard/a76/vehicles.ts +++ b/frontend/src/lib/api/dashboard/a76/vehicles.ts @@ -2,6 +2,7 @@ import { api, type ApiResponse } from '$lib/api'; export interface Vehicle { vehicle_key: string; + vehicle_id?: number; ace_vehicle_key?: string; transporter_key?: string; transport_identifier?: string; From 7a029307a89bd384e854a7f6999dfe743b71d2bb Mon Sep 17 00:00:00 2001 From: hreyes Date: Fri, 27 Mar 2026 12:56:47 -0600 Subject: [PATCH 03/32] feature/validaciones-invoices-transortes --- .../a76/invoices/common/common_validators.py | 244 +++++++++++++++--- .../api/v1/modules/a76/invoices/services.py | 121 ++++++--- .../modules/a76/layouts_csv/facturas/tasks.py | 210 ++++++++++----- .../layouts_csv/facturas/template_config.py | 8 + .../facturas/validators/encabezados_cmex.py | 12 - .../validators/encabezados_impo_temp.py | 12 - .../facturas/validators/transport_catalog.py | 228 ++++++++++++++++ .../layouts_csv/trailers/common/fk_loader.py | 40 ++- .../transportistas/common/fk_loader.py | 31 ++- .../layouts_csv/vehicles/common/fk_loader.py | 33 ++- .../a76/transportation/drivers/services.py | 13 +- .../a76/transportation/trailers/services.py | 13 +- .../transportation/transporters/services.py | 14 +- .../a76/transportation/vehicles/services.py | 13 +- .../test_fk_loader_state_mex_key.py | 130 ++++++++++ ...gracion_csv_transporte_sin_validaciones.md | 141 ++++++++++ .../invoices/edit/general-tab-form.svelte | 88 ++++++- .../dashboard/invoices/edit/save-invoice.ts | 35 ++- .../dashboard/invoices/edit/[id]/+page.svelte | 9 +- .../dashboard/invoices/settings/+page.svelte | 4 +- 20 files changed, 1172 insertions(+), 227 deletions(-) create mode 100644 backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py create mode 100644 backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py create mode 100644 docs/analisis_flujo_migracion_csv_transporte_sin_validaciones.md diff --git a/backend/api/v1/modules/a76/invoices/common/common_validators.py b/backend/api/v1/modules/a76/invoices/common/common_validators.py index 558f6940..f4c0db4c 100644 --- a/backend/api/v1/modules/a76/invoices/common/common_validators.py +++ b/backend/api/v1/modules/a76/invoices/common/common_validators.py @@ -13,12 +13,19 @@ from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRat from api.v1.modules.a76.clients_and_providers.models import ClientProvider from api.v1.modules.a76.customs_brokers.models import CustomsBroker from api.v1.modules.a76.transportation.transporters.models import Transporter +from api.v1.modules.a76.transportation.vehicles.models import Vehicle +from api.v1.modules.a76.transportation.trailers.models import Trailer +from api.v1.modules.a76.transportation.drivers.models import Driver from api.v1.modules.a76.manifests.manifest.models import Manifest from api.v1.modules.public.reference_data.incoterms.models import Incoterm from api.v1.modules.public.reference_data.currency_types.models import CurrencyType from api.v1.modules.public.reference_data.customs_sections.models import CustomsSection +def _logistics_str_nonempty(value) -> bool: + return value is not None and str(value).strip() != "" + + def _normalize_invoice_currency_value(value) -> str: """Lowercase currency code (foreign/local/manual) for comparisons.""" if value is None or value == "": @@ -344,25 +351,40 @@ def validate_common( code="INVALID_VALUE", value=invoice.compliance_mx.remesa, ) + else: + # Misma remesa puede repetirse en distintos pedimentos; solo debe ser única + # por (pedimento_id, empresa, tenant). Creación: existing_invoice es None. + current_invoice_id = None + if existing_invoice is not None: + current_invoice_id = existing_invoice.id + else: + current_invoice_id = getattr(invoice, "id", None) - duplicated_remesa = ( - db.query(InvoiceComplianceMx) - .filter( - InvoiceComplianceMx.remesa == invoice.compliance_mx.remesa, - InvoiceComplianceMx.tenant_id == tenant_id, - InvoiceComplianceMx.company_id == company_id, - ) - .first() - ) - if duplicated_remesa and hasattr(invoice, "id"): - if invoice.id != duplicated_remesa.invoice_id: - errors.add_error( - field="compliance_mx.remesa", - message="El valor de Remesa ya está asociado a otro Pedimento.", - solution=["Proporciona un valor único para Remesa"], - code="DUPLICATE_VALUE", - value=invoice.compliance_mx.remesa, + duplicated_remesa = ( + db.query(InvoiceComplianceMx) + .filter( + InvoiceComplianceMx.remesa == invoice.compliance_mx.remesa, + InvoiceComplianceMx.pedimento_id + == invoice.compliance_mx.pedimento_id, + InvoiceComplianceMx.tenant_id == tenant_id, + InvoiceComplianceMx.company_id == company_id, + ) + .first() ) + if duplicated_remesa and ( + current_invoice_id is None + or duplicated_remesa.invoice_id != current_invoice_id + ): + errors.add_error( + field="compliance_mx.remesa", + message=( + "El valor de Remesa ya está registrado para este mismo Pedimento " + "en otra factura." + ), + solution=["Usa otra remesa o revisa la factura que ya la tiene capturada"], + code="DUPLICATE_VALUE", + value=invoice.compliance_mx.remesa, + ) # Financials checks (if provided) if invoice.financials: @@ -511,6 +533,7 @@ def validate_common( ) if invoice.logistics: + carrier_exists = None if invoice.logistics.carrier_id: # `carrier_id` is the frontend/export key (string). Prefer validating by the internal int # when present, otherwise validate by `transporter_key`. @@ -542,8 +565,36 @@ def validate_common( solution=["Verifica el ID del Transportista", "Revisa el catálogo"], code="NOT_FOUND", value=invoice.logistics.carrier_id, - ) - + ) + + if carrier_exists and _logistics_str_nonempty( + getattr(invoice.logistics, "driver_name", None) + ): + driver_row = ( + db.query(Driver) + .filter( + Driver.transporter_key == str(invoice.logistics.carrier_id).strip(), + Driver.driver_name + == str(invoice.logistics.driver_name).strip(), + Driver.tenant_id == tenant_id, + Driver.company_id == company_id, + ) + .first() + ) + if not driver_row: + errors.add_error( + field="logistics.driver_name", + message=( + "El conductor no existe en el catálogo de conductores " + "para el transportista indicado." + ), + solution=[ + "Registra el conductor en el catálogo o elige uno de la lista" + ], + code="NOT_FOUND", + value=invoice.logistics.driver_name, + ) + if invoice.logistics.transport_type not in [t.value for t in TransportType]: errors.add_error( field="logistics.transport_type", @@ -555,31 +606,138 @@ def validate_common( value=invoice.logistics.transport_type, ) else: - if ( - invoice.logistics.transport_type == "none" - and invoice.logistics.transport_num - ): - errors.add_error( - field="logistics.transport_num", - message="El Número de Transporte no debe proporcionarse cuando el Tipo de Transporte es 'none'.", - solution=[ - "Elimina el Número de Transporte o selecciona un Tipo de Transporte válido" - ], - code="INVALID_VALUE", - value=invoice.logistics.transport_num, - ) - else: - if ( - not invoice.logistics.transport_num - and invoice.logistics.transport_type != "none" - ): - errors.add_error( - field="logistics.transport_num", - message="El Número de Transporte es obligatorio cuando se proporciona un Tipo de Transporte distinto de 'none'.", - solution=["Proporciona un Número de Transporte válido"], - code="REQUIRED_FIELD", - value=invoice.logistics.transport_num, + # Vehículo: solo catálogo si viene informado (cualquier tipo de transporte). + tid = getattr(invoice.logistics, "transport_int_id", None) + tcode = invoice.logistics.transport_id + if tid is not None: + v_by_int = ( + db.query(Vehicle) + .filter( + Vehicle.vehicle_id == tid, + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, ) + .first() + ) + if not v_by_int: + errors.add_error( + field="logistics.transport_int_id", + message="El Vehículo (transporte) no existe en el catálogo.", + solution=[ + "Verifica el ID interno del vehículo o usa la clave de transporte" + ], + code="NOT_FOUND", + value=tid, + ) + elif _logistics_str_nonempty(tcode): + v_by_key = ( + db.query(Vehicle) + .filter( + func.upper(Vehicle.vehicle_key) + == str(tcode).strip().upper(), + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, + ) + .first() + ) + if not v_by_key: + errors.add_error( + field="logistics.transport_id", + message="El Vehículo (clave transporte) no existe en el catálogo.", + solution=["Verifica la clave en el catálogo de vehículos"], + code="NOT_FOUND", + value=tcode, + ) + + # Remolque: obligatorio si tipo != 'none'; con 'none' solo se valida catálogo si hay dato. + tr_int = getattr(invoice.logistics, "trailer_int_id", None) + tr_num = invoice.logistics.trailer_num + if invoice.logistics.transport_type != "none": + if tr_int is not None: + tr = ( + db.query(Trailer) + .filter( + Trailer.trailer_id == tr_int, + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + if not tr: + errors.add_error( + field="logistics.trailer_int_id", + message="El Remolque no existe en el catálogo.", + solution=["Verifica el ID interno del remolque"], + code="NOT_FOUND", + value=tr_int, + ) + elif not _logistics_str_nonempty(tr_num): + errors.add_error( + field="logistics.trailer_num", + message=( + "El Remolque es obligatorio cuando el Tipo de Transporte es distinto de 'none'." + ), + solution=["Selecciona un remolque del catálogo"], + code="REQUIRED_FIELD", + value=tr_num, + ) + else: + tr = ( + db.query(Trailer) + .filter( + func.upper(Trailer.trailer_number) + == str(tr_num).strip().upper(), + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + if not tr: + errors.add_error( + field="logistics.trailer_num", + message="El Remolque no existe en el catálogo.", + solution=["Verifica el número de remolque"], + code="NOT_FOUND", + value=tr_num, + ) + else: + if tr_int is not None: + tr = ( + db.query(Trailer) + .filter( + Trailer.trailer_id == tr_int, + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + if not tr: + errors.add_error( + field="logistics.trailer_int_id", + message="El Remolque no existe en el catálogo.", + solution=["Verifica el ID interno del remolque"], + code="NOT_FOUND", + value=tr_int, + ) + elif _logistics_str_nonempty(tr_num): + tr = ( + db.query(Trailer) + .filter( + func.upper(Trailer.trailer_number) + == str(tr_num).strip().upper(), + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + if not tr: + errors.add_error( + field="logistics.trailer_num", + message="El Remolque no existe en el catálogo.", + solution=["Verifica el número de remolque"], + code="NOT_FOUND", + value=tr_num, + ) if invoice.financials: submitted = invoice.financials.currency diff --git a/backend/api/v1/modules/a76/invoices/services.py b/backend/api/v1/modules/a76/invoices/services.py index 5bcd6fd0..676a57d3 100644 --- a/backend/api/v1/modules/a76/invoices/services.py +++ b/backend/api/v1/modules/a76/invoices/services.py @@ -11,7 +11,6 @@ from .exports.validators.create import validate_create as validate_create_export from .exports.validators.update import validate_update as validate_update_export from .common.common_validators import invoice_exists from api.v1.modules.a76.items.models import LineItem - from . import models, schemas @@ -22,10 +21,11 @@ def _autofill_transport_int_ids( company_id: int, ) -> None: """ - Ensures invoice logistics internal int IDs are populated when their string codes exist. + Ensures invoice logistics internal int IDs are populated when catalog rows exist. - The frontend and exports/imports keep using the string keys/codes (carrier_id, transport_id, trailer_num), - but internally we also persist integer surrogate IDs for fast/consistent joins. + String keys (carrier_id, transport_id, trailer_num) are matched to existing catalog + rows for the tenant/company; no placeholder rows are created. Invalid references must + be rejected by validate_common before persist. """ # Local imports to avoid circular dependencies. @@ -49,51 +49,102 @@ def _autofill_transport_int_ids( if carrier_code and carrier_int_id is None: transporter_obj = ( db.query(Transporter) - .filter(Transporter.transporter_key == carrier_code) + .filter( + Transporter.transporter_key == carrier_code, + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) .first() ) - if not transporter_obj: - # Minimal insert so mapping doesn't end up NULL. - transporter_obj = Transporter( - transporter_key=carrier_code, - tenant_id=tenant_id, - company_id=company_id, - ) - db.add(transporter_obj) - db.flush() - _set("carrier_int_id", transporter_obj.transporter_id) + if transporter_obj: + _set("carrier_int_id", transporter_obj.transporter_id) transport_code = _get("transport_id") transport_int_id = _get("transport_int_id") if transport_code and transport_int_id is None: vehicle_obj = ( - db.query(Vehicle).filter(Vehicle.vehicle_key == transport_code).first() - ) - if not vehicle_obj: - vehicle_obj = Vehicle( - vehicle_key=transport_code, - tenant_id=tenant_id, - company_id=company_id, + db.query(Vehicle) + .filter( + func.upper(Vehicle.vehicle_key) == str(transport_code).strip().upper(), + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, ) - db.add(vehicle_obj) - db.flush() - _set("transport_int_id", vehicle_obj.vehicle_id) + .first() + ) + if vehicle_obj: + _set("transport_int_id", vehicle_obj.vehicle_id) trailer_code = _get("trailer_num") trailer_int_id = _get("trailer_int_id") if trailer_code and trailer_int_id is None: trailer_obj = ( - db.query(Trailer).filter(Trailer.trailer_number == trailer_code).first() - ) - if not trailer_obj: - trailer_obj = Trailer( - trailer_number=trailer_code, - tenant_id=tenant_id, - company_id=company_id, + db.query(Trailer) + .filter( + func.upper(Trailer.trailer_number) + == str(trailer_code).strip().upper(), + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, ) - db.add(trailer_obj) - db.flush() - _set("trailer_int_id", trailer_obj.trailer_id) + .first() + ) + if trailer_obj: + _set("trailer_int_id", trailer_obj.trailer_id) + + # Placa: si no vino valor, tomar del vehículo y si no del remolque (catálogo) + lp = _get("license_plate") + if lp is None or str(lp).strip() == "": + veh = None + tv_id = _get("transport_int_id") + tv_code = _get("transport_id") + if tv_id is not None: + veh = ( + db.query(Vehicle) + .filter( + Vehicle.vehicle_id == tv_id, + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, + ) + .first() + ) + elif tv_code: + veh = ( + db.query(Vehicle) + .filter( + func.upper(Vehicle.vehicle_key) == str(tv_code).strip().upper(), + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, + ) + .first() + ) + if veh and getattr(veh, "plate_number", None): + _set("license_plate", veh.plate_number) + else: + tr = None + tr_id = _get("trailer_int_id") + tr_code = _get("trailer_num") + if tr_id is not None: + tr = ( + db.query(Trailer) + .filter( + Trailer.trailer_id == tr_id, + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + elif tr_code: + tr = ( + db.query(Trailer) + .filter( + func.upper(Trailer.trailer_number) + == str(tr_code).strip().upper(), + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + if tr and getattr(tr, "plate_number", None): + _set("license_plate", tr.plate_number) def _get_current_username() -> str: diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index 7717a874..4169392b 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -2453,6 +2453,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = parse_pedimento_col_a, _pedimento_key_from_parsed, ) + from .validators.transport_catalog import logistics_scan_row_errors def _ped_key_from_row(ped_str: str) -> Optional[str]: parsed = parse_pedimento_col_a(ped_str) @@ -2849,6 +2850,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = parse_date_fn=parse_date, warnings=warnings_row, ) + row_errors.extend( + logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i) + ) blocking = [e for e in row_errors if not e.get("warning")] if blocking: error_count += 1 @@ -2909,6 +2913,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = parse_pedimento_col_a_impo_def, ) from .validators.encabezados_impo_temp import _pedimento_key_from_parsed + from .validators.transport_catalog import logistics_scan_row_errors DEF_INVOICE_TYPES = ("DEF", "MATDE", "EXDEF") @@ -3205,6 +3210,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = parse_date_fn=parse_date, warnings=warnings_row, ) + row_errors.extend( + logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i) + ) blocking = [e for e in row_errors if not e.get("warning")] if blocking: error_count += 1 @@ -3260,6 +3268,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = from .validators.encabezados_expo import validate_row_encabezados_expo from .validators.encabezados_impo_temp import _pedimento_key_from_parsed from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def + from .validators.transport_catalog import logistics_scan_row_errors _fc = parse_footer_config(meta.get("footer_config")) actualizar = meta.get("actualizar", False) @@ -3582,6 +3591,9 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = parse_date_fn=parse_date, warnings=warnings_row, ) + row_errors.extend( + logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i) + ) blocking = [e for e in row_errors if not e.get("warning")] if blocking: error_count += 1 @@ -3630,6 +3642,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = from api.v1.modules.a76.general_catalogs.exchange_rate.models import ExchangeRate from api.v1.modules.a76.transportation.transporters.models import Transporter from .validators.encabezados_cmex import validate_row_encabezados_cmex + from .validators.transport_catalog import logistics_scan_row_errors _fc = parse_footer_config(meta.get("footer_config")) actualizar = meta.get("actualizar", False) @@ -3751,62 +3764,69 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = error_lines_list = [] errors_detail = [] - with open(error_path, "w", encoding="utf-8") as f_err: - for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) - row_norm = row_from_template(row, "cmex_header", normalize_header) - warnings_row = [] - row_errors = validate_row_encabezados_cmex( - row_norm, - i, - actualizar=actualizar, - invoice_exists_by_number=invoice_exists_by_number, - invoice_processed_by_number=invoice_processed_by_number, - valid_provider_ids=valid_provider_ids, - valid_sold_to_ids=valid_sold_to_ids, - valid_shipped_to_ids=valid_shipped_to_ids, - valid_transporter_keys=valid_transporter_keys, - valid_incoterms=valid_incoterms, - valid_currency_codes=valid_currency_codes, - exchange_rate_by_date=exchange_rate_by_date, - invoice_has_partidas_by_number=invoice_has_partidas_by_number, - existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, - valid_provider_short_names=valid_provider_short_names, - valid_sold_to_short_names=valid_sold_to_short_names, - valid_shipped_to_short_names=valid_shipped_to_short_names, - date_format=date_format, - parse_date_fn=parse_date, - warnings=warnings_row, - ) - blocking = [e for e in row_errors if not e.get("warning")] - if blocking: - error_count += 1 - error_lines_list.append(i) - for e in blocking: - f_err.write( - json.dumps( + with CoreSessionLocal() as session: + with open(error_path, "w", encoding="utf-8") as f_err: + for i, row in enumerate(rows_list, start=1): + if i % 1000 == 0: + self.update_state( + state="PROGRESS", + meta={"current": i, "total": total_rows, "errors": error_count}, + ) + row_norm = row_from_template(row, "cmex_header", normalize_header) + warnings_row = [] + row_errors = validate_row_encabezados_cmex( + row_norm, + i, + actualizar=actualizar, + invoice_exists_by_number=invoice_exists_by_number, + invoice_processed_by_number=invoice_processed_by_number, + valid_provider_ids=valid_provider_ids, + valid_sold_to_ids=valid_sold_to_ids, + valid_shipped_to_ids=valid_shipped_to_ids, + valid_transporter_keys=valid_transporter_keys, + valid_incoterms=valid_incoterms, + valid_currency_codes=valid_currency_codes, + exchange_rate_by_date=exchange_rate_by_date, + invoice_has_partidas_by_number=invoice_has_partidas_by_number, + existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, + valid_provider_short_names=valid_provider_short_names, + valid_sold_to_short_names=valid_sold_to_short_names, + valid_shipped_to_short_names=valid_shipped_to_short_names, + date_format=date_format, + parse_date_fn=parse_date, + warnings=warnings_row, + ) + row_errors.extend( + logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i) + ) + blocking = [e for e in row_errors if not e.get("warning")] + if blocking: + error_count += 1 + error_lines_list.append(i) + for e in blocking: + f_err.write( + json.dumps( + { + "line": e["line"], + "col": e.get("col", ""), + "msg": e.get("msg", ""), + "solution": e.get("solution", ""), + } + ) + + "\n" + ) + for e in row_errors + warnings_row: + if len(errors_detail) < 5000: + errors_detail.append( { "line": e["line"], "col": e.get("col", ""), "msg": e.get("msg", ""), "solution": e.get("solution", ""), + "warning": bool(e.get("warning", False)), } ) - + "\n" - ) - for e in row_errors + warnings_row: - if len(errors_detail) < 5000: - errors_detail.append( - { - "line": e["line"], - "col": e.get("col", ""), - "msg": e.get("msg", ""), - "solution": e.get("solution", ""), - "warning": bool(e.get("warning", False)), - } - ) - processed_rows += 1 + processed_rows += 1 common_storage.store_error_lines(effective_job_type, job_id, error_lines_list) return common_responses.scan_result( @@ -4223,6 +4243,18 @@ def validate_row_strict( if err: return err + from .validators.transport_catalog import logistics_scan_row_errors + + log_errs = logistics_scan_row_errors( + row, + validator.session, + validator.tenant_id, + validator.company_id, + line_num, + ) + if log_errs: + return log_errs[0] + elif target == "invoice_details": invoice_number = (row.get("NUMERO FACTURA") or row.get("NUM FACTURA") or row.get("FACTURA") or "").strip() if not invoice_number: @@ -5370,6 +5402,12 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt row_to_transport_type_clarion, _patente_from_agente_aduanal, ) + from .validators.transport_catalog import ( + normalize_transport_cell, + resolve_logistics_transport_fields_for_commit, + resolve_transport_csv_fields, + validate_csv_invoice_logistics_transport, + ) from .validators.encabezados_impo_def import parse_pedimento_col_a_impo_def from .validators.pedimento_resolution import resolve_pedimento_candidates @@ -6064,40 +6102,84 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt weight_type = parse_weight_unit(row_norm.get('TIPO PESO')) logistics = None - if weight_type or row_norm.get('TIPO TRANSPORTE') or row_norm.get('NUMERO TRANSPORTE'): + transport_key_csv, trailer_num_csv, transport_num_csv = resolve_transport_csv_fields(row_norm) + has_logistics_data = bool( + weight_type + or row_norm.get('TIPO TRANSPORTE') + or row_norm.get('NUMERO TRANSPORTE') + or transport_key_csv + or trailer_num_csv + or row_norm.get('CLAVE TRANSPORTISTA') + or row_norm.get('NOMBRE CONDUCTOR') + ) + if has_logistics_data: raw_transport = row_norm.get('TIPO TRANSPORTE') transport_str = (row_to_transport_type_clarion(raw_transport) or str(raw_transport or "").strip().lower() or "none") try: transport_type = TransportType(transport_str) except ValueError: transport_type = TransportType.NONE + err_transport = validate_csv_invoice_logistics_transport( + row_norm=row_norm, + session=session, + tenant_id=tenant_id, + company_id=company_id, + transport_type=transport_type, + ) + if err_transport: + skipped_invalid += 1 + reason = err_transport + skipped_fk_details.append( + {"line": i, "invoice": invoice_number, "reason": reason} + ) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue - carrier_code = row_norm.get('CLAVE TRANSPORTISTA') or None + transport_num_effective = (transport_num_csv or None) + if transport_type == TransportType.NONE: + transport_num_effective = None + + carrier_code = normalize_transport_cell(row_norm.get('CLAVE TRANSPORTISTA')) or None carrier_int_id = None if carrier_code: - # Internal int mapping for carrier_code -> transporter_id - from api.v1.modules.a76.transportation.transporters.models import Transporter + from api.v1.modules.a76.transportation.transporters.models import ( + Transporter, + ) carrier_obj = ( session.query(Transporter) - .filter(Transporter.transporter_key == carrier_code) + .filter( + Transporter.transporter_key == carrier_code, + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) .first() ) - if not carrier_obj: - carrier_obj = Transporter( - transporter_key=carrier_code, - tenant_id=tenant_id, - company_id=company_id, - ) - session.add(carrier_obj) - session.flush() - carrier_int_id = carrier_obj.transporter_id + if carrier_obj: + carrier_int_id = carrier_obj.transporter_id + + ( + transport_id_value, + trailer_num_value, + transport_int_id, + trailer_int_id, + ) = resolve_logistics_transport_fields_for_commit( + session=session, + tenant_id=tenant_id, + company_id=company_id, + clave_transporte_raw=transport_key_csv, + numero_caja_raw=trailer_num_csv, + ) logistics = InvoiceLogistics( carrier_id=carrier_code, carrier_int_id=carrier_int_id, driver_name=(row_norm.get('NOMBRE CONDUCTOR') or None), transport_type=transport_type, - transport_num=(row_norm.get('NUMERO TRANSPORTE') or None), + transport_num=transport_num_effective, + transport_id=transport_id_value, + trailer_num=trailer_num_value, + transport_int_id=transport_int_id, + trailer_int_id=trailer_int_id, weight_type=weight_type or WeightUnit.KGS, seal_number=(row_norm.get('PRECINTO') or None), incoterm=(row_norm.get('CLAVE INCOTERM') or None), diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py b/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py index 711f4653..e972528c 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py @@ -29,6 +29,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "CLAVE TRANSPORTISTA"}, {"canonical": "NOMBRE CONDUCTOR"}, {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "CLAVE TRANSPORTE"}, + {"canonical": "NUMERO CAJA"}, {"canonical": "NUMERO TRANSPORTE"}, {"canonical": "TIPO MONEDA"}, {"canonical": "CLAVE MONEDA"}, @@ -66,6 +68,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "CLAVE TRANSPORTISTA"}, {"canonical": "NOMBRE CONDUCTOR"}, {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "CLAVE TRANSPORTE"}, + {"canonical": "NUMERO CAJA"}, {"canonical": "NUMERO TRANSPORTE"}, {"canonical": "TIPO MONEDA"}, {"canonical": "CLAVE MONEDA"}, @@ -102,6 +106,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "CLAVE TRANSPORTISTA"}, {"canonical": "NOMBRE CONDUCTOR"}, {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "CLAVE TRANSPORTE"}, + {"canonical": "NUMERO CAJA"}, {"canonical": "NUMERO TRANSPORTE"}, {"canonical": "TIPO MONEDA"}, {"canonical": "CLAVE MONEDA"}, @@ -135,6 +141,8 @@ TEMPLATE_COLUMNS: Dict[str, List[Dict[str, Any]]] = { {"canonical": "CLAVE TRANSPORTISTA"}, {"canonical": "NOMBRE CONDUCTOR"}, {"canonical": "TIPO TRANSPORTE"}, + {"canonical": "CLAVE TRANSPORTE"}, + {"canonical": "NUMERO CAJA"}, {"canonical": "NUMERO TRANSPORTE"}, {"canonical": "TIPO MONEDA"}, {"canonical": "CLAVE MONEDA"}, diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_cmex.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_cmex.py index 320924d3..f46b639b 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_cmex.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_cmex.py @@ -51,18 +51,6 @@ def _validaciones_transporte_cmex(row: Dict[str, Any], line_num: int) -> Optiona "NUMERO TRANSPORTE", "Error: (Celda N{}) El Tipo de Transporte está vacío y está capturado un número de transporte.".format(line_num), ) - if m_raw and (m == "NINGUNO" or m_with_space == "NINGUNO") and n: - return _err( - line_num, - "NUMERO TRANSPORTE", - "Error: (Celda N{}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.".format(line_num), - ) - if m_raw and m != "NINGUNO" and m_with_space != "NINGUNO" and not n: - return _err( - line_num, - "NUMERO TRANSPORTE", - "Error: (Celda N{}) El Tipo de Transporte es {} y no está capturado el número de transporte.".format(line_num, m_raw), - ) return None diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py index c3213dad..d79df42e 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py @@ -344,18 +344,6 @@ def _validaciones_transporte(row: Dict[str, Any], line_num: int) -> Optional[Dic "NUMERO TRANSPORTE", f"Error: (Celda N{line_num}) El Tipo de Transporte está vacío y está capturado un número de transporte.", ) - if m == "NINGUNO" and n: - return _err( - line_num, - "NUMERO TRANSPORTE", - f"Error: (Celda N{line_num}) El Tipo de Transporte es NINGUNO y está capturado un número de transporte.", - ) - if m and m != "NINGUNO" and not n: - return _err( - line_num, - "NUMERO TRANSPORTE", - f"Error: (Celda N{line_num}) El Tipo de Transporte es {m} y no está capturado el número de transporte.", - ) return None diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py new file mode 100644 index 00000000..4597e479 --- /dev/null +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py @@ -0,0 +1,228 @@ +"""Mapeo y validación de transporte para CSV (paridad con facturas manuales / validate_common).""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import func +from sqlalchemy.orm import Session + +from api.v1.modules.a76.invoices.models import TransportType + + +def normalize_transport_cell(raw: Optional[str]) -> str: + if raw is None: + return "" + return str(raw).replace("\u00a0", " ").strip() + + +def resolve_transport_csv_fields(row: dict) -> Tuple[str, str, str]: + """ + Prioridad de columnas: + 1) CLAVE TRANSPORTE / NUMERO CAJA + 2) NUMERO TRANSPORTE como fallback legacy. + """ + legacy = normalize_transport_cell(row.get("NUMERO TRANSPORTE")) + clave_transporte = normalize_transport_cell(row.get("CLAVE TRANSPORTE")) or legacy + numero_caja = normalize_transport_cell(row.get("NUMERO CAJA")) or legacy + numero_transporte = legacy + return clave_transporte, numero_caja, numero_transporte + + +def resolve_logistics_transport_fields_for_commit( + session: Session, + tenant_id: int, + company_id: int, + clave_transporte_raw: str, + numero_caja_raw: str, +) -> Tuple[Optional[str], Optional[str], Optional[int], Optional[int]]: + """ + Resuelve claves y IDs internos para logística: + - transport_id / transport_int_id a partir de Vehicle.vehicle_key + - trailer_num / trailer_int_id a partir de Trailer.trailer_number + """ + from api.v1.modules.a76.transportation.vehicles.models import Vehicle + from api.v1.modules.a76.transportation.trailers.models import Trailer + + clave_transporte = normalize_transport_cell(clave_transporte_raw) + numero_caja = normalize_transport_cell(numero_caja_raw) + + transport_id: Optional[str] = clave_transporte or None + trailer_num: Optional[str] = numero_caja or None + transport_int_id: Optional[int] = None + trailer_int_id: Optional[int] = None + + if transport_id: + vehicle = ( + session.query(Vehicle) + .filter( + func.upper(Vehicle.vehicle_key) == transport_id.upper(), + Vehicle.tenant_id == tenant_id, + Vehicle.company_id == company_id, + ) + .first() + ) + if vehicle is not None: + transport_int_id = vehicle.vehicle_id + + if trailer_num: + trailer = ( + session.query(Trailer) + .filter( + func.upper(Trailer.trailer_number) == trailer_num.upper(), + Trailer.tenant_id == tenant_id, + Trailer.company_id == company_id, + ) + .first() + ) + if trailer is not None: + trailer_int_id = trailer.trailer_id + + return transport_id, trailer_num, transport_int_id, trailer_int_id + + +def validate_csv_invoice_logistics_transport( + row_norm: dict, + session: Session, + tenant_id: int, + company_id: int, + transport_type: TransportType, +) -> Optional[str]: + """ + Paridad con validate_common (factura manual): + - Transportista y conductor: solo catálogo si vienen informados. + - Vehículo: solo catálogo si viene CLAVE TRANSPORTE explícita y no resuelve. + - Remolque: obligatorio (y catálogo) si tipo ≠ none; con none, solo catálogo si NUMERO CAJA explícito. + """ + from api.v1.modules.a76.transportation.transporters.models import Transporter + from api.v1.modules.a76.transportation.drivers.models import Driver + + carrier_code = normalize_transport_cell(row_norm.get("CLAVE TRANSPORTISTA")) + carrier_ok = False + if carrier_code: + tr_obj = ( + session.query(Transporter) + .filter( + Transporter.transporter_key == carrier_code, + Transporter.tenant_id == tenant_id, + Transporter.company_id == company_id, + ) + .first() + ) + if not tr_obj: + return "CLAVE TRANSPORTISTA no existe en el catálogo" + carrier_ok = True + + driver_raw = normalize_transport_cell(row_norm.get("NOMBRE CONDUCTOR")) + if carrier_ok and driver_raw: + dr = ( + session.query(Driver) + .filter( + Driver.transporter_key == carrier_code, + Driver.driver_name == driver_raw, + Driver.tenant_id == tenant_id, + Driver.company_id == company_id, + ) + .first() + ) + if not dr: + return ( + "NOMBRE CONDUCTOR: no existe en el catálogo de conductores " + "para el transportista indicado" + ) + + transport_key_csv, trailer_num_csv, _ = resolve_transport_csv_fields(row_norm) + ( + _transport_id_value, + trailer_num_resolved, + transport_int_id, + trailer_int_id, + ) = resolve_logistics_transport_fields_for_commit( + session=session, + tenant_id=tenant_id, + company_id=company_id, + clave_transporte_raw=transport_key_csv, + numero_caja_raw=trailer_num_csv, + ) + + explicit_clave = normalize_transport_cell(row_norm.get("CLAVE TRANSPORTE")) + if explicit_clave and transport_int_id is None: + return "CLAVE TRANSPORTE: vehículo no existe en el catálogo" + + explicit_caja = normalize_transport_cell(row_norm.get("NUMERO CAJA")) + if transport_type == TransportType.NONE: + if explicit_caja and trailer_int_id is None: + return "NUMERO CAJA: remolque no existe en el catálogo" + return None + + if trailer_int_id is None: + if not (trailer_num_resolved and str(trailer_num_resolved).strip()): + return ( + "NUMERO CAJA (remolque) obligatorio cuando TIPO TRANSPORTE es distinto de NINGUNO" + ) + return "NUMERO CAJA: remolque no existe en el catálogo" + + return None + + +def logistics_scan_row_errors( + row_norm: dict, + session: Session, + tenant_id: int, + company_id: int, + line_num: int, +) -> List[Dict[str, Any]]: + """ + Errores de scan (misma lógica que commit: validate_csv_invoice_logistics_transport). + Solo corre si la fila trae algún dato de logística de transporte. + """ + from .encabezados_impo_temp import row_to_transport_type_clarion + + transport_key_csv, trailer_num_csv, _ = resolve_transport_csv_fields(row_norm) + + def _cell(key: str) -> str: + return normalize_transport_cell(row_norm.get(key)) + + has_data = bool( + _cell("TIPO TRANSPORTE") + or _cell("NUMERO TRANSPORTE") + or _cell("CLAVE TRANSPORTE") + or transport_key_csv + or trailer_num_csv + or _cell("CLAVE TRANSPORTISTA") + or _cell("NOMBRE CONDUCTOR") + ) + if not has_data: + return [] + + raw_transport = row_norm.get("TIPO TRANSPORTE") + transport_str = ( + row_to_transport_type_clarion(raw_transport) + or str(raw_transport or "").strip().lower() + or "none" + ) + try: + tt = TransportType(transport_str) + except ValueError: + tt = TransportType.NONE + + msg = validate_csv_invoice_logistics_transport( + row_norm=row_norm, + session=session, + tenant_id=tenant_id, + company_id=company_id, + transport_type=tt, + ) + if not msg: + return [] + return [ + { + "line": line_num, + "col": "LOGISTICA TRANSPORTE", + "msg": msg, + "solution": ( + "Verifica CLAVE TRANSPORTISTA, NOMBRE CONDUCTOR, CLAVE TRANSPORTE, " + "NUMERO CAJA y TIPO TRANSPORTE (remolque obligatorio si no es NINGUNO)." + ), + } + ] diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py index c87adcaa..f9e2c3cf 100644 --- a/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/common/fk_loader.py @@ -25,8 +25,8 @@ def load_trailers_fk_sets( Devuelve: - valid_trailer_type_keys: códigos de trailer_type (GTipoTrailer) - valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas - - state_descriptions_upper: descripciones de estados en mayúsculas (GEstados) - - state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país" + - state_descriptions_upper: descripciones y mex_key en mayúsculas (GEstados) + - state_country_set: (ame_key_pais, token_upper) para description o mex_key - state_ame_to_description: dict clave_ame_upper -> description_upper (opcional; vacío si State no tiene ame_key) """ valid_trailer_type_keys: Set[str] = set() @@ -50,21 +50,35 @@ def load_trailers_fk_sets( valid_country_ame.add((row[0] or "").strip().upper()) for state in session.query(State).all(): + country = ( + session.query(Country) + .filter(Country.m3_key == state.m3_key) + .first() + ) + ame_country = None + if country and (country.ame_key or "").strip(): + ame_country = (country.ame_key or "").strip().upper() + desc = (state.description or "").strip() if desc: state_descriptions_upper.add(desc.upper()) - country = ( - session.query(Country) - .filter(Country.m3_key == state.m3_key) - .first() + if ame_country: + state_country_set.add((ame_country, desc.upper())) + + mex_key = (state.mex_key or "").strip() + if mex_key: + mk = mex_key.upper() + state_descriptions_upper.add(mk) + if ame_country: + state_country_set.add((ame_country, mk)) + if desc: + state_ame_to_description[mk] = desc.upper() + + ame_state = getattr(state, "ame_key", None) + if ame_state and (ame_state or "").strip(): + state_ame_to_description[(ame_state or "").strip().upper()] = ( + desc.upper() if desc else (ame_state or "").strip().upper() ) - if country and (country.ame_key or "").strip(): - state_country_set.add( - ((country.ame_key or "").strip().upper(), desc.upper()) - ) - ame = getattr(state, "ame_key", None) - if ame and (ame or "").strip(): - state_ame_to_description[(ame or "").strip().upper()] = desc.upper() if desc else (ame or "").strip().upper() except Exception as e: logger.warning("Trailers import: could not load FK sets: %s", e) diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py index 86ad4740..4e2f7f32 100644 --- a/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/common/fk_loader.py @@ -24,8 +24,8 @@ def load_transportistas_fk_sets( Devuelve: - existing_transporter_keys: claves de transportistas existentes (tenant/company) en mayúsculas - valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas - - state_descriptions_upper: descripciones de estados en mayúsculas (GEstados) - - state_country_set: set de (ame_key_pais, description_estado_upper) para validar estado pertenece a país + - state_descriptions_upper: descripciones y claves cortas (mex_key) de estados en mayúsculas (GEstados) + - state_country_set: set de (ame_key_pais, token_upper) para description o mex_key """ existing_transporter_keys: Set[str] = set() valid_country_ame: Set[str] = set() @@ -55,18 +55,27 @@ def load_transportistas_fk_sets( valid_country_ame.add((row[0] or "").strip().upper()) for state in session.query(State).all(): + country = ( + session.query(Country) + .filter(Country.m3_key == state.m3_key) + .first() + ) + ame = None + if country and (country.ame_key or "").strip(): + ame = (country.ame_key or "").strip().upper() + desc = (state.description or "").strip() if desc: state_descriptions_upper.add(desc.upper()) - country = ( - session.query(Country) - .filter(Country.m3_key == state.m3_key) - .first() - ) - if country and (country.ame_key or "").strip(): - state_country_set.add( - ((country.ame_key or "").strip().upper(), desc.upper()) - ) + if ame: + state_country_set.add((ame, desc.upper())) + + mex_key = (state.mex_key or "").strip() + if mex_key: + mk = mex_key.upper() + state_descriptions_upper.add(mk) + if ame: + state_country_set.add((ame, mk)) except Exception as e: logger.warning("Transportistas import: could not load FK sets: %s", e) diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py index f76aa77d..1cf2b300 100644 --- a/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/common/fk_loader.py @@ -24,8 +24,8 @@ def load_vehicles_fk_sets( Devuelve: - valid_transport_codes: códigos de transport_types (GTipoTransportes) - valid_country_ame: claves americana de países (GPaises.Pais_Ame), mayúsculas - - state_descriptions_upper: descripciones de estados en mayúsculas (GEstados), para "estado existe" - - state_country_set: set de (ame_key_pais, description_estado_upper) para validar "estado pertenece a país" + - state_descriptions_upper: descripciones y mex_key en mayúsculas (GEstados) + - state_country_set: (ame_key_pais, token_upper) para description o mex_key """ valid_transport_codes: Set[str] = set() valid_country_ame: Set[str] = set() @@ -46,21 +46,28 @@ def load_vehicles_fk_sets( if row[0]: valid_country_ame.add((row[0] or "").strip().upper()) - # States: description (GEstados.Descripcion); State.m3_key = Country.m3_key for state in session.query(State).all(): + country = ( + session.query(Country) + .filter(Country.m3_key == state.m3_key) + .first() + ) + ame = None + if country and (country.ame_key or "").strip(): + ame = (country.ame_key or "").strip().upper() + desc = (state.description or "").strip() if desc: state_descriptions_upper.add(desc.upper()) - # País para este estado vía m3_key - country = ( - session.query(Country) - .filter(Country.m3_key == state.m3_key) - .first() - ) - if country and (country.ame_key or "").strip(): - state_country_set.add( - ((country.ame_key or "").strip().upper(), desc.upper()) - ) + if ame: + state_country_set.add((ame, desc.upper())) + + mex_key = (state.mex_key or "").strip() + if mex_key: + mk = mex_key.upper() + state_descriptions_upper.add(mk) + if ame: + state_country_set.add((ame, mk)) except Exception as e: logger.warning("Vehicles import: could not load FK sets: %s", e) diff --git a/backend/api/v1/modules/a76/transportation/drivers/services.py b/backend/api/v1/modules/a76/transportation/drivers/services.py index d9d9d261..66cc1976 100644 --- a/backend/api/v1/modules/a76/transportation/drivers/services.py +++ b/backend/api/v1/modules/a76/transportation/drivers/services.py @@ -1,9 +1,17 @@ from typing import List, Optional +from sqlalchemy import text from sqlalchemy.orm import Session from . import dto, models +DRIVER_ID_SEQ = "a76.driver_driver_id_seq" + + +def allocate_driver_id(db: Session) -> int: + """Next surrogate driver_id (sequence from migration ca7d3c4e8b2a).""" + return db.execute(text(f"SELECT nextval('{DRIVER_ID_SEQ}')")).scalar() + class DriverService: @staticmethod @@ -34,7 +42,10 @@ class DriverService: @staticmethod def create_driver(db: Session, driver_data: dto.DriverCreateDTO): - new_driver = models.Driver(**driver_data.dict()) + data = driver_data.model_dump() + if data.get("driver_id") is None: + data["driver_id"] = allocate_driver_id(db) + new_driver = models.Driver(**data) db.add(new_driver) db.commit() db.refresh(new_driver) diff --git a/backend/api/v1/modules/a76/transportation/trailers/services.py b/backend/api/v1/modules/a76/transportation/trailers/services.py index 4ffb7c30..d82caa87 100644 --- a/backend/api/v1/modules/a76/transportation/trailers/services.py +++ b/backend/api/v1/modules/a76/transportation/trailers/services.py @@ -1,9 +1,17 @@ from typing import Optional, Tuple, List, Dict, Any from sqlalchemy.orm import Session +from sqlalchemy import text from . import dto, models +TRAILER_ID_SEQ = "a76.trailer_trailer_id_seq" + + +def allocate_trailer_id(db: Session) -> int: + """Next surrogate trailer_id (sequence from migration ca7d3c4e8b2a).""" + return db.execute(text(f"SELECT nextval('{TRAILER_ID_SEQ}')")).scalar() + class TrailerService: """Service for Trailer CRUD operations with tenant support""" @@ -66,8 +74,11 @@ class TrailerService: company_id: int, ) -> models.Trailer: """Create a new trailer""" + data = trailer_data.model_dump() + if data.get("trailer_id") is None: + data["trailer_id"] = allocate_trailer_id(db) new_trailer = models.Trailer( - **trailer_data.model_dump(), tenant_id=tenant_id, company_id=company_id + **data, tenant_id=tenant_id, company_id=company_id ) db.add(new_trailer) db.commit() diff --git a/backend/api/v1/modules/a76/transportation/transporters/services.py b/backend/api/v1/modules/a76/transportation/transporters/services.py index d92880a9..e71a4c68 100644 --- a/backend/api/v1/modules/a76/transportation/transporters/services.py +++ b/backend/api/v1/modules/a76/transportation/transporters/services.py @@ -2,12 +2,19 @@ from typing import Optional, Tuple, List, Dict, Any import logging from sqlalchemy.orm import Session -from sqlalchemy import func +from sqlalchemy import func, text from . import dto, models logger = logging.getLogger(__name__) +TRANSPORTER_ID_SEQ = "a76.transporter_transporter_id_seq" + + +def allocate_transporter_id(db: Session) -> int: + """Next surrogate transporter_id (sequence from migration ca7d3c4e8b2a).""" + return db.execute(text(f"SELECT nextval('{TRANSPORTER_ID_SEQ}')")).scalar() + class TransporterService: """Service for Transporter CRUD operations with tenant support""" @@ -91,8 +98,11 @@ class TransporterService: company_id: int, ) -> models.Transporter: """Create a new transporter""" + data = transporter_data.model_dump() + if data.get("transporter_id") is None: + data["transporter_id"] = allocate_transporter_id(db) new_transporter = models.Transporter( - **transporter_data.model_dump(), tenant_id=tenant_id, company_id=company_id + **data, tenant_id=tenant_id, company_id=company_id ) db.add(new_transporter) db.commit() diff --git a/backend/api/v1/modules/a76/transportation/vehicles/services.py b/backend/api/v1/modules/a76/transportation/vehicles/services.py index 2b9801ac..fd953f27 100644 --- a/backend/api/v1/modules/a76/transportation/vehicles/services.py +++ b/backend/api/v1/modules/a76/transportation/vehicles/services.py @@ -1,9 +1,17 @@ from typing import Optional, Tuple, List, Dict, Any from sqlalchemy.orm import Session +from sqlalchemy import text from . import dto, models +VEHICLE_ID_SEQ = "a76.vehicle_vehicle_id_seq" + + +def allocate_vehicle_id(db: Session) -> int: + """Next surrogate vehicle_id (sequence from migration ca7d3c4e8b2a).""" + return db.execute(text(f"SELECT nextval('{VEHICLE_ID_SEQ}')")).scalar() + class VehicleService: """Service for Vehicle CRUD operations with tenant support""" @@ -66,8 +74,11 @@ class VehicleService: company_id: int, ) -> models.Vehicle: """Create a new vehicle""" + data = vehicle_data.model_dump() + if data.get("vehicle_id") is None: + data["vehicle_id"] = allocate_vehicle_id(db) new_vehicle = models.Vehicle( - **vehicle_data.model_dump(), tenant_id=tenant_id, company_id=company_id + **data, tenant_id=tenant_id, company_id=company_id ) db.add(new_vehicle) db.commit() diff --git a/backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py b/backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py new file mode 100644 index 00000000..e28b42e5 --- /dev/null +++ b/backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py @@ -0,0 +1,130 @@ +"""Smoke: loaders CSV incluyen State.mex_key en validación de ESTADO/PAÍS.""" +from unittest.mock import MagicMock, patch + + +def _fake_state(m3_key="MEX", description="Chihuahua", mex_key="CHH"): + s = MagicMock() + s.m3_key = m3_key + s.description = description + s.mex_key = mex_key + return s + + +def _fake_country(m3_key="MEX", ame_key="MX"): + c = MagicMock() + c.m3_key = m3_key + c.ame_key = ame_key + return c + + +def _session_with_state_queries(fake_state, fake_country): + """Simula session.query en el orden de transportistas/vehicles/trailers loaders.""" + n = [0] + + def query_side_effect(*_args, **_kwargs): + n[0] += 1 + m = MagicMock() + if n[0] == 1: + m.filter.return_value.all.return_value = [] + elif n[0] == 2: + m.all.return_value = [("MX",)] + elif n[0] == 3: + m.all.return_value = [fake_state] + elif n[0] == 4: + m.filter.return_value.first.return_value = fake_country + return m + + session = MagicMock() + session.query.side_effect = query_side_effect + return session + + +@patch("api.v1.modules.a76.layouts_csv.transportistas.common.fk_loader.CoreSessionLocal") +def test_transportistas_fk_sets_includes_mex_key(mock_session_local): + from api.v1.modules.a76.layouts_csv.transportistas.common.fk_loader import ( + load_transportistas_fk_sets, + ) + + st = _fake_state() + mock_session_local.return_value.__enter__.return_value = _session_with_state_queries( + st, _fake_country() + ) + + _, _, state_desc, state_cc = load_transportistas_fk_sets(1, 1) + + assert "CHH" in state_desc + assert "CHIHUAHUA" in state_desc + assert ("MX", "CHH") in state_cc + assert ("MX", "CHIHUAHUA") in state_cc + + +def _session_vehicles(fake_state, fake_country): + n = [0] + + def query_side_effect(*_args, **_kwargs): + n[0] += 1 + m = MagicMock() + if n[0] == 1: + m.all.return_value = [("VH",)] + elif n[0] == 2: + m.all.return_value = [("MX",)] + elif n[0] == 3: + m.all.return_value = [fake_state] + elif n[0] == 4: + m.filter.return_value.first.return_value = fake_country + return m + + session = MagicMock() + session.query.side_effect = query_side_effect + return session + + +@patch("api.v1.modules.a76.layouts_csv.vehicles.common.fk_loader.CoreSessionLocal") +def test_vehicles_fk_sets_includes_mex_key(mock_session_local): + from api.v1.modules.a76.layouts_csv.vehicles.common.fk_loader import load_vehicles_fk_sets + + mock_session_local.return_value.__enter__.return_value = _session_vehicles( + _fake_state(), _fake_country() + ) + + _, _, state_desc, state_cc = load_vehicles_fk_sets(1, 1) + + assert "CHH" in state_desc + assert ("MX", "CHH") in state_cc + + +def _session_trailers(fake_state, fake_country): + n = [0] + + def query_side_effect(*_args, **_kwargs): + n[0] += 1 + m = MagicMock() + if n[0] == 1: + m.all.return_value = [("BX",)] + elif n[0] == 2: + m.all.return_value = [("MX",)] + elif n[0] == 3: + fake_state.ame_key = None + m.all.return_value = [fake_state] + elif n[0] == 4: + m.filter.return_value.first.return_value = fake_country + return m + + session = MagicMock() + session.query.side_effect = query_side_effect + return session + + +@patch("api.v1.modules.a76.layouts_csv.trailers.common.fk_loader.CoreSessionLocal") +def test_trailers_fk_sets_includes_mex_key(mock_session_local): + from api.v1.modules.a76.layouts_csv.trailers.common.fk_loader import load_trailers_fk_sets + + mock_session_local.return_value.__enter__.return_value = _session_trailers( + _fake_state(), _fake_country() + ) + + _, _, state_desc, state_cc, ame_map = load_trailers_fk_sets(1, 1) + + assert "CHH" in state_desc + assert ("MX", "CHH") in state_cc + assert ame_map.get("CHH") == "CHIHUAHUA" diff --git a/docs/analisis_flujo_migracion_csv_transporte_sin_validaciones.md b/docs/analisis_flujo_migracion_csv_transporte_sin_validaciones.md new file mode 100644 index 00000000..730c66ae --- /dev/null +++ b/docs/analisis_flujo_migracion_csv_transporte_sin_validaciones.md @@ -0,0 +1,141 @@ +# Analisis de flujo de migracion CSV de transporte (sin validaciones) + +## Objetivo + +Definir como migrar el flujo de importacion CSV de facturas hacia columnas nuevas de transporte, priorizando continuidad operativa y consistencia de IDs en `invoice_logistics`, sin depender de reglas de validacion funcional. + +## Estado actual de la rama + +- El pipeline actual ya separa `scan` y `commit` en `tasks.py`. +- El mapeo de plantillas (`template_config.py`) aun esta centrado en `NUMERO TRANSPORTE`. +- En `commit`, la persistencia de logistica usa principalmente: + - `carrier_id` / `carrier_int_id` desde `CLAVE TRANSPORTISTA`. + - `transport_type` desde `TIPO TRANSPORTE`. + - `transport_num` desde `NUMERO TRANSPORTE`. +- No existe aun una ruta establecida para columnas nuevas `CLAVE TRANSPORTE` y `NUMERO CAJA`. + +## Intencion funcional identificada en los commits analizados + +Sin copiar su logica de validacion, la idea de flujo que se quiso introducir es: + +1. Separar los datos de transporte en dos entradas semanticas: + - clave de vehiculo (`CLAVE TRANSPORTE`) + - numero de caja/remolque (`NUMERO CAJA`) +2. Mantener compatibilidad con archivos legacy: + - usar `NUMERO TRANSPORTE` como fallback cuando no existan columnas nuevas. +3. Resolver IDs internos en commit contra catalogos: + - `vehicle_key` -> `vehicle_id` + - `trailer_number` -> `trailer_id` +4. Persistir tanto la clave legible (string) como su ID interno (int) en `invoice_logistics`. +5. Mantener paridad de flujo entre scan y commit en cuanto a normalizacion y resolucion de campos, pero sin convertir scan en un bloqueo por validaciones de negocio. + +## Flujo propuesto (sin validaciones de negocio) + +### 1) Scan (preprocesamiento y trazabilidad) + +Objetivo: preparar datos y metadatos, no rechazar por reglas funcionales. + +- Leer CSV con `row_from_template`. +- Normalizar celdas relevantes de transporte: + - trim + - reemplazo de NBSP por espacio + - `None`/vacio a cadena vacia +- Construir una estructura por renglon con campos de transporte efectivos: + - `effective_transport_type` + - `effective_vehicle_key` + - `effective_trailer_number` + - `effective_legacy_transport_num` (solo trazabilidad) +- Guardar errores tecnicos (parseo CSV, formato bruto no interpretable), pero no bloquear por reglas de catalogo/negocio. + +### 2) Commit (resolucion y persistencia de IDs) + +Objetivo: persistir de forma consistente los IDs nuevos de migracion. + +- Determinar campos efectivos por precedencia (ver tabla siguiente). +- Resolver `carrier_int_id` con `CLAVE TRANSPORTISTA` (flujo ya existente). +- Resolver `transport_int_id` cuando exista `effective_vehicle_key`: + - lookup en `vehicle.vehicle_key` +- Resolver `trailer_int_id` cuando exista `effective_trailer_number`: + - lookup en `trailer.trailer_number` +- Persistir en `InvoiceLogistics`: + - string keys: `carrier_id`, `transport_id`, `trailer_num`, `transport_num` + - int refs: `carrier_int_id`, `transport_int_id`, `trailer_int_id` + +Importante: para migracion, si hay string pero no hay match de ID, **no romper flujo**; persistir string y dejar int en `NULL` (el FK ya permite `SET NULL`). + +## Matriz de mapeo CSV -> `invoice_logistics` + +| Entrada CSV | Rol | Campo destino (string) | Campo destino (int) | Catalogo/lookup | +|---|---|---|---|---| +| `CLAVE TRANSPORTISTA` | Transportista | `carrier_id` | `carrier_int_id` | `transporter.transporter_key -> transporter_id` | +| `CLAVE TRANSPORTE` | Vehiculo | `transport_id` | `transport_int_id` | `vehicle.vehicle_key -> vehicle_id` | +| `NUMERO CAJA` | Caja/Remolque | `trailer_num` | `trailer_int_id` | `trailer.trailer_number -> trailer_id` | +| `NUMERO TRANSPORTE` (legacy) | Fallback | `transport_num` (siempre trazable) y apoyo para resolver vehicle/trailer segun precedencia | opcional | depende de reglas de precedencia | +| `TIPO TRANSPORTE` | Tipo logistica | `transport_type` | n/a | enum interno | + +## Reglas de precedencia de datos (nuevas vs legacy) + +Definicion para no generar ambiguedad: + +1. Si viene `CLAVE TRANSPORTE`, usarla para `transport_id`. +2. Si viene `NUMERO CAJA`, usarla para `trailer_num`. +3. Si faltan columnas nuevas y viene `NUMERO TRANSPORTE`: + - usarlo como `transport_num` (trazabilidad legacy) + - y usarlo como fallback de resolucion para `transport_id`/`trailer_num` solo cuando el campo nuevo correspondiente este vacio. +4. Nunca sobreescribir un dato nuevo con legacy si el nuevo viene poblado. + +Sugerencia para parciales (`actualizar=true`): +- aplicar merge campo a campo: solo actualizar datos de transporte que lleguen informados en CSV; conservar los demas en la fila existente. + +## Relacion con la migracion de IDs + +La migracion `ca7d3c4e8b2a` ya formaliza: + +- `carrier_int_id` -> FK a `transporter.transporter_id` +- `transport_int_id` -> FK a `vehicle.vehicle_id` +- `trailer_int_id` -> FK a `trailer.trailer_id` + +Por lo tanto, el flujo CSV debe priorizar: + +- resolver claves string de catalogo de forma determinista +- poblar IDs internos cuando haya match +- mantener string keys para trazabilidad y backfill futuro + +## Impacto de implementacion por archivo (sin validaciones) + +### `backend/api/v1/modules/a76/layouts_csv/facturas/template_config.py` + +- Agregar columnas canonicas nuevas en encabezados: + - `CLAVE TRANSPORTE` + - `NUMERO CAJA` +- Mantener `NUMERO TRANSPORTE` como compatibilidad legacy (no eliminar de inmediato). +- Ajustar aliases para tolerar variantes de cabecera. + +### `backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py` + +- Incorporar helper de resolucion de campos efectivos de transporte: + - prioridad nuevas columnas + - fallback legacy +- En `scan`, registrar estructura normalizada (sin rechazo por reglas de catalogo). +- En `commit`, poblar `InvoiceLogistics` con: + - `transport_id`/`transport_int_id` + - `trailer_num`/`trailer_int_id` + - `transport_num` como legado/trazabilidad +- Mantener comportamiento de no falla por ausencia de match de IDs. + +### Nuevo helper recomendado: `backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py` + +- Aunque no se usen validaciones de negocio, centralizar funciones de: + - normalizacion de celdas + - resolucion de keys efectivas + - lookups de IDs de vehiculo/remolque +- Evita duplicar logica entre scan y commit. + +## Criterios de aceptacion de esta migracion de flujo + +- Queda definida una sola fuente de verdad para precedencia de columnas nuevas/legacy. +- `commit` persiste consistentemente claves string e IDs int de logistica. +- El flujo funciona aun cuando no haya match de catalogo (sin bloqueo por validacion). +- No hay ambiguedad entre: + - `transport_id` vs `transport_num` + - `transport_int_id` vs `trailer_int_id` diff --git a/frontend/src/lib/components/dashboard/invoices/edit/general-tab-form.svelte b/frontend/src/lib/components/dashboard/invoices/edit/general-tab-form.svelte index 6edc6d3d..9cfe1c10 100644 --- a/frontend/src/lib/components/dashboard/invoices/edit/general-tab-form.svelte +++ b/frontend/src/lib/components/dashboard/invoices/edit/general-tab-form.svelte @@ -95,7 +95,7 @@ transport_id: invoice.logistics?.transport_id || '', driver_name: invoice.logistics?.driver_name || '', transport_type: invoice.logistics?.transport_type || '', - transport_num: invoice.logistics?.vehicle_num || '', + trailer_num: invoice.logistics?.trailer_num || '', aduana: invoice.compliance_mx?.aduana || '', document_type: invoice.document_type || '', manifest_number: invoice.compliance_mx?.manifest_number || '', @@ -127,7 +127,7 @@ transport_id: '', driver_name: '', transport_type: '', - transport_num: '', + trailer_num: '', aduana: '', document_type: '', manifest_number: '', @@ -161,6 +161,10 @@ if (formData.electronic_signature === undefined) { formData.electronic_signature = ''; } + if (formData.trailer_num === undefined && (formData as any).transport_num) { + formData.trailer_num = (formData as any).transport_num; + delete (formData as any).transport_num; + } } // Opciones de tipo de peso @@ -213,6 +217,52 @@ ] ); + /** Conductores del transportista elegido (evita opciones inválidas ante validate_common). */ + const driversForCarrier = $derived.by(() => { + if (invoiceType === 'MEX') { + return drivers; + } + const cid = + formData.carrier_id !== null && formData.carrier_id !== undefined + ? String(formData.carrier_id).trim() + : ''; + if (!cid) { + return []; + } + return drivers.filter((d) => String(d.transporter_key) === cid); + }); + + $effect(() => { + if (invoiceType === 'MEX') { + return; + } + const cid = + formData.carrier_id !== null && formData.carrier_id !== undefined + ? String(formData.carrier_id).trim() + : ''; + if (!cid) { + if (formData.driver_name) { + formData.driver_name = ''; + } + if (formData.trailer_num) { + formData.trailer_num = ''; + } + return; + } + if (!drivers.length) { + return; + } + if (formData.driver_name) { + const ok = drivers.some( + (d) => + String(d.transporter_key) === cid && d.driver_name === formData.driver_name + ); + if (!ok) { + formData.driver_name = ''; + } + } + }); + const shippedByHeaderOptions = $derived( operationType === 1 || invoiceType === 'CR' ? [ @@ -683,6 +733,8 @@ value={formData.carrier_id ? String(formData.carrier_id) : ''} onValueChange={(v) => { formData.carrier_id = v || null; + formData.driver_name = ''; + formData.trailer_num = ''; }} > @@ -747,6 +799,8 @@ { formData.driver_name = v ?? ''; }} @@ -755,7 +809,11 @@ {#if formData.driver_name} {formData.driver_name} - {:else if drivers.length > 0} + {:else if invoiceType !== 'MEX' && !formData.carrier_id} + Primero elige transportista... + {:else if invoiceType !== 'MEX' && driversForCarrier.length === 0} + Sin conductores para este transportista + {:else if driversForCarrier.length > 0} Selecciona conductor... {:else} Sin datos @@ -763,8 +821,8 @@ - {#each drivers as driver} - + {#each driversForCarrier as driver} + {driver.driver_name} {/each} @@ -799,19 +857,25 @@
- + { - formData.transport_num = v ?? ''; + formData.trailer_num = v ?? ''; }} > - + - {#if formData.transport_num} - {trailers.find((t) => t.trailer_number === formData.transport_num) - ?.plate_number || formData.transport_num} + {#if formData.trailer_num} + {trailers.find((t) => t.trailer_number === formData.trailer_num) + ?.plate_number || formData.trailer_num} + {:else if invoiceType !== 'MEX' && !formData.carrier_id} + Primero elige transportista... {:else if trailers.length > 0} Selecciona remolque... {:else} diff --git a/frontend/src/lib/components/dashboard/invoices/edit/save-invoice.ts b/frontend/src/lib/components/dashboard/invoices/edit/save-invoice.ts index 8cd4db12..bb555672 100644 --- a/frontend/src/lib/components/dashboard/invoices/edit/save-invoice.ts +++ b/frontend/src/lib/components/dashboard/invoices/edit/save-invoice.ts @@ -1,5 +1,5 @@ import { goto } from '$app/navigation'; -import { invoicesApi, type CreateInvoiceData, type UpdateInvoiceData, type OperationType } from '$lib/api/dashboard/a76/invoices'; +import { invoicesApi, type CreateInvoiceData, type UpdateInvoiceData } from '$lib/api/dashboard/a76/invoices'; interface FormDataSet { InvoiceTopFieldsFormData: any; @@ -30,6 +30,28 @@ interface SaveInvoiceResult { }>; } +function _logisticsStrNonempty(value: unknown): boolean { + return value != null && String(value).trim() !== ''; +} + +/** Remolque obligatorio si tipo != none (mismo criterio que validate_common). */ +function validateClientLogistics(formData: FormDataSet): SaveInvoiceResult['validationErrors'] { + const g = formData.generalFormData; + const transportType = _logisticsStrNonempty(g?.transport_type) ? String(g.transport_type).trim() : 'none'; + if (transportType === 'none' || _logisticsStrNonempty(g?.trailer_num)) { + return undefined; + } + return [ + { + field: 'logistics.trailer_num', + message: + 'El Remolque es obligatorio cuando el Tipo de Transporte es distinto de Ninguno.', + code: 'REQUIRED_FIELD', + value: g?.trailer_num + } + ]; +} + export async function saveInvoice(options: SaveInvoiceOptions): Promise { const { invoiceId, isCreate, companyId, formData } = options; const { InvoiceTopFieldsFormData, generalFormData, observationFormData, itemsFormData, othersFormData, continuationFormData } = formData; @@ -66,6 +88,13 @@ export async function saveInvoice(options: SaveInvoiceOptions): Promise value !== undefined) ); + // Compat: remolque vivía en transport_num en layouts antiguos + if (filtered.trailer_num === undefined && filtered.transport_num !== undefined) { + filtered.trailer_num = filtered.transport_num; + delete filtered.transport_num; + } return { ...skeleton, ...filtered }; } @@ -274,7 +279,7 @@ transport_id: invoice.logistics?.transport_id || '', driver_name: invoice.logistics?.driver_name || '', transport_type: invoice.logistics?.transport_type || '', - transport_num: invoice.logistics?.vehicle_num || '', + trailer_num: invoice.logistics?.trailer_num || '', aduana: invoice.compliance_mx?.aduana || '', document_type: invoice.document_type || '' }; diff --git a/frontend/src/routes/dashboard/invoices/settings/+page.svelte b/frontend/src/routes/dashboard/invoices/settings/+page.svelte index f8970eb7..df10a010 100644 --- a/frontend/src/routes/dashboard/invoices/settings/+page.svelte +++ b/frontend/src/routes/dashboard/invoices/settings/+page.svelte @@ -163,7 +163,7 @@ transport_id: '', driver_name: '', transport_type: '', - transport_num: '', + trailer_num: '', aduana: '', document_type: '', ...(settings.generalFormData || {}) @@ -266,7 +266,7 @@ transport_id: '', driver_name: '', transport_type: '', - transport_num: '', + trailer_num: '', aduana: '', document_type: '' }; From a08a62bc31f843284262468472fd75b360ad2146 Mon Sep 17 00:00:00 2001 From: hreyes Date: Fri, 27 Mar 2026 14:57:38 -0600 Subject: [PATCH 04/32] feature/alineacion-validaciones-carga-csv-con-manual-invoices --- .../modules/a76/layouts_csv/facturas/tasks.py | 221 +++++++++++------- .../validators/encabezados_impo_temp.py | 10 + .../facturas/validators/transport_catalog.py | 11 + .../test_fk_loader_state_mex_key.py | 130 ----------- 4 files changed, 152 insertions(+), 220 deletions(-) delete mode 100644 backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index 4169392b..f2fda307 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -27,10 +27,20 @@ from ..common import storage as common_storage from ..common import meta as common_meta from ..common import responses as common_responses from .template_config import row_from_template +from .validators.encabezados_impo_temp import csv_tipo_moneda_es_me_mn_mc # Models are imported inside tasks to avoid circular dependencies and mapper initialization issues in the API process logger = logging.getLogger(__name__) + +def _merge_unique_invoice_scan_error_lines( + precheck_lines: Set[int], + row_validation_lines: List[int], +) -> List[int]: + """Unión deduplicada: líneas marcadas en precheck + líneas con error en validación por fila (imp_temp_header).""" + return sorted(set(precheck_lines) | set(row_validation_lines)) + + # Job type vacío para facturas (prefijo Redis "import_" sin tipo, ver common/storage.py) JOB_TYPE = "" @@ -2782,7 +2792,31 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = } ) + # No return temprano: el commit siempre valida por fila (logística, etc.). Si solo se + # devolvía el precheck, valid_rows del scan quedaba inflado vs inserted/skipped del commit. + scan_precheck_message: Optional[str] = None if precheck_errors: + scan_precheck_message = ( + "Precheck de referencias falló. Corrige catálogos/pedimentos según los errores indicados. " + "Se aplicó también validación completa por fila (logística, etc.) para paridad con el commit." + ) + + processed_rows = 0 + error_lines_list: List[int] = [] + errors_detail: List[Dict[str, Any]] = [] + for e in precheck_errors: + if len(errors_detail) < 5000: + errors_detail.append( + { + "line": e["line"], + "col": e.get("col", ""), + "msg": e.get("msg", ""), + "solution": e.get("solution", ""), + "warning": False, + } + ) + + with CoreSessionLocal() as session: with open(error_path, "w", encoding="utf-8") as f_err: for e in precheck_errors: f_err.write( @@ -2796,99 +2830,96 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = ) + "\n" ) - unique_precheck_lines = sorted(precheck_lines) - common_storage.store_error_lines(effective_job_type, job_id, unique_precheck_lines) - return common_responses.scan_result( - job_id, - len(rows_list), - len(unique_precheck_lines), - precheck_errors, - total_rows_in_file=total_rows, - message="Precheck de referencias falló. Corrige catálogos/pedimentos antes de confirmar importación.", - ) - - error_count = 0 - processed_rows = 0 - error_lines_list: List[int] = [] - errors_detail: List[Dict[str, Any]] = [] - - with open(error_path, "w", encoding="utf-8") as f_err: - for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) - row_norm = row_from_template(row, "imp_temp_header", normalize_header) - warnings_row: List[Dict[str, Any]] = [] - row_errors = validate_row_encabezados_impo_temp( - row_norm, - i, - actualizar=actualizar, - invoice_exists_by_number=invoice_exists_by_number, - invoice_processed_by_number=invoice_processed_by_number, - pedimento_rows=pedimento_rows, - remesa_por_pedimento_bd=remesa_por_pedimento_bd, - remesa_por_pedimento_csv=remesa_por_pedimento_csv, - valid_provider_ids=valid_provider_ids, - valid_sold_to_ids=valid_sold_to_ids, - valid_shipped_to_ids=valid_shipped_to_ids, - valid_provider_short_names=valid_provider_short_names, - valid_sold_to_short_names=valid_sold_to_short_names, - valid_shipped_to_short_names=valid_shipped_to_short_names, - valid_broker_ids=valid_broker_ids, - valid_broker_claves=valid_broker_claves, - valid_transporter_keys=valid_transporter_keys, - valid_incoterms=valid_incoterms, - valid_aduana_codes=valid_aduana_codes, - valid_currency_codes=valid_currency_codes, - exchange_rate_by_date=exchange_rate_by_date, - invoice_has_partidas_by_number=invoice_has_partidas_by_number, - existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, - autonumerar_remesas=autonumerar_remesas, - control_remesa=control_remesa, - remesa_inicio=remesa_inicio, - remesa_fin=remesa_fin, - date_format=date_format, - parse_date_fn=parse_date, - warnings=warnings_row, - ) - row_errors.extend( - logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i) - ) - blocking = [e for e in row_errors if not e.get("warning")] - if blocking: - error_count += 1 - error_lines_list.append(i) - for e in blocking: - f_err.write( - json.dumps( + for i, row in enumerate(rows_list, start=1): + if i % 1000 == 0: + self.update_state( + state="PROGRESS", + meta={ + "current": i, + "total": total_rows, + "errors": len( + _merge_unique_invoice_scan_error_lines( + precheck_lines, error_lines_list + ) + ), + }, + ) + row_norm = row_from_template(row, "imp_temp_header", normalize_header) + warnings_row: List[Dict[str, Any]] = [] + row_errors = validate_row_encabezados_impo_temp( + row_norm, + i, + actualizar=actualizar, + invoice_exists_by_number=invoice_exists_by_number, + invoice_processed_by_number=invoice_processed_by_number, + pedimento_rows=pedimento_rows, + remesa_por_pedimento_bd=remesa_por_pedimento_bd, + remesa_por_pedimento_csv=remesa_por_pedimento_csv, + valid_provider_ids=valid_provider_ids, + valid_sold_to_ids=valid_sold_to_ids, + valid_shipped_to_ids=valid_shipped_to_ids, + valid_provider_short_names=valid_provider_short_names, + valid_sold_to_short_names=valid_sold_to_short_names, + valid_shipped_to_short_names=valid_shipped_to_short_names, + valid_broker_ids=valid_broker_ids, + valid_broker_claves=valid_broker_claves, + valid_transporter_keys=valid_transporter_keys, + valid_incoterms=valid_incoterms, + valid_aduana_codes=valid_aduana_codes, + valid_currency_codes=valid_currency_codes, + exchange_rate_by_date=exchange_rate_by_date, + invoice_has_partidas_by_number=invoice_has_partidas_by_number, + existing_tipo_moneda_by_number=existing_tipo_moneda_by_number, + autonumerar_remesas=autonumerar_remesas, + control_remesa=control_remesa, + remesa_inicio=remesa_inicio, + remesa_fin=remesa_fin, + date_format=date_format, + parse_date_fn=parse_date, + warnings=warnings_row, + ) + row_errors.extend( + logistics_scan_row_errors(row_norm, session, tenant_id, company_id, i) + ) + blocking = [e for e in row_errors if not e.get("warning")] + if blocking: + error_lines_list.append(i) + for e in blocking: + f_err.write( + json.dumps( + { + "line": e["line"], + "col": e.get("col", ""), + "msg": e.get("msg", ""), + "solution": e.get("solution", ""), + } + ) + + "\n" + ) + for e in row_errors + warnings_row: + if len(errors_detail) < 5000: + errors_detail.append( { "line": e["line"], "col": e.get("col", ""), "msg": e.get("msg", ""), "solution": e.get("solution", ""), + "warning": bool(e.get("warning", False)), } ) - + "\n" - ) - for e in row_errors + warnings_row: - if len(errors_detail) < 5000: - errors_detail.append( - { - "line": e["line"], - "col": e.get("col", ""), - "msg": e.get("msg", ""), - "solution": e.get("solution", ""), - "warning": bool(e.get("warning", False)), - } - ) - processed_rows += 1 + processed_rows += 1 - # El commit usa líneas únicas para omitir filas; el resumen preliminar - # debe usar la misma base para evitar discrepancias de válidos/errores. - unique_error_lines = sorted(set(error_lines_list)) + # El commit usa líneas únicas para omitir filas; unir precheck + validación por fila. + unique_error_lines = _merge_unique_invoice_scan_error_lines(precheck_lines, error_lines_list) error_count = len(unique_error_lines) common_storage.store_error_lines(effective_job_type, job_id, unique_error_lines) return common_responses.scan_result( - job_id, processed_rows, error_count, errors_detail, total_rows_in_file=total_rows + job_id, + processed_rows, + error_count, + errors_detail, + total_rows_in_file=total_rows, + message=scan_precheck_message, ) except Exception as e: logger.exception("Encabezados importación temporal scan failed: %s", e) @@ -4036,6 +4067,8 @@ def validate_row_phase_1( def check_currency(col_name): val = row.get(col_name) if val and str(val).strip(): + if csv_tipo_moneda_es_me_mn_mc(str(val)): + return None parsed_currency = parse_currency(val, None) val_norm = normalize_header(val) # parse_currency returns MANUAL if unknown, so if it wasn't explicitly MANUAL, it's invalid @@ -4430,6 +4463,13 @@ def parse_currency(value: Optional[str], currency_type: Optional[str]): return Currency.FOREIGN if "MANUAL" in normalized: return Currency.MANUAL + # MC (moneda por clave): paridad con encabezados_impo_temp / InvoiceFinancials + CLAVE MONEDA + if normalized.replace(" ", "") == "MC": + if currency_type and str(currency_type).strip().upper() == "MXN": + return Currency.LOCAL + if currency_type: + return Currency.FOREIGN + return Currency.FOREIGN if currency_type and str(currency_type).strip().upper() == "MXN": return Currency.LOCAL if currency_type: @@ -5870,14 +5910,15 @@ def _do_insert_valid_rows(job_id: str, model_target: str, job_type_override: Opt currency_val = row_norm.get('TIPO MONEDA') if currency_val and str(currency_val).strip(): - parsed_currency = parse_currency(currency_val, None) - val_norm = normalize_header(currency_val) - if parsed_currency.value == "manual" and "MANUAL" not in val_norm: - skipped_invalid += 1 - reason = "TIPO MONEDA: Moneda invalida" - skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) - logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") - continue + if not csv_tipo_moneda_es_me_mn_mc(str(currency_val)): + parsed_currency = parse_currency(currency_val, None) + val_norm = normalize_header(currency_val) + if parsed_currency.value == "manual" and "MANUAL" not in val_norm: + skipped_invalid += 1 + reason = "TIPO MONEDA: Moneda invalida" + skipped_fk_details.append({"line": i, "invoice": invoice_number, "reason": reason}) + logger.warning(f"Row {i} (Invoice {invoice_number}): {reason}") + continue # 2. Client/Provider and broker checks are handled above diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py index d79df42e..bd6910ab 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/encabezados_impo_temp.py @@ -19,6 +19,16 @@ REGIMENES_VALIDOS = frozenset({"ITE", "ITR"}) TIPOS_MONEDA_VALIDOS = frozenset({"ME", "MN", "MC"}) TIPO_PESO_VALIDOS = frozenset({"KILOS", "LIBRAS"}) + +def csv_tipo_moneda_es_me_mn_mc(value: Any) -> bool: + """ + ME / MN / MC con la misma regla que _validaciones_moneda (strip + upper). + Compartido entre scan (encabezados) y commit / validate_row_strict en tasks. + """ + if value is None or not str(value).strip(): + return False + return str(value).strip().upper() in TIPOS_MONEDA_VALIDOS + # Clarion Col M → valor normalizado (minúscula para TransportType enum) TIPO_TRANSPORTE_CLARION_TO_NORM = { "NINGUNO": "none", diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py index 4597e479..9f6151e3 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/validators/transport_catalog.py @@ -165,6 +165,16 @@ def validate_csv_invoice_logistics_transport( return None +def _csv_row_has_resolved_weight_unit(row_norm: dict) -> bool: + """ + Paridad con insert_valid_rows: has_logistics_data incluye parse_weight_unit(TIPO PESO). + Import diferido para evitar ciclo al cargar tasks. + """ + from api.v1.modules.a76.layouts_csv.facturas.tasks import parse_weight_unit + + return parse_weight_unit(row_norm.get("TIPO PESO")) is not None + + def logistics_scan_row_errors( row_norm: dict, session: Session, @@ -191,6 +201,7 @@ def logistics_scan_row_errors( or trailer_num_csv or _cell("CLAVE TRANSPORTISTA") or _cell("NOMBRE CONDUCTOR") + or _csv_row_has_resolved_weight_unit(row_norm) ) if not has_data: return [] diff --git a/backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py b/backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py deleted file mode 100644 index e28b42e5..00000000 --- a/backend/tests/unit/layouts_csv/test_fk_loader_state_mex_key.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Smoke: loaders CSV incluyen State.mex_key en validación de ESTADO/PAÍS.""" -from unittest.mock import MagicMock, patch - - -def _fake_state(m3_key="MEX", description="Chihuahua", mex_key="CHH"): - s = MagicMock() - s.m3_key = m3_key - s.description = description - s.mex_key = mex_key - return s - - -def _fake_country(m3_key="MEX", ame_key="MX"): - c = MagicMock() - c.m3_key = m3_key - c.ame_key = ame_key - return c - - -def _session_with_state_queries(fake_state, fake_country): - """Simula session.query en el orden de transportistas/vehicles/trailers loaders.""" - n = [0] - - def query_side_effect(*_args, **_kwargs): - n[0] += 1 - m = MagicMock() - if n[0] == 1: - m.filter.return_value.all.return_value = [] - elif n[0] == 2: - m.all.return_value = [("MX",)] - elif n[0] == 3: - m.all.return_value = [fake_state] - elif n[0] == 4: - m.filter.return_value.first.return_value = fake_country - return m - - session = MagicMock() - session.query.side_effect = query_side_effect - return session - - -@patch("api.v1.modules.a76.layouts_csv.transportistas.common.fk_loader.CoreSessionLocal") -def test_transportistas_fk_sets_includes_mex_key(mock_session_local): - from api.v1.modules.a76.layouts_csv.transportistas.common.fk_loader import ( - load_transportistas_fk_sets, - ) - - st = _fake_state() - mock_session_local.return_value.__enter__.return_value = _session_with_state_queries( - st, _fake_country() - ) - - _, _, state_desc, state_cc = load_transportistas_fk_sets(1, 1) - - assert "CHH" in state_desc - assert "CHIHUAHUA" in state_desc - assert ("MX", "CHH") in state_cc - assert ("MX", "CHIHUAHUA") in state_cc - - -def _session_vehicles(fake_state, fake_country): - n = [0] - - def query_side_effect(*_args, **_kwargs): - n[0] += 1 - m = MagicMock() - if n[0] == 1: - m.all.return_value = [("VH",)] - elif n[0] == 2: - m.all.return_value = [("MX",)] - elif n[0] == 3: - m.all.return_value = [fake_state] - elif n[0] == 4: - m.filter.return_value.first.return_value = fake_country - return m - - session = MagicMock() - session.query.side_effect = query_side_effect - return session - - -@patch("api.v1.modules.a76.layouts_csv.vehicles.common.fk_loader.CoreSessionLocal") -def test_vehicles_fk_sets_includes_mex_key(mock_session_local): - from api.v1.modules.a76.layouts_csv.vehicles.common.fk_loader import load_vehicles_fk_sets - - mock_session_local.return_value.__enter__.return_value = _session_vehicles( - _fake_state(), _fake_country() - ) - - _, _, state_desc, state_cc = load_vehicles_fk_sets(1, 1) - - assert "CHH" in state_desc - assert ("MX", "CHH") in state_cc - - -def _session_trailers(fake_state, fake_country): - n = [0] - - def query_side_effect(*_args, **_kwargs): - n[0] += 1 - m = MagicMock() - if n[0] == 1: - m.all.return_value = [("BX",)] - elif n[0] == 2: - m.all.return_value = [("MX",)] - elif n[0] == 3: - fake_state.ame_key = None - m.all.return_value = [fake_state] - elif n[0] == 4: - m.filter.return_value.first.return_value = fake_country - return m - - session = MagicMock() - session.query.side_effect = query_side_effect - return session - - -@patch("api.v1.modules.a76.layouts_csv.trailers.common.fk_loader.CoreSessionLocal") -def test_trailers_fk_sets_includes_mex_key(mock_session_local): - from api.v1.modules.a76.layouts_csv.trailers.common.fk_loader import load_trailers_fk_sets - - mock_session_local.return_value.__enter__.return_value = _session_trailers( - _fake_state(), _fake_country() - ) - - _, _, state_desc, state_cc, ame_map = load_trailers_fk_sets(1, 1) - - assert "CHH" in state_desc - assert ("MX", "CHH") in state_cc - assert ame_map.get("CHH") == "CHIHUAHUA" From 4edeaa32731b0408c195d3d7fb8e5816a6a1e770 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:09:58 -0500 Subject: [PATCH 05/32] Implement sorting mechanism for tables to enhance data organization and user experience. --- Jenkinsfile | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..88280eae --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,175 @@ +pipeline { + agent any + + options { + timestamps() + disableConcurrentBuilds() + } + + parameters { + choice(name: 'ENVIRONMENT', choices: ['development', 'main'], description: 'Rama a ejecutar') + } + + environment { + REGISTRY = 'dev.aduanasoft.com' + IMAGE_NAMESPACE = 'anexo76' + VERSION_MAYOR = credentials('version-mayor') // secret text, ej: 1 + } + + stages { + stage('Checkout') { + steps { + checkout([ + $class: 'GitSCM', + branches: [[name: "*/${params.ENVIRONMENT}"]], + userRemoteConfigs: [[ + url: 'git@gitea.company.com:repo/anexo76.git', + credentialsId: 'gitea-ssh' + ]], + extensions: [[$class: 'CloneOption', depth: 0, shallow: false]] + ]) + } + } + + stage('Test backend') { + environment { + TEST_DATABASE_URL = credentials('test-database-url') + } + steps { + sh ''' + set -euo pipefail + python3 --version + + VENV="$WORKSPACE/.pytest-venv" + python3 -m venv "$VENV" + . "$VENV/bin/activate" + + python -m pip install --upgrade pip + pip install -r "$WORKSPACE/backend/requirements.txt" + + cd "$WORKSPACE/backend" + export TEST_DATABASE_URL + alembic upgrade head + pytest -q tests -v -ra -s + ''' + } + } + + stage('Generate version') { + steps { + script { + def year = sh(returnStdout: true, script: 'date +%y').trim() + def month = sh(returnStdout: true, script: 'date +%m').trim() + + if (params.ENVIRONMENT == 'development') { + def shortHash = sh(returnStdout: true, script: 'git rev-parse --short=8 HEAD').trim() + env.APP_VERSION = "${year}.${month}.${env.VERSION_MAYOR}.${shortHash}" + } else { + def lastTag = sh(returnStdout: true, script: 'git describe --tags --abbrev=0 2>/dev/null || true').trim() + def commitCount = lastTag + ? sh(returnStdout: true, script: "git rev-list ${lastTag}..HEAD --count").trim() + : sh(returnStdout: true, script: 'git rev-list --count HEAD').trim() + + if (commitCount == '0') { + commitCount = '1' + } + env.APP_VERSION = "${year}.${month}.${env.VERSION_MAYOR}.${commitCount}" + } + + echo "VERSION: ${env.APP_VERSION}" + } + } + } + + stage('Docker login') { + steps { + withCredentials([usernamePassword(credentialsId: 'harbor-credentials', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) { + sh ''' + set -euo pipefail + echo "$HARBOR_PASSWORD" | docker login "$REGISTRY" -u "$HARBOR_USERNAME" --password-stdin + ''' + } + } + } + + stage('Build + push backend') { + steps { + sh ''' + set -euo pipefail + docker build \ + --build-arg APP_VERSION="${APP_VERSION}" \ + -t "${REGISTRY}/${IMAGE_NAMESPACE}/backend:latest" \ + -f ./backend/Dockerfile \ + ./backend + + docker push "${REGISTRY}/${IMAGE_NAMESPACE}/backend:latest" + ''' + } + } + + stage('Build + push frontend') { + steps { + sh ''' + set -euo pipefail + docker build \ + --build-arg VITE_API_URL=https://anexo76-dev.aduanasoft.com/api/ \ + --build-arg VITE_KEYCLOAK_URL=https://anexo76-dev.aduanasoft.com/kcauth/ \ + --build-arg INTERNAL_API_URL=http://backend:3467/api/ \ + -t "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:latest" \ + -f ./frontend/Dockerfile.prod \ + ./frontend + + docker push "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:latest" + ''' + } + } + + stage('Tag release (main)') { + when { + expression { params.ENVIRONMENT == 'main' } + } + steps { + sshagent(credentials: ['gitea-ssh']) { + sh ''' + set -euo pipefail + git -c user.name='Jenkins' -c user.email='jenkins@gitea.local' \ + tag -a "v${APP_VERSION}" -m "Release version ${APP_VERSION}" + git push origin "v${APP_VERSION}" + ''' + } + } + } + + stage('Deploy development') { + when { + expression { params.ENVIRONMENT == 'development' } + } + steps { + withCredentials([ + usernamePassword(credentialsId: 'harbor-credentials', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD'), + string(credentialsId: 'dev-server-user', variable: 'DEV_SERVER_USER'), + string(credentialsId: 'dev-server-host', variable: 'DEV_SERVER_HOST') + ]) { + sshagent(credentials: ['dev-server-ssh']) { + sh ''' + set -euo pipefail + ssh -o StrictHostKeyChecking=accept-new "${DEV_SERVER_USER}@${DEV_SERVER_HOST}" ' + cd ~/projects/anexo76 && + echo '"'"'"${HARBOR_PASSWORD}"'"'"' | docker login '"${REGISTRY}"' -u '"${HARBOR_USERNAME}"' --password-stdin && + docker compose -f docker-compose.prod.yml pull && + docker compose -f docker-compose.prod.yml up -d && + docker image prune -f + ' + ''' + } + } + } + } + } + + post { + always { + sh 'docker logout "${REGISTRY}" || true' + } + } +} From 44b1f4b033caa496120795aa06b45a4e5a6e8a7a Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:11:15 -0500 Subject: [PATCH 06/32] Add VERSION_MAYOR parameter to Jenkinsfile and update APP_VERSION calculation --- Jenkinsfile | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 88280eae..15343777 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,12 +8,12 @@ pipeline { parameters { choice(name: 'ENVIRONMENT', choices: ['development', 'main'], description: 'Rama a ejecutar') + string(name: 'VERSION_MAYOR', defaultValue: '1', description: 'Componente mayor de versión') } environment { REGISTRY = 'dev.aduanasoft.com' IMAGE_NAMESPACE = 'anexo76' - VERSION_MAYOR = credentials('version-mayor') // secret text, ej: 1 } stages { @@ -63,7 +63,7 @@ pipeline { if (params.ENVIRONMENT == 'development') { def shortHash = sh(returnStdout: true, script: 'git rev-parse --short=8 HEAD').trim() - env.APP_VERSION = "${year}.${month}.${env.VERSION_MAYOR}.${shortHash}" + env.APP_VERSION = "${year}.${month}.${params.VERSION_MAYOR}.${shortHash}" } else { def lastTag = sh(returnStdout: true, script: 'git describe --tags --abbrev=0 2>/dev/null || true').trim() def commitCount = lastTag @@ -73,7 +73,7 @@ pipeline { if (commitCount == '0') { commitCount = '1' } - env.APP_VERSION = "${year}.${month}.${env.VERSION_MAYOR}.${commitCount}" + env.APP_VERSION = "${year}.${month}.${params.VERSION_MAYOR}.${commitCount}" } echo "VERSION: ${env.APP_VERSION}" @@ -167,9 +167,4 @@ pipeline { } } - post { - always { - sh 'docker logout "${REGISTRY}" || true' - } - } } From 0d249b526cb346275e3fd263a9ce0f862b8141b5 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:12:32 -0500 Subject: [PATCH 07/32] Update Jenkinsfile to change Git repository URL and credentials for deployment --- Jenkinsfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 15343777..06ea3e89 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,8 +23,8 @@ pipeline { $class: 'GitSCM', branches: [[name: "*/${params.ENVIRONMENT}"]], userRemoteConfigs: [[ - url: 'git@gitea.company.com:repo/anexo76.git', - credentialsId: 'gitea-ssh' + url: 'https://git.aduanasoft.com/ADUANASOFT/anexo76.git', + credentialsId: 'gitea_acazares' ]], extensions: [[$class: 'CloneOption', depth: 0, shallow: false]] ]) @@ -129,14 +129,15 @@ pipeline { expression { params.ENVIRONMENT == 'main' } } steps { - sshagent(credentials: ['gitea-ssh']) { + withCredentials([usernamePassword(credentialsId: 'gitea_acazares', usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) { sh ''' set -euo pipefail git -c user.name='Jenkins' -c user.email='jenkins@gitea.local' \ tag -a "v${APP_VERSION}" -m "Release version ${APP_VERSION}" + git remote set-url origin "https://${GIT_USERNAME}:${GIT_PASSWORD}@git.aduanasoft.com/ADUANASOFT/anexo76.git" git push origin "v${APP_VERSION}" ''' - } + } } } From d84d081f3e8b5ef1173e9f7969fa8952ca066a1d Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:19:48 -0500 Subject: [PATCH 08/32] Enhance backend testing in Jenkinsfile by adding Docker container setup for PostgreSQL, improving error handling, and updating environment variable management. --- Jenkinsfile | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 06ea3e89..924a260d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,13 +32,42 @@ pipeline { } stage('Test backend') { - environment { - TEST_DATABASE_URL = credentials('test-database-url') - } steps { sh ''' - set -euo pipefail + set -euxo pipefail + echo "== Test backend stage started ==" + echo "Workspace: $WORKSPACE" python3 --version + docker --version + + DB_CONTAINER="anexo76-test-db-${BUILD_NUMBER}" + DB_NAME="anexo76_test" + DB_USER="anexo76" + DB_PASS="anexo76" + DB_PORT="55432" + export TEST_DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@127.0.0.1:${DB_PORT}/${DB_NAME}" + + cleanup() { + docker rm -f "$DB_CONTAINER" >/dev/null 2>&1 || true + } + trap cleanup EXIT + + docker rm -f "$DB_CONTAINER" >/dev/null 2>&1 || true + docker run -d --name "$DB_CONTAINER" \ + -e POSTGRES_DB="$DB_NAME" \ + -e POSTGRES_USER="$DB_USER" \ + -e POSTGRES_PASSWORD="$DB_PASS" \ + -p "${DB_PORT}:5432" \ + postgres:16-alpine + + # Espera a que Postgres acepte conexiones + for i in $(seq 1 30); do + if docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + break + fi + sleep 2 + done + docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" VENV="$WORKSPACE/.pytest-venv" python3 -m venv "$VENV" @@ -48,7 +77,6 @@ pipeline { pip install -r "$WORKSPACE/backend/requirements.txt" cd "$WORKSPACE/backend" - export TEST_DATABASE_URL alembic upgrade head pytest -q tests -v -ra -s ''' From c45f97d480a0b233fe8d60335cd3b3d53267104a Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:21:27 -0500 Subject: [PATCH 09/32] Refactor Jenkinsfile to utilize Docker for Python environment setup during backend testing, enhancing consistency and isolation of dependencies. --- Jenkinsfile | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 924a260d..289fb47c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,6 @@ pipeline { set -euxo pipefail echo "== Test backend stage started ==" echo "Workspace: $WORKSPACE" - python3 --version docker --version DB_CONTAINER="anexo76-test-db-${BUILD_NUMBER}" @@ -46,8 +45,10 @@ pipeline { DB_PASS="anexo76" DB_PORT="55432" export TEST_DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@127.0.0.1:${DB_PORT}/${DB_NAME}" + PY_CONTAINER="anexo76-test-py-${BUILD_NUMBER}" cleanup() { + docker rm -f "$PY_CONTAINER" >/dev/null 2>&1 || true docker rm -f "$DB_CONTAINER" >/dev/null 2>&1 || true } trap cleanup EXIT @@ -69,16 +70,21 @@ pipeline { done docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" - VENV="$WORKSPACE/.pytest-venv" - python3 -m venv "$VENV" - . "$VENV/bin/activate" - - python -m pip install --upgrade pip - pip install -r "$WORKSPACE/backend/requirements.txt" - - cd "$WORKSPACE/backend" - alembic upgrade head - pytest -q tests -v -ra -s + docker run --name "$PY_CONTAINER" --rm \ + --network host \ + -e TEST_DATABASE_URL="$TEST_DATABASE_URL" \ + -v "$WORKSPACE:/workspace" \ + -w /workspace \ + python:3.12-slim \ + sh -lc ' + set -euxo pipefail + python --version + python -m pip install --upgrade pip + pip install -r /workspace/backend/requirements.txt + cd /workspace/backend + alembic upgrade head + pytest -q tests -v -ra -s + ' ''' } } From 943c24875050d0a5d9eb0f8224ae19d8f08c9773 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:27:41 -0500 Subject: [PATCH 10/32] Add preflight checks in Jenkinsfile to ensure Docker is installed on the agent before proceeding with the pipeline, improving error handling and setup validation. --- Jenkinsfile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 289fb47c..7a85b081 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,7 @@ pipeline { - agent any + agent { + label 'any' + } options { timestamps() @@ -17,6 +19,21 @@ pipeline { } stages { + stage('Preflight tools') { + steps { + sh ''' + set -euo pipefail + echo "Node: $(hostname)" + if ! command -v docker >/dev/null 2>&1; then + echo "ERROR: este agente no tiene docker instalado." + echo "Usa un nodo Jenkins con label 'docker' y acceso al daemon." + exit 1 + fi + docker --version + ''' + } + } + stage('Checkout') { steps { checkout([ From 7caed1b875defd0cdceb58e52ad4154495cdbcfb Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:28:57 -0500 Subject: [PATCH 11/32] Update Jenkinsfile to specify 'Built-In Node' as the agent label, enhancing clarity in the pipeline configuration. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7a85b081..603a8a87 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { agent { - label 'any' + label 'Built-In Node' } options { From bb2d27865b0e9fd8ce7647509d6369315b65cf42 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Tue, 31 Mar 2026 18:39:35 -0500 Subject: [PATCH 12/32] Update Jenkinsfile to change agent label from 'Built-In Node' to 'master', aligning with current infrastructure standards. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 603a8a87..2a8a466f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { agent { - label 'Built-In Node' + label 'master' } options { From e8b4cbd377e4530e1684f44ea0c62f39d3773523 Mon Sep 17 00:00:00 2001 From: hreyes Date: Wed, 1 Apr 2026 13:40:46 -0600 Subject: [PATCH 13/32] checkpoint --- .../v1/modules/a76/layouts_csv/boms/routes.py | 5 +- .../v1/modules/a76/layouts_csv/boms/tasks.py | 3 +- .../cambio_regimen_regularizacion/routes.py | 5 +- .../cambio_regimen_regularizacion/tasks.py | 3 +- .../modules/a76/layouts_csv/classes/routes.py | 7 +- .../modules/a76/layouts_csv/classes/tasks.py | 3 +- .../clients_and_providers/routes.py | 5 +- .../clients_and_providers/tasks.py | 2 +- .../a76/layouts_csv/common/responses.py | 33 ++ .../a76/layouts_csv/customs_brokers/routes.py | 5 +- .../a76/layouts_csv/customs_brokers/tasks.py | 2 +- .../modules/a76/layouts_csv/drivers/routes.py | 7 +- .../modules/a76/layouts_csv/drivers/tasks.py | 2 +- .../a76/layouts_csv/exchange_rate/routes.py | 5 +- .../a76/layouts_csv/exchange_rate/tasks.py | 2 +- .../a76/layouts_csv/exportacion/routes.py | 5 +- .../a76/layouts_csv/exportacion/tasks.py | 3 +- .../a76/layouts_csv/facturas/routes.py | 3 +- .../modules/a76/layouts_csv/facturas/tasks.py | 44 +-- .../modules/a76/layouts_csv/parts/routes.py | 7 +- .../v1/modules/a76/layouts_csv/parts/tasks.py | 3 +- .../a76/layouts_csv/pedmientos/routes.py | 5 +- .../a76/layouts_csv/pedmientos/tasks.py | 2 +- .../a76/layouts_csv/trailers/routes.py | 7 +- .../modules/a76/layouts_csv/trailers/tasks.py | 2 +- .../a76/layouts_csv/transportistas/routes.py | 7 +- .../a76/layouts_csv/transportistas/tasks.py | 2 +- .../layouts_csv/us_tariff_fractions/routes.py | 5 +- .../layouts_csv/us_tariff_fractions/tasks.py | 2 +- .../a76/layouts_csv/vehicles/routes.py | 7 +- .../modules/a76/layouts_csv/vehicles/tasks.py | 2 +- .../dashboard/csv-upload/CsvParamsBar.svelte | 13 +- .../csv-upload/ProcessingResultModal.svelte | 159 ++++++--- .../csv-upload/UploadLauncherGrid.svelte | 74 +++-- frontend/src/lib/csv-import-commit-metrics.ts | 36 ++ frontend/src/lib/csv-upload-row-count.ts | 15 + .../routes/dashboard/csv-upload/+page.svelte | 307 ++++++++++++------ 37 files changed, 551 insertions(+), 248 deletions(-) create mode 100644 frontend/src/lib/csv-import-commit-metrics.ts create mode 100644 frontend/src/lib/csv-upload-row-count.ts diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/routes.py b/backend/api/v1/modules/a76/layouts_csv/boms/routes.py index f99183d5..f7d46af7 100644 --- a/backend/api/v1/modules/a76/layouts_csv/boms/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/boms/routes.py @@ -28,6 +28,7 @@ from .tasks import ( ) from ..common.error_csv import download_scan_errors_csv_stream from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -127,12 +128,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("BOMs import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py b/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py index 99e9c6d4..9f249de4 100644 --- a/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/boms/tasks.py @@ -62,8 +62,7 @@ def scan_file(self, job_id: str, config: str = None): try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv.iter_csv_rows(file_path): - if i % 500 == 0: - self.update_state( + self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py index 9f8a4ccd..d8612d2e 100644 --- a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py @@ -22,6 +22,7 @@ from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit from .schemas import ImportJobResponse, CommitRequest from .tasks import scan_file, insert_valid_rows, JOB_TYPE, CRREG_IMPORT_REDIS_TTL from ..common import storage as common_storage +from ..common.responses import normalize_commit_status_payload from ..common.error_csv import download_scan_errors_csv_stream router = APIRouter() @@ -133,11 +134,11 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"): - return task_result.result + return normalize_commit_status_payload(task_result.result) logger.warning("Cambio régimen/Regularización import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py index c06ce2b9..a0a6c545 100644 --- a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/tasks.py @@ -75,8 +75,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): try: for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None): - if i % 500 == 0: - on_progress(i, total_rows) + on_progress(i, total_rows) _norm_row(row, template_id) processed_rows += 1 except Exception as e: diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/routes.py b/backend/api/v1/modules/a76/layouts_csv/classes/routes.py index 53464802..0d711a72 100644 --- a/backend/api/v1/modules/a76/layouts_csv/classes/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/classes/routes.py @@ -19,6 +19,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload from .schemas import ImportJobResponse from .tasks import ( scan_file, @@ -133,12 +134,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) # Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica if isinstance(result, str): @@ -150,7 +151,7 @@ async def get_import_status(job_id: str): ): return parsed if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"): - return parsed + return normalize_commit_status_payload(parsed) except (json.JSONDecodeError, TypeError): pass diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py b/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py index a455ce8b..1d1abb43 100644 --- a/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/classes/tasks.py @@ -84,8 +84,7 @@ def scan_file(self, job_id: str, config: str = None): try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): - if i % 500 == 0: - self.update_state( + self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py index fcc092dd..a9f14507 100644 --- a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py @@ -19,6 +19,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload from .schemas import ImportJobResponse from .tasks import ( scan_file, @@ -135,13 +136,13 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("CP import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py index 21cb2e46..411bcc42 100644 --- a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/tasks.py @@ -77,7 +77,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) diff --git a/backend/api/v1/modules/a76/layouts_csv/common/responses.py b/backend/api/v1/modules/a76/layouts_csv/common/responses.py index 848a2a3c..cac67a28 100644 --- a/backend/api/v1/modules/a76/layouts_csv/common/responses.py +++ b/backend/api/v1/modules/a76/layouts_csv/common/responses.py @@ -72,3 +72,36 @@ def commit_result( if error: out["error"] = error return out + + +def normalize_commit_status_payload(d: Dict[str, Any]) -> Dict[str, Any]: + """ + Homogeneiza respuestas de commit CSV (finished / warning) al contrato usado por pedimentos: + inserted, updated, skipped_*, skipped_details, skipped_summary, critical_reference_gaps, + reference_state_ready. No modifica waiting_confirmation ni otros status. + """ + if not isinstance(d, dict): + return d + st = d.get("status") + if st not in ("finished", "warning"): + return d + out = dict(d) + out.setdefault("inserted", 0) + out.setdefault("updated", 0) + out.setdefault("skipped_invalid", 0) + out.setdefault("skipped_missing_fk", 0) + out.setdefault("skipped_duplicate", 0) + out.setdefault("skipped_missing_invoice", 0) + details = out.get("skipped_details") + if not isinstance(details, list): + out["skipped_details"] = [] + else: + out["skipped_details"] = details + out.setdefault("critical_reference_gaps", 0) + if "reference_state_ready" not in out: + out["reference_state_ready"] = out.get("critical_reference_gaps", 0) == 0 + if "skipped_summary" not in out: + out["skipped_summary"] = ( + _summarize_reasons(out["skipped_details"], "reason") if out["skipped_details"] else [] + ) + return out diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py index f0cebaaf..dc17a477 100644 --- a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py @@ -19,6 +19,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload from .schemas import ImportJobResponse from .tasks import ( scan_file, @@ -135,13 +136,13 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} # A veces Celery tiene el result disponible pero state aún no es SUCCESS; si el result es éxito, devolverlo result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("CB import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py index 1ddb3e51..0c13b14f 100644 --- a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/tasks.py @@ -88,7 +88,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, fieldnames=CUSTOMS_BROKERS_FIELDNAMES_ORDER, headerless_first_cell_values=CUSTOMS_BROKERS_HEADERLESS_FIRST_CELL, ): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py b/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py index 4a155a6c..bb118f17 100644 --- a/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py @@ -28,6 +28,7 @@ from .tasks import ( DRV_IMPORT_REDIS_TTL, ) from ..common.error_csv import download_scan_errors_csv_stream +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -113,6 +114,8 @@ async def get_import_status(job_id: str): raw = r.get(f"{DRV_IMPORT_STATUS_PREFIX}{job_id}") if raw: data = json.loads(raw.decode("utf-8")) + if isinstance(data, dict) and data.get("status") in ("finished", "warning"): + return normalize_commit_status_payload(data) return data except Exception as e: logger.debug(f"Drivers import: could not read status from Redis: {e}") @@ -131,12 +134,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("Drivers import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py index f81a341b..c03e88a1 100644 --- a/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/tasks.py @@ -135,7 +135,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) for i, row in enumerate(dict_reader, start=1): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py index db570733..4887092f 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py @@ -19,6 +19,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload from .schemas import ImportJobResponse from .tasks import ( scan_file, @@ -146,12 +147,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("ER import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py index c8f5521e..3ebf4bbd 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/tasks.py @@ -66,7 +66,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = _norm_row(row) diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py index dca65485..49eba2b8 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py @@ -25,6 +25,7 @@ from .schemas import ImportJobResponse, CommitRequest from .tasks import scan_file, insert_valid_rows, JOB_TYPE, EXP_IMPORT_REDIS_TTL from ..common import storage as common_storage from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -138,11 +139,11 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} if isinstance(getattr(task_result, "result", None), dict) and task_result.result.get("status") in ("finished", "warning"): - return task_result.result + return normalize_commit_status_payload(task_result.result) logger.warning("Exportación import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py index 2895bf40..bbd06283 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/tasks.py @@ -74,8 +74,7 @@ def scan_file(self, job_id: str, model_target: str, config: str = None): processed_rows = 0 try: for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=None): - if i % 500 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows}) _norm_row(row, template_id) processed_rows += 1 except Exception as e: diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py b/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py index c9ac5285..0ae9556f 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py @@ -27,6 +27,7 @@ from .tasks import ( ) from .schemas import ImportJobResponse, ImportJobStatus, CommitRequest from ..common import storage as common_storage +from ..common.responses import normalize_commit_status_payload from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit router = APIRouter() @@ -150,7 +151,7 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} # FAILURE: obtener mensaje real (traceback, result o get(propagate=False)) logger.warning("Import task %s failed: state=%s", job_id, task_result.state) diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py index f2fda307..c364981e 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/tasks.py @@ -628,8 +628,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = errors_detail = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "imp_def_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] row_errors = validate_row_series_impo_def( @@ -822,8 +821,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = errors_detail = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "exp_def_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] row_errors = validate_row_series_expo( @@ -1004,8 +1002,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = errors_detail = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "cmex_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] row_errors = validate_row_series_impo_def( @@ -1159,8 +1156,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = errors_detail: List[Dict[str, Any]] = [] error_lines_list: List[int] = [] for i, row in enumerate(reader, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": len(error_lines_list)}) row_norm = row_from_template(row, "imp_temp_series", normalize_header) warnings_list: List[Dict[str, Any]] = [] row_errors = validate_row_series_impo_temp( @@ -1495,8 +1491,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_temp_details", normalize_header) row_errors = validate_row_partidas_impo_temp( row_norm, @@ -1806,8 +1801,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_def_details", normalize_header) row_errors = validate_row_partidas_impo_def( row_norm, @@ -2373,8 +2367,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "cmex_details", normalize_header) row_errors = validate_row_partidas_impo_def( row_norm, @@ -2831,8 +2824,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = + "\n" ) for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state( + self.update_state( state="PROGRESS", meta={ "current": i, @@ -3205,8 +3197,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "imp_def_header", normalize_header) warnings_row = [] row_errors = validate_row_encabezados_impo_def( @@ -3582,8 +3573,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) + self.update_state(state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}) row_norm = row_from_template(row, "exp_def_header", normalize_header) warnings_row = [] row_errors = validate_row_encabezados_expo( @@ -3798,8 +3788,7 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = with CoreSessionLocal() as session: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in enumerate(rows_list, start=1): - if i % 1000 == 0: - self.update_state( + self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) @@ -3949,12 +3938,11 @@ def _do_scan_file(self, job_id: str, model_target: str, config: Optional[str] = for i, row in enumerate(reader, start=1): # Check for Progress Update - if i % 1000 == 0: - self.update_state(state='PROGRESS', meta={ - 'current': i, - 'total': total_rows, - 'errors': error_count - }) + self.update_state(state='PROGRESS', meta={ + 'current': i, + 'total': total_rows, + 'errors': error_count + }) # Solo columnas de la plantilla (respetar plantilla tal cual) row_norm = row_from_template(row, template_id, normalize_header) diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/routes.py b/backend/api/v1/modules/a76/layouts_csv/parts/routes.py index 5184073e..6e1fb792 100644 --- a/backend/api/v1/modules/a76/layouts_csv/parts/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/parts/routes.py @@ -28,6 +28,7 @@ from .tasks import ( ) from ..common.error_csv import download_scan_errors_csv_stream from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -131,12 +132,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) # Si Celery devolvió el resultado como string (p. ej. JSON), parsear y devolver como scan si aplica if isinstance(result, str): @@ -148,7 +149,7 @@ async def get_import_status(job_id: str): ): return parsed if isinstance(parsed, dict) and parsed.get("status") in ("finished", "warning"): - return parsed + return normalize_commit_status_payload(parsed) except (json.JSONDecodeError, TypeError): pass diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py b/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py index 4c3088ed..6d2fe8c2 100644 --- a/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/parts/tasks.py @@ -94,8 +94,7 @@ def scan_file(self, job_id: str, config: str = None): try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv.iter_csv_rows(file_path, fieldnames=fieldnames): - if i % 500 == 0: - self.update_state( + self.update_state( state="PROGRESS", meta={"current": i, "total": total_rows, "errors": error_count}, ) diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py index eda3ea86..69e26d65 100644 --- a/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py @@ -19,6 +19,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload from .schemas import ImportJobResponse from .tasks import ( scan_file, @@ -143,12 +144,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("Pedimentos import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py index 1ac7aa76..5bd482a9 100644 --- a/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/tasks.py @@ -99,7 +99,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path, fieldnames=fieldnames): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = _norm_row(row) diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py b/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py index f1543f78..73547beb 100644 --- a/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py @@ -30,6 +30,7 @@ from .tasks import ( TRL_IMPORT_REDIS_TTL, ) from ..common.error_csv import download_scan_errors_csv_stream +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -132,6 +133,8 @@ async def get_import_status(job_id: str): raw = r.get(f"{TRL_IMPORT_STATUS_PREFIX}{job_id}") if raw: data = json.loads(raw.decode("utf-8")) + if isinstance(data, dict) and data.get("status") in ("finished", "warning"): + return normalize_commit_status_payload(data) return data except Exception as e: logger.debug(f"Trailers import: could not read status from Redis: {e}") @@ -150,12 +153,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("Trailers import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py b/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py index a920076b..78a3ec24 100644 --- a/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/tasks.py @@ -121,7 +121,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, dict_reader = csv.DictReader(f_in, fieldnames=headers, dialect=dialect) for i, row in enumerate(dict_reader, start=1): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py index 18671ddb..518b3d3f 100644 --- a/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py @@ -30,6 +30,7 @@ from .tasks import ( TRP_IMPORT_REDIS_TTL, ) from ..common.error_csv import download_scan_errors_csv_stream +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -132,6 +133,8 @@ async def get_import_status(job_id: str): raw = r.get(f"{TRP_IMPORT_STATUS_PREFIX}{job_id}") if raw: data = json.loads(raw.decode("utf-8")) + if isinstance(data, dict) and data.get("status") in ("finished", "warning"): + return normalize_commit_status_payload(data) return data except Exception as e: logger.debug("Transportistas import: could not read status from Redis: %s", e) @@ -150,12 +153,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("Transportistas import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py index a1c9ac00..09664a24 100644 --- a/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/tasks.py @@ -79,7 +79,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py index afb8c3bb..b35846f5 100644 --- a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py @@ -19,6 +19,7 @@ from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit +from ..common.responses import normalize_commit_status_payload from .schemas import ImportJobResponse from .tasks import ( scan_file, @@ -138,12 +139,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("FA import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py index 1924e229..06d8d5f2 100644 --- a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/tasks.py @@ -68,7 +68,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) warn = validate_row_desfase_fa(row, i) diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py index da98d135..95539a68 100644 --- a/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py @@ -30,6 +30,7 @@ from .tasks import ( VEHL_IMPORT_REDIS_TTL, ) from ..common.error_csv import download_scan_errors_csv_stream +from ..common.responses import normalize_commit_status_payload router = APIRouter() logger = logging.getLogger(__name__) @@ -131,6 +132,8 @@ async def get_import_status(job_id: str): raw = r.get(f"{VEHL_IMPORT_STATUS_PREFIX}{job_id}") if raw: data = json.loads(raw.decode("utf-8")) + if isinstance(data, dict) and data.get("status") in ("finished", "warning"): + return normalize_commit_status_payload(data) return data except Exception as e: logger.debug(f"Vehicles import: could not read status from Redis: {e}") @@ -149,12 +152,12 @@ async def get_import_status(job_id: str): if task_result.state == "SUCCESS": result = task_result.result if isinstance(result, dict) and "status" in result: - return result + return normalize_commit_status_payload(result) return {"status": "finished", "result": result} result = getattr(task_result, "result", None) if isinstance(result, dict) and result.get("status") in ("finished", "warning"): - return result + return normalize_commit_status_payload(result) logger.warning("Vehicles import task %s failed: state=%s", job_id, task_result.state) err_msg = None diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py index 9bebb21a..ad9608ed 100644 --- a/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/tasks.py @@ -92,7 +92,7 @@ def _do_scan(job_id: str, progress_callback: Optional[Any] = None) -> Dict[str, try: with open(error_path, "w", encoding="utf-8") as f_err: for i, row in common_csv_reader.iter_csv_rows(file_path): - if progress_callback and i % 500 == 0: + if progress_callback: progress_callback(i, total_rows, error_count) row_norm = row_from_template(row, common_normalize.normalize_header) diff --git a/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte b/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte index 45c640e8..38278510 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/CsvParamsBar.svelte @@ -2,15 +2,19 @@ import { Label } from '$lib/components/ui/label/index.js'; import { Settings2 } from 'lucide-svelte'; import { globalCsvParams, tabSettings, type CsvUploadField } from '$lib/config/csv-upload'; + import { cn } from '$lib/utils'; let { globalSettings = $bindable(), activeTab, - tabSettingsValues = $bindable() + tabSettingsValues = $bindable(), + embedded = false }: { globalSettings: Record; activeTab: string; tabSettingsValues: Record; + /** Si true, el padre aporta el contenedor fijo (p. ej. pie apilado con progreso CSV). */ + embedded?: boolean; } = $props(); const globalParamNames = $derived(new Set(globalCsvParams.map((p) => p.name))); @@ -27,7 +31,12 @@
diff --git a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte index 05a98de3..dd342907 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/ProcessingResultModal.svelte @@ -11,6 +11,11 @@ } from 'lucide-svelte'; import { tick } from 'svelte'; import { api } from '$lib/api'; + import { + totalSkippedFromCommit, + criticalReferenceGaps, + referenceStateReady + } from '$lib/csv-import-commit-metrics'; let { open = $bindable(false), @@ -50,29 +55,33 @@ let isPending = $derived(!!scanResults && !commitResults); let isFinished = $derived(!!commitResults); - // Derived metrics for UI logic - let hasErrors = $derived( - scanResults?.error_count > 0 || - commitResults?.skipped_invalid > 0 || - commitResults?.skipped_missing_fk > 0 - ); - let scanErrorsShown = $derived(Array.isArray(scanResults?.errors) ? scanResults.errors.length : 0); let scanErrorsTotal = $derived(scanResults?.error_count || 0); let scanErrorsTruncated = $derived(scanErrorsShown > 0 && scanErrorsShown < scanErrorsTotal); - let totalSkipped = $derived( - (commitResults?.skipped_invalid || 0) + - (commitResults?.skipped_missing_fk || 0) + - (commitResults?.skipped_missing_invoice || 0) + - (commitResults?.skipped_duplicate || 0) + let scanErrorSummary = $derived( + Array.isArray(scanResults?.error_summary) ? scanResults.error_summary : [] ); - let insertedCount = $derived(commitResults?.inserted || 0); - let updatedCount = $derived(commitResults?.updated || 0); - // Algunos módulos usan `inserted` solo para altas y `updated` para modo "actualizar". - // Para la UI, reflejamos "cambios positivos" como insertados + actualizados. - let insertedOrUpdatedCount = $derived(insertedCount + updatedCount); + let totalSkipped = $derived(totalSkippedFromCommit(commitResults)); + let refGaps = $derived(criticalReferenceGaps(commitResults)); + let refReady = $derived(referenceStateReady(commitResults)); + + let insertedCount = $derived(Number(commitResults?.inserted) || 0); + let updatedCount = $derived(Number(commitResults?.updated) || 0); + + /** Cabecera: errores de escaneo (solo pendiente) o observaciones de commit. */ + let hasErrors = $derived( + (isPending && (scanResults?.error_count ?? 0) > 0) || + (isFinished && + (totalSkipped > 0 || + refGaps > 0 || + commitResults?.status === 'warning')) + ); + + let commitSkippedSummary = $derived( + Array.isArray(commitResults?.skipped_summary) ? commitResults.skipped_summary : [] + ); function handleOpenChange(newOpen: boolean) { if (!newOpen) { @@ -298,6 +307,24 @@
+ {#if scanResults.message} +

+ {scanResults.message} +

+ {/if} + {#if scanErrorSummary.length > 0} +
+ {#each scanErrorSummary as item} + + {(item as { reason?: string; count?: number }).reason ?? '—'} · {(item as { count?: number }) + .count ?? 0} + + {/each} +
+ {/if} + {#if scanResults.error_count > 0}
{#if isFinished}
- -
- +
-
- +
+ Insertados
- {insertedOrUpdatedCount} - {#if updatedCount > 0} -

- {insertedCount} insertados, {updatedCount} actualizados -

- {/if} + {insertedCount}
- -
-
- - + + Actualizados +
+ {updatedCount} +
+
+
+ + Rechazados
{totalSkipped} {#if totalSkipped > 0 && commitResults.skipped_details && commitResults.skipped_details.length > 0} -

+

Revisa el detalle por línea en la tabla inferior.

{/if}
+
+ {#if refGaps > 0} + +
+

Brechas de referencia (FK / catálogos)

+

+ Hay {refGaps} brecha(s) crítica(s) de referencia. Revisa catálogos y el detalle de + filas rechazadas antes de reintentar. +

+
+ {:else} + +
+

Estado de referencias

+

+ {refReady + ? 'Referencias listas para operar (sin brechas críticas reportadas).' + : 'Sin brechas numéricas; revisa el mensaje del servidor si aplica.'} +

+
+ {/if} +
+ + {#if commitResults.message} +

+ {commitResults.message} +

+ {/if} + {#if commitSkippedSummary.length > 0} +
+

+ Resumen de motivos de rechazo +

+
+ {#each commitSkippedSummary as item} + + {(item as { reason?: string; count?: number }).reason ?? '—'} · {(item as { + count?: number; + }).count ?? 0} + + {/each} +
+
+ {/if} + {#if commitResults.skipped_details && commitResults.skipped_details.length > 0}
@@ -466,7 +545,7 @@ {detail.invoice || '-'} - {detail.reason} + {detail.reason ?? '-'} {detail.solution || '-'} {/each} diff --git a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte index 35d1f218..0c9bdaaa 100644 --- a/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte +++ b/frontend/src/lib/components/dashboard/csv-upload/UploadLauncherGrid.svelte @@ -8,12 +8,17 @@ let { items, - onUpload + onUpload, + busy = false }: { items: CsvUploadItem[]; onUpload: (file: File, config: CsvUploadItem) => void; + /** Bloquea nuevas cargas (p. ej. importación en curso sin overlay). */ + busy?: boolean; } = $props(); + const gridLocked = $derived(busy); + let dragOverId = $state(null); // Group items @@ -34,7 +39,7 @@ }); function handleDragEnter(e: DragEvent, id: string, disabled?: boolean) { - if (disabled) return; + if (disabled || gridLocked) return; e.preventDefault(); e.stopPropagation(); dragOverId = id; @@ -47,7 +52,7 @@ } function handleDragOver(e: DragEvent, disabled?: boolean) { - if (disabled) return; + if (disabled || gridLocked) return; e.preventDefault(); e.stopPropagation(); dragOverId = null; @@ -65,7 +70,7 @@ } function handleDrop(e: DragEvent, item: CsvUploadItem) { - if (item.disabled) return; + if (item.disabled || gridLocked) return; e.preventDefault(); e.stopPropagation(); dragOverId = null; @@ -76,12 +81,13 @@ } function handleClick(id: string, disabled?: boolean) { - if (disabled) return; + if (disabled || gridLocked) return; const input = document.getElementById(`file-input-${id}`) as HTMLInputElement; if (input) input.click(); } function handleFileChange(e: Event, item: CsvUploadItem) { + if (gridLocked) return; const target = e.target as HTMLInputElement; if (target.files && target.files.length > 0) { validateAndUpload(target.files[0], item); @@ -90,7 +96,7 @@ } async function handleContextMenu(e: MouseEvent, item: CsvUploadItem) { - if (item.disabled) { + if (item.disabled || gridLocked) { e.preventDefault(); return; } @@ -117,25 +123,30 @@ } -
+
{#if groupedItems.ungrouped.length > 0}
{#each groupedItems.ungrouped as item}
handleDragEnter(e, item.id, item.disabled)} + ondragenter={(e) => handleDragEnter(e, item.id, item.disabled || gridLocked)} ondragleave={handleDragLeave} - ondragover={(e) => handleDragOver(e, item.disabled)} + ondragover={(e) => handleDragOver(e, item.disabled || gridLocked)} ondrop={(e) => handleDrop(e, item)} oncontextmenu={(e) => handleContextMenu(e, item)} role="button" - tabindex={item.disabled ? -1 : 0} - onclick={() => handleClick(item.id, item.disabled)} - onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)} + tabindex={item.disabled || gridLocked ? -1 : 0} + onclick={() => handleClick(item.id, item.disabled || gridLocked)} + onkeydown={(e) => + !item.disabled && !gridLocked && e.key === 'Enter' && handleClick(item.id)} > handleFileChange(e, item)} - disabled={item.disabled} + disabled={item.disabled || gridLocked} /> - {#if !item.disabled && dragOverId === item.id} + {#if !item.disabled && !gridLocked && dragOverId === item.id}
@@ -177,7 +188,7 @@
@@ -201,18 +212,19 @@
handleDragEnter(e, item.id, item.disabled)} + ondragenter={(e) => handleDragEnter(e, item.id, item.disabled || gridLocked)} ondragleave={handleDragLeave} - ondragover={(e) => handleDragOver(e, item.disabled)} + ondragover={(e) => handleDragOver(e, item.disabled || gridLocked)} ondrop={(e) => handleDrop(e, item)} oncontextmenu={(e) => handleContextMenu(e, item)} role="button" - tabindex={item.disabled ? -1 : 0} - onclick={() => handleClick(item.id, item.disabled)} - onkeydown={(e) => !item.disabled && e.key === 'Enter' && handleClick(item.id)} + tabindex={item.disabled || gridLocked ? -1 : 0} + onclick={() => handleClick(item.id, item.disabled || gridLocked)} + onkeydown={(e) => + !item.disabled && !gridLocked && e.key === 'Enter' && handleClick(item.id)} > handleFileChange(e, item)} - disabled={item.disabled} + disabled={item.disabled || gridLocked} /> - {#if !item.disabled && dragOverId === item.id} + {#if !item.disabled && !gridLocked && dragOverId === item.id}
@@ -256,7 +268,7 @@
diff --git a/frontend/src/lib/csv-import-commit-metrics.ts b/frontend/src/lib/csv-import-commit-metrics.ts new file mode 100644 index 00000000..0eccf8db --- /dev/null +++ b/frontend/src/lib/csv-import-commit-metrics.ts @@ -0,0 +1,36 @@ +/** + * Métricas homogéneas post-commit CSV (contrato tipo pedimentos + extras p. ej. facturas). + */ +export function totalSkippedFromCommit(cr: Record | null | undefined): number { + if (!cr || typeof cr !== 'object') return 0; + const n = (k: string) => { + const v = (cr as Record)[k]; + if (typeof v === 'number' && Number.isFinite(v)) return v; + if (typeof v === 'string' && v.trim() !== '') return parseInt(v, 10) || 0; + return 0; + }; + return ( + n('skipped_invalid') + + n('skipped_missing_fk') + + n('skipped_missing_invoice') + + n('skipped_duplicate') + ); +} + +export function criticalReferenceGaps(cr: Record | null | undefined): number { + if (!cr || typeof cr !== 'object') return 0; + const v = (cr as Record).critical_reference_gaps; + if (typeof v === 'number' && Number.isFinite(v)) return v; + return 0; +} + +export function referenceStateReady(cr: Record | null | undefined): boolean { + if (!cr || typeof cr !== 'object') return true; + if ('reference_state_ready' in cr) { + const v = (cr as Record).reference_state_ready; + if (typeof v === 'boolean') return v; + if (v === 'True' || v === 'true') return true; + if (v === 'False' || v === 'false') return false; + } + return criticalReferenceGaps(cr) === 0; +} diff --git a/frontend/src/lib/csv-upload-row-count.ts b/frontend/src/lib/csv-upload-row-count.ts new file mode 100644 index 00000000..0fb17515 --- /dev/null +++ b/frontend/src/lib/csv-upload-row-count.ts @@ -0,0 +1,15 @@ +/** + * Cuenta filas de datos (excluye 1 línea de encabezado) en un CSV local. + * Asume primera línea no vacía = encabezado. + */ +export async function countCsvDataRows(file: File): Promise { + const text = await file.text(); + if (!text.trim()) return 0; + const lines = text.split(/\r\n|\r|\n/); + let nonEmpty = 0; + for (const line of lines) { + if (line.trim().length > 0) nonEmpty += 1; + } + if (nonEmpty <= 1) return 0; + return nonEmpty - 1; +} diff --git a/frontend/src/routes/dashboard/csv-upload/+page.svelte b/frontend/src/routes/dashboard/csv-upload/+page.svelte index 26cdc4bc..4fef2855 100644 --- a/frontend/src/routes/dashboard/csv-upload/+page.svelte +++ b/frontend/src/routes/dashboard/csv-upload/+page.svelte @@ -32,6 +32,8 @@ type CsvImportPendingEntry } from '$lib/csv-import-pending'; import { fetchCsvImportStatus } from '$lib/csv-import-status-api'; + import { totalSkippedFromCommit } from '$lib/csv-import-commit-metrics'; + import { countCsvDataRows } from '$lib/csv-upload-row-count'; import CsvPendingImportsSheet from '$lib/components/dashboard/csv-upload/CsvPendingImportsSheet.svelte'; /** Intenta extraer un objeto tipo scan desde string tipo repr de Python. */ @@ -113,6 +115,24 @@ ? safeInt(skippedDuplicateMatch[1]) : 0; + const skippedMissingInvMatch = + s.match(/['"]skipped_missing_invoice['"]:\s*['"]?(\d+)['"]?/) || + s.match(/['"]skipped_missing_invoice['"]:\s*([^,}]+)/); + const skipped_missing_invoice = skippedMissingInvMatch + ? safeInt(skippedMissingInvMatch[1]) + : 0; + + const gapsMatch = + s.match(/['"]critical_reference_gaps['"]:\s*['"]?(\d+)['"]?/) || + s.match(/['"]critical_reference_gaps['"]:\s*([^,}]+)/); + const critical_reference_gaps = gapsMatch ? safeInt(gapsMatch[1]) : 0; + + const refReadyMatch = s.match(/['"]reference_state_ready['"]:\s*(True|False|true|false)/); + let reference_state_ready = true; + if (refReadyMatch) { + reference_state_ready = refReadyMatch[1].toLowerCase() === 'true'; + } + // Detalle opcional: lista de dicts con {line, reason, solution?} const skipped_details: Array<{ line: number; reason: string; solution?: string }> = []; const detailRegex = @@ -132,11 +152,19 @@ const message = messageMatch ? unescapePy(messageMatch[1]) : undefined; // Si no hay contadores en el texto, no es un commit. - if (inserted === 0 && updated === 0 && skipped_invalid === 0 && skipped_missing_fk === 0 && skipped_duplicate === 0) { + if ( + inserted === 0 && + updated === 0 && + skipped_invalid === 0 && + skipped_missing_fk === 0 && + skipped_duplicate === 0 && + skipped_missing_invoice === 0 && + critical_reference_gaps === 0 + ) { return null; } - // Mínimo requerido por el modal + // Mínimo requerido por el modal (contrato tipo pedimentos) return { status, inserted, @@ -144,6 +172,9 @@ skipped_invalid, skipped_missing_fk, skipped_duplicate, + skipped_missing_invoice, + critical_reference_gaps, + reference_state_ready, skipped_details, ...(message ? { message } : {}) }; @@ -198,58 +229,85 @@ /** Job id del escaneo listo para confirmar (para quitar de pendientes al hacer commit). */ let scanPhaseJobId = $state(null); + /** + * Total de filas de datos conocido para la barra (máx. entre conteo local, total del poll Celery y total_rows del escaneo). + * No se reinicia mientras el modal de resultados siga abierto, para poder usar el mismo total en el commit. + */ + let csvImportRowTotal = $state(0); + const csvProgressStepTitle = $derived.by(() => { switch (csvProgressPhase) { case 'upload': - return 'Paso 1 de 2 — Subiendo el archivo'; + return 'Subiendo archivo CSV'; case 'scan': - return 'Paso 2 de 2 — Escaneando en el servidor'; + return 'Validando registros en el servidor'; case 'commit': - return 'Finalizando importación'; + return 'Grabando registros en base de datos'; default: return ''; } }); + /** Denominador único: filas de la tarea (registros del CSV / total reportado por el worker). */ + const csvImportProgressDenom = $derived.by(() => { + const d = Math.max(csvImportRowTotal || 0, scanProgressTotal || 0); + return d > 0 ? d : 1; + }); + const csvProgressDetailLine = $derived.by(() => { + const denom = csvImportProgressDenom; switch (csvProgressPhase) { case 'upload': - return ''; + if (csvImportRowTotal > 0) { + return `Archivo: ~${csvImportRowTotal} fila(s) de datos — subiendo (aún no se validan registros en servidor)…`; + } + return uploadLengthComputable + ? 'Subiendo archivo…' + : 'Subiendo archivo (tamaño desconocido en el navegador)…'; case 'scan': - return scanProgressTotal > 0 - ? `Filas procesadas: ${scanProgressCurrent} / ${scanProgressTotal}` - : 'Preparando resultados…'; + return `Registros procesados: ${Math.min(scanProgressCurrent, denom)} / ${denom}`; case 'commit': - return 'Escribiendo registros en base de datos…'; + return scanProgressTotal > 0 + ? `Registros grabados: ${Math.min(scanProgressCurrent, denom)} / ${denom}` + : `Grabando en base de datos… (${Math.min(scanProgressCurrent, denom)} / ${denom} según último total conocido)`; default: return ''; } }); - const csvProgressBarIndeterminate = $derived( - csvProgressPhase === 'commit' || - (csvProgressPhase === 'scan' && scanProgressTotal <= 0) || - (csvProgressPhase === 'upload' && !uploadLengthComputable) + /** Porcentaje según filas procesadas / total de filas (sin tramos artificiales por etapa). */ + const csvCombinedProgressPct = $derived.by(() => { + if (!isUploading || csvProgressPhase === 'idle') return 0; + if (csvProgressPhase === 'upload') { + return 0; + } + const denom = csvImportProgressDenom; + const numer = Math.min(Math.max(0, scanProgressCurrent), denom); + return Math.min(100, Math.round((100 * numer) / denom)); + }); + + /** + * Commit: muchas tareas no publican PROGRESS por fila; si current sigue en 0, barra indeterminada. + * Cuando Celery envía current > 0, la barra pasa a reflejar registros reales. + */ + const csvFooterProgressIndeterminate = $derived( + isUploading && !showResultModal && csvProgressPhase === 'commit' && scanProgressCurrent <= 0 ); - const csvProgressBarValue = $derived( - csvProgressPhase === 'upload' - ? uploadProgressPct - : csvProgressPhase === 'scan' && scanProgressTotal > 0 - ? Math.min(100, Math.round((scanProgressCurrent / scanProgressTotal) * 100)) - : 0 + const csvFooterPercentLabel = $derived( + csvFooterProgressIndeterminate + ? null + : csvProgressPhase === 'upload' + ? '0%' + : `${csvCombinedProgressPct}%` ); - const csvProgressPercentText = $derived( - csvProgressBarIndeterminate ? null : `${csvProgressBarValue}%` - ); - - const csvProgressAriaValueText = $derived( - csvProgressPercentText - ? `${csvProgressStepTitle}, ${csvProgressPercentText}` + const csvFooterAriaValueText = $derived( + csvFooterProgressIndeterminate + ? `${csvProgressStepTitle}. ${csvProgressDetailLine || 'En proceso…'}` : csvProgressDetailLine - ? `${csvProgressStepTitle}. ${csvProgressDetailLine}` - : csvProgressStepTitle + ? `${csvProgressStepTitle}, ${csvFooterPercentLabel}. ${csvProgressDetailLine}` + : `${csvProgressStepTitle}, ${csvFooterPercentLabel}` ); $effect(() => { @@ -257,8 +315,12 @@ csvProgressPhase = 'idle'; uploadProgressPct = 0; uploadLengthComputable = false; - scanProgressCurrent = 0; - scanProgressTotal = 0; + // Con modal abierto (escaneo listo o resultado de commit) conservar totales para la siguiente fase o la UI. + if (!showResultModal) { + scanProgressCurrent = 0; + scanProgressTotal = 0; + csvImportRowTotal = 0; + } currentImportLabel = null; csvResumeOverlayHint = false; scanPhaseJobId = null; @@ -523,6 +585,12 @@ uploadLengthComputable = false; scanProgressCurrent = 0; scanProgressTotal = 0; + csvImportRowTotal = 0; + try { + csvImportRowTotal = await countCsvDataRows(file); + } catch (e) { + console.warn('csv-upload: no se pudo contar filas del archivo', e); + } csvResumeOverlayHint = false; skipScanCompleteToastOnce = false; scanPhaseJobId = null; @@ -914,13 +982,20 @@ const p = (res.data as { progress?: unknown }).progress; const t = (res.data as { total?: unknown }).total; if (typeof p === 'number') scanProgressCurrent = p; - if (typeof t === 'number') scanProgressTotal = t; + if (typeof t === 'number') { + scanProgressTotal = t; + csvImportRowTotal = Math.max(csvImportRowTotal, t); + } } // Tratar como resultado de escaneo si viene status waiting_confirmation O si el payload tiene forma de scan (job_id + total_rows) const looksLikeScanResult = res.data?.status === 'waiting_confirmation' || (res.data?.job_id && typeof res.data?.total_rows === 'number'); if (looksLikeScanResult) { + const tr = (res.data as { total_rows?: unknown }).total_rows; + if (typeof tr === 'number' && tr > 0) { + csvImportRowTotal = Math.max(csvImportRowTotal, tr); + } scanResults = res.data; pushScanToPendingLocal(res.data as Record); showResultModal = true; @@ -965,6 +1040,10 @@ } } if (parsedScan) { + const tr = parsedScan.total_rows; + if (typeof tr === 'number' && tr > 0) { + csvImportRowTotal = Math.max(csvImportRowTotal, tr); + } scanResults = parsedScan; pushScanToPendingLocal(parsedScan); showResultModal = true; @@ -1019,16 +1098,21 @@ commitResults = res.data; showResultModal = true; const inserted = res.data?.inserted || 0; - const skippedInvalid = res.data?.skipped_invalid || 0; - const skippedFk = res.data?.skipped_missing_fk || 0; - const skippedDup = res.data?.skipped_duplicate || 0; - const totalSkipped = skippedInvalid + skippedFk + skippedDup; + const updated = res.data?.updated || 0; + const totalSkipped = totalSkippedFromCommit(res.data as Record); const backendMessage = res.data?.message; + const totalOk = inserted + updated; - if (inserted === 0) { - toast.error(backendMessage || `No se insertaron registros. ${totalSkipped} fueron rechazados.`); + if (totalOk === 0) { + toast.error( + backendMessage || + `No se insertaron ni actualizaron registros. ${totalSkipped} fueron rechazados.` + ); } else { - toast.warning(backendMessage || `Solo se insertaron ${inserted} de ${inserted + totalSkipped} registros.`); + toast.warning( + backendMessage || + `Se aplicaron ${totalOk} registros (${inserted} insertados, ${updated} actualizados). ${totalSkipped} rechazados.` + ); } finalizeCommitAndClearPending(); isUploading = false; @@ -1037,18 +1121,14 @@ showResultModal = true; const inserted = res.data?.inserted || 0; const updated = res.data?.updated || 0; - const skippedInvalid = res.data?.skipped_invalid || 0; - const skippedFk = res.data?.skipped_missing_fk || 0; - const skippedDup = res.data?.skipped_duplicate || 0; - const skippedDetails = res.data?.skipped_details || []; + const totalSkipped = totalSkippedFromCommit(res.data as Record); if (inserted > 0 || updated > 0) { const parts = []; if (inserted > 0) parts.push(`${inserted} insertados`); if (updated > 0) parts.push(`${updated} actualizados`); toast.success(`Importación completada: ${parts.join(', ')}`); - if (skippedInvalid > 0 || skippedFk > 0 || skippedDup > 0) { - const totalSkipped = skippedInvalid + skippedFk + skippedDup; + if (totalSkipped > 0) { toast.warning(`${totalSkipped} registros fueron rechazados u omitidos`); } } else { @@ -1059,14 +1139,14 @@ } else { // Continue polling console.log('Status not final, polling again in 2s...', res.data?.status); - setTimeout(pollStatus, 2000); + setTimeout(pollStatus, 800); } } catch (e) { console.error('Poll exception', e); // Retry on network error? Or fail? // For now, let's keep retrying a few times or hard fail. // Let's just log and retry. - setTimeout(pollStatus, 2000); + setTimeout(pollStatus, 800); } } @@ -1093,7 +1173,11 @@

Catálogos Generales

- + @@ -1101,7 +1185,11 @@

Logística y Transporte

- + @@ -1109,67 +1197,93 @@

Operaciones de Importación

- +

Operaciones de Exportación

- +
- - + +
- {#if isUploading && !showResultModal} -
- {#if csvResumeOverlayHint} -

- Reanudando la importación guardada en esta pestaña… -

- {/if} - {#if currentImportLabel} -

{currentImportLabel}

- {/if} -
-

{csvProgressStepTitle}

- {#if csvProgressPercentText} - - {/if} -
- {#if csvProgressDetailLine} -

{csvProgressDetailLine}

- {/if} -
- {#if csvProgressBarIndeterminate} -
-
+ +
+ {#if isUploading && !showResultModal} +
+
+ {#if csvResumeOverlayHint} +

Reanudando la importación guardada en esta pestaña…

+ {/if} + {#if currentImportLabel} +

{currentImportLabel}

+ {/if} +
+

+ {csvProgressStepTitle} +

+ {#if csvFooterPercentLabel} + + {/if}
- {:else} - - {/if} + {#if csvProgressDetailLine} +

{csvProgressDetailLine}

+ {/if} +
+ {#if csvFooterProgressIndeterminate} +
+
+
+ {:else} + + {/if} +
+
-
- {/if} - - - + {/if} + +
{#if scanResults || commitResults} @@ -1213,7 +1327,6 @@ isUploading = true; csvProgressPhase = 'commit'; scanProgressCurrent = 0; - scanProgressTotal = 0; const res = useCustomsBrokerImport ? await api.customsBrokerImports.commit(currentJobId) : useClientProviderImport From 78c534df10cec816edcd5206b743548f18cf3d92 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 08:32:08 -0500 Subject: [PATCH 14/32] Add models for transportation in test configuration Updated the test configuration file to include imports for Trailer, Transporter, and Vehicle models from the transportation module, enhancing the testing capabilities for related functionalities. --- backend/tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 3f601f19..a1335dd1 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -10,6 +10,9 @@ from sqlalchemy.orm import Session, sessionmaker # Evita errores tipo: expression 'Pedimentos' failed to locate a name. from api.v1.modules.a76.pedmientos.models.pedimentos import Pedimentos # noqa: F401 from api.v1.modules.a76.invoices.models import InvoiceHeader +from api.v1.modules.a76.transportation.trailers.models import Trailer # noqa: F401 +from api.v1.modules.a76.transportation.transporters.models import Transporter # noqa: F401 +from api.v1.modules.a76.transportation.vehicles.models import Vehicle # noqa: F401 from api.v1.modules.a76.invoices.imports.process import routes as process_routes from api.v1.modules.a76.invoices.imports.process import main_process as import_main_process from api.v1.modules.a76.invoices.exports.process import main_process as export_main_process From 309a26bf8f6e838ae816b6e40e0dd934694f5148 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 08:36:16 -0500 Subject: [PATCH 15/32] Refactor sort_order parameter in tenant and invoice routes to use 'pattern' instead of 'regex' for validation. This change enhances consistency across the API query parameters. --- backend/api/v1/common/tenant_crud_routes.py | 4 ++-- backend/api/v1/modules/a76/invoices/routes.py | 2 +- backend/api/v1/modules/a76/items/routes.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/api/v1/common/tenant_crud_routes.py b/backend/api/v1/common/tenant_crud_routes.py index 01c65047..2bb7dffe 100644 --- a/backend/api/v1/common/tenant_crud_routes.py +++ b/backend/api/v1/common/tenant_crud_routes.py @@ -149,7 +149,7 @@ class TenantCRUDRoutes( description="Page size", ), sort_by: Optional[str] = Query(None, description="Column to sort by"), - sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"), + sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"), db: Session = Depends(self.db_dependency), current_user: Dict[str, Any] = Depends(self.auth_dependency), ): @@ -226,7 +226,7 @@ class TenantCRUDRoutes( description="Page size", ), sort_by: Optional[str] = Query(None, description="Column to sort by"), - sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"), + sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"), db: Session = Depends(self.db_dependency), current_user: Dict[str, Any] = Depends(self.auth_dependency), ): diff --git a/backend/api/v1/modules/a76/invoices/routes.py b/backend/api/v1/modules/a76/invoices/routes.py index bf4757aa..05b983fb 100644 --- a/backend/api/v1/modules/a76/invoices/routes.py +++ b/backend/api/v1/modules/a76/invoices/routes.py @@ -99,7 +99,7 @@ def list_invoices( project_number: str = Query(None, description="Filter by project number"), year: str = Query(None, description="Filter by year"), sort_by: Optional[str] = Query(None, description="Column to sort by"), - sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"), + sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): diff --git a/backend/api/v1/modules/a76/items/routes.py b/backend/api/v1/modules/a76/items/routes.py index 4e9a83e4..3f8fb8a8 100644 --- a/backend/api/v1/modules/a76/items/routes.py +++ b/backend/api/v1/modules/a76/items/routes.py @@ -84,7 +84,7 @@ async def list_items( search: Optional[str] = Query( None, description="Search term for invoice number, reference, order, or guide"), sort_by: Optional[str] = Query(None, description="Column to sort by"), - sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"), + sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): @@ -178,7 +178,7 @@ async def list_items_by_invoice( skip: int = Query(0, ge=0, description="Number of records to skip"), limit: int = Query(100, ge=1, le=1000, description="Maximum records to return"), sort_by: Optional[str] = Query(None, description="Column to sort by"), - sort_order: Optional[str] = Query("asc", regex="^(asc|desc)$", description="Sort order (asc or desc)"), + sort_order: Optional[str] = Query("asc", pattern="^(asc|desc)$", description="Sort order (asc or desc)"), db: Session = Depends(get_core_db), current_user: Dict[str, Any] = Depends(get_current_user), ): From 18e3c0a1bd98be70344569bab09136193cfeaeb6 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 09:54:50 -0500 Subject: [PATCH 16/32] Refactor invoice mapping logic in edit page to improve data handling. Updated the mapping of logistics and compliance fields to use optional chaining and default values, enhancing robustness and clarity in the invoice data structure. --- .../dashboard/invoices/edit/[id]/+page.svelte | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte b/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte index d61d846f..cebf8012 100644 --- a/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte +++ b/frontend/src/routes/dashboard/invoices/edit/[id]/+page.svelte @@ -347,28 +347,38 @@ function mapInvoiceToContinuation(invoice: any) { if (!invoice) return continuationSkeleton; + const logistics = invoice.logistics ?? {}; + const complianceMx = invoice.compliance_mx ?? {}; + return { - numero_tipo_transporte: invoice.logistics?.numero_tipo_transporte || '', - es_ferrocarril: invoice.logistics?.es_ferrocarril || 'no', - numero_bl: invoice.logistics?.numero_bl || '', - cantidad_guias_embarque: invoice.logistics?.cantidad_guias_embarque || null, - destino_origen: invoice.logistics?.destino_origen || '', - puerto_entrada: invoice.logistics?.puerto_entrada || '', - vehicle_data: invoice.logistics?.vehicle_data || '', - fue_revisado_equipo: invoice.logistics?.fue_revisado_equipo || false, - sub_division: invoice.compliance_mx?.subdivision || false, - funge_como_cd: invoice.logistics?.acts_as_cd || false, - llego_pedimento: invoice.compliance_mx?.llego_pedimento || false, + numero_tipo_transporte: + logistics.transport_num ?? logistics.vehicle_num ?? logistics.numero_tipo_transporte ?? '', + es_ferrocarril: + typeof logistics.is_rail === 'boolean' + ? logistics.is_rail + ? 'si' + : 'no' + : (logistics.es_ferrocarril ?? 'no'), + numero_bl: logistics.bill_number ?? logistics.numero_bl ?? '', + cantidad_guias_embarque: logistics.guide_number ?? logistics.cantidad_guias_embarque ?? null, + destino_origen: logistics.destination_location ?? logistics.destino_origen ?? '', + puerto_entrada: logistics.origin_location ?? logistics.puerto_entrada ?? '', + vehicle_data: logistics.vehicle_data ?? '', + fue_revisado_equipo: logistics.equipment_reviewed ?? logistics.fue_revisado_equipo ?? false, + sub_division: logistics.is_subdivision ?? complianceMx.subdivision ?? false, + funge_como_cd: logistics.acts_as_cd ?? false, + llego_pedimento: logistics.pedimento_arrived ?? complianceMx.llego_pedimento ?? false, errores_facturacion: invoice.errores_facturacion || [], semaforo_verde_aduana_mexicana: - invoice.compliance_mx?.semaforo_verde_aduana_mexicana || false, + logistics.green_light_mx ?? complianceMx.semaforo_verde_aduana_mexicana ?? false, semaforo_verde_aduana_americana: - invoice.compliance_mx?.semaforo_verde_aduana_americana || false, - semaforo_rojo_aduana_mexicana: invoice.compliance_mx?.semaforo_rojo_aduana_mexicana || false, + logistics.green_light_us ?? complianceMx.semaforo_verde_aduana_americana ?? false, + semaforo_rojo_aduana_mexicana: + logistics.red_light_mx ?? complianceMx.semaforo_rojo_aduana_mexicana ?? false, semaforo_rojo_aduana_americana: - invoice.compliance_mx?.semaforo_rojo_aduana_americana || false, - is_mixed: invoice.compliance_mx?.is_mixed || false, - reason_export: invoice.compliance_mx?.reason_export || '1', + logistics.red_light_us ?? complianceMx.semaforo_rojo_aduana_americana ?? false, + is_mixed: complianceMx.is_mixed ?? false, + reason_export: complianceMx.reason_export ?? '1', purchase_order: invoice.purchase_order || '', payment_terms: invoice.payment_terms || '', handling_fees: invoice.financials?.handling_fees || 0, From 22e69b07eecc140613d733cee973e1f0c82a19e4 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:06:53 -0500 Subject: [PATCH 17/32] Refactor Jenkinsfile to simplify agent configuration. Changed from a specific agent label to 'any', allowing for more flexible build agent selection. --- Jenkinsfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2a8a466f..df4af8f4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,5 @@ pipeline { - agent { - label 'master' - } - + agent any options { timestamps() disableConcurrentBuilds() From 8d04a945b01bc4473c0123ae253a30c249982a79 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:08:59 -0500 Subject: [PATCH 18/32] Enhance Jenkinsfile to handle multiple requirements files. Added conditional checks to install dependencies from either requirements.txt or base.txt, with error handling for missing files. --- Jenkinsfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index df4af8f4..0af8f6ef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -94,7 +94,16 @@ pipeline { set -euxo pipefail python --version python -m pip install --upgrade pip - pip install -r /workspace/backend/requirements.txt + if [ -f /workspace/backend/requirements.txt ]; then + pip install -r /workspace/backend/requirements.txt + elif [ -f /workspace/backend/requirements/base.txt ]; then + pip install -r /workspace/backend/requirements/base.txt + else + echo "ERROR: No requirements file found in /workspace/backend" + ls -la /workspace/backend || true + ls -la /workspace/backend/requirements || true + exit 1 + fi cd /workspace/backend alembic upgrade head pytest -q tests -v -ra -s From 65030361cfb796a3fa8ea6d012f4daf73f4cbffa Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:11:40 -0500 Subject: [PATCH 19/32] Refactor Jenkinsfile to dynamically locate the repository root for backend dependencies. Improved error handling for missing requirements files and updated the installation commands to reflect the new directory structure. --- Jenkinsfile | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0af8f6ef..ab2a87b4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -94,17 +94,30 @@ pipeline { set -euxo pipefail python --version python -m pip install --upgrade pip - if [ -f /workspace/backend/requirements.txt ]; then - pip install -r /workspace/backend/requirements.txt - elif [ -f /workspace/backend/requirements/base.txt ]; then - pip install -r /workspace/backend/requirements/base.txt - else - echo "ERROR: No requirements file found in /workspace/backend" - ls -la /workspace/backend || true - ls -la /workspace/backend/requirements || true + REPO_ROOT="" + for d in /workspace /workspace/*; do + if [ -d "$d/backend" ]; then + REPO_ROOT="$d" + break + fi + done + if [ -z "$REPO_ROOT" ]; then + echo "ERROR: Could not locate repository root containing backend/ directory" + ls -la /workspace || true exit 1 fi - cd /workspace/backend + echo "Detected REPO_ROOT=$REPO_ROOT" + if [ -f "$REPO_ROOT/backend/requirements.txt" ]; then + pip install -r "$REPO_ROOT/backend/requirements.txt" + elif [ -f "$REPO_ROOT/backend/requirements/base.txt" ]; then + pip install -r "$REPO_ROOT/backend/requirements/base.txt" + else + echo "ERROR: No requirements file found in $REPO_ROOT/backend" + ls -la "$REPO_ROOT/backend" || true + ls -la "$REPO_ROOT/backend/requirements" || true + exit 1 + fi + cd "$REPO_ROOT/backend" alembic upgrade head pytest -q tests -v -ra -s ' From 1790838a44fa2daa57c3e5a0e17920ab7bb808c3 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:13:15 -0500 Subject: [PATCH 20/32] Update Jenkinsfile to use Docker networks for test containers. Changed database connection URL to reference the container name and added cleanup for the test network. Improved handling of workspace paths for dependency installation. --- Jenkinsfile | 51 +++++++++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ab2a87b4..c16620cc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,21 +58,26 @@ pipeline { DB_USER="anexo76" DB_PASS="anexo76" DB_PORT="55432" - export TEST_DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@127.0.0.1:${DB_PORT}/${DB_NAME}" + export TEST_DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@${DB_CONTAINER}:5432/${DB_NAME}" PY_CONTAINER="anexo76-test-py-${BUILD_NUMBER}" + TEST_NETWORK="anexo76-test-net-${BUILD_NUMBER}" cleanup() { docker rm -f "$PY_CONTAINER" >/dev/null 2>&1 || true docker rm -f "$DB_CONTAINER" >/dev/null 2>&1 || true + docker network rm "$TEST_NETWORK" >/dev/null 2>&1 || true } trap cleanup EXIT + docker network rm "$TEST_NETWORK" >/dev/null 2>&1 || true + docker network create "$TEST_NETWORK" + docker rm -f "$DB_CONTAINER" >/dev/null 2>&1 || true docker run -d --name "$DB_CONTAINER" \ + --network "$TEST_NETWORK" \ -e POSTGRES_DB="$DB_NAME" \ -e POSTGRES_USER="$DB_USER" \ -e POSTGRES_PASSWORD="$DB_PASS" \ - -p "${DB_PORT}:5432" \ postgres:16-alpine # Espera a que Postgres acepte conexiones @@ -84,40 +89,30 @@ pipeline { done docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" - docker run --name "$PY_CONTAINER" --rm \ - --network host \ + docker rm -f "$PY_CONTAINER" >/dev/null 2>&1 || true + docker run -d --name "$PY_CONTAINER" --network "$TEST_NETWORK" python:3.12-slim sleep infinity + docker exec "$PY_CONTAINER" mkdir -p /workspace + docker cp "$WORKSPACE/." "$PY_CONTAINER:/workspace" + + docker exec \ -e TEST_DATABASE_URL="$TEST_DATABASE_URL" \ - -v "$WORKSPACE:/workspace" \ - -w /workspace \ - python:3.12-slim \ + "$PY_CONTAINER" \ sh -lc ' set -euxo pipefail python --version python -m pip install --upgrade pip - REPO_ROOT="" - for d in /workspace /workspace/*; do - if [ -d "$d/backend" ]; then - REPO_ROOT="$d" - break - fi - done - if [ -z "$REPO_ROOT" ]; then - echo "ERROR: Could not locate repository root containing backend/ directory" - ls -la /workspace || true - exit 1 - fi - echo "Detected REPO_ROOT=$REPO_ROOT" - if [ -f "$REPO_ROOT/backend/requirements.txt" ]; then - pip install -r "$REPO_ROOT/backend/requirements.txt" - elif [ -f "$REPO_ROOT/backend/requirements/base.txt" ]; then - pip install -r "$REPO_ROOT/backend/requirements/base.txt" + if [ -f /workspace/backend/requirements.txt ]; then + pip install -r /workspace/backend/requirements.txt + elif [ -f /workspace/backend/requirements/base.txt ]; then + pip install -r /workspace/backend/requirements/base.txt else - echo "ERROR: No requirements file found in $REPO_ROOT/backend" - ls -la "$REPO_ROOT/backend" || true - ls -la "$REPO_ROOT/backend/requirements" || true + echo "ERROR: No requirements file found in /workspace/backend" + ls -la /workspace || true + ls -la /workspace/backend || true + ls -la /workspace/backend/requirements || true exit 1 fi - cd "$REPO_ROOT/backend" + cd /workspace/backend alembic upgrade head pytest -q tests -v -ra -s ' From 5ef77a9eefaf2befa7f252f95e5b6ed6e94f6b82 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:15:08 -0500 Subject: [PATCH 21/32] Add schema creation commands in Jenkinsfile for database setup Updated the Jenkinsfile to include commands for creating necessary database schemas (core, a24, a76, and public) during the setup process, ensuring the database is properly initialized for testing. --- Jenkinsfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index c16620cc..cdd45885 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -88,6 +88,12 @@ pipeline { sleep 2 done docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" + docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -v ON_ERROR_STOP=1 -c " + CREATE SCHEMA IF NOT EXISTS core; + CREATE SCHEMA IF NOT EXISTS a24; + CREATE SCHEMA IF NOT EXISTS a76; + CREATE SCHEMA IF NOT EXISTS public; + " docker rm -f "$PY_CONTAINER" >/dev/null 2>&1 || true docker run -d --name "$PY_CONTAINER" --network "$TEST_NETWORK" python:3.12-slim sleep infinity From da6358b63d2e8cbb112fdf7123819833232e8deb Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:39:15 -0500 Subject: [PATCH 22/32] Enhance Jenkinsfile to improve SSH command execution. Added SSH private key handling and refactored the deployment commands into a script block for better readability and maintainability. --- Jenkinsfile | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cdd45885..afebe04c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -220,19 +220,25 @@ pipeline { withCredentials([ usernamePassword(credentialsId: 'harbor-credentials', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD'), string(credentialsId: 'dev-server-user', variable: 'DEV_SERVER_USER'), - string(credentialsId: 'dev-server-host', variable: 'DEV_SERVER_HOST') + string(credentialsId: 'dev-server-host', variable: 'DEV_SERVER_HOST'), + sshUserPrivateKey(credentialsId: 'dev-server-ssh', keyFileVariable: 'DEV_SERVER_KEY') ]) { - sshagent(credentials: ['dev-server-ssh']) { - sh ''' + script { + def remote = [ + name: 'dev-server', + host: env.DEV_SERVER_HOST, + user: env.DEV_SERVER_USER, + identityFile: env.DEV_SERVER_KEY, + allowAnyHosts: true + ] + sshCommand remote: remote, command: """ set -euo pipefail - ssh -o StrictHostKeyChecking=accept-new "${DEV_SERVER_USER}@${DEV_SERVER_HOST}" ' - cd ~/projects/anexo76 && - echo '"'"'"${HARBOR_PASSWORD}"'"'"' | docker login '"${REGISTRY}"' -u '"${HARBOR_USERNAME}"' --password-stdin && - docker compose -f docker-compose.prod.yml pull && - docker compose -f docker-compose.prod.yml up -d && - docker image prune -f - ' - ''' + cd ~/projects/anexo76 + echo '${env.HARBOR_PASSWORD}' | docker login '${env.REGISTRY}' -u '${env.HARBOR_USERNAME}' --password-stdin + docker compose -f docker-compose.prod.yml pull + docker compose -f docker-compose.prod.yml up -d + docker image prune -f + """ } } } From d3318deb859e592faf282326574e79db89998c7e Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:42:20 -0500 Subject: [PATCH 23/32] Update Jenkinsfile to improve SSH user handling and enhance security. Replaced direct user variable with a dedicated SSH user variable and added comments to clarify the docker login process, ensuring sensitive information is not exposed. --- Jenkinsfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index afebe04c..ca8b37fd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -219,26 +219,29 @@ pipeline { steps { withCredentials([ usernamePassword(credentialsId: 'harbor-credentials', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD'), - string(credentialsId: 'dev-server-user', variable: 'DEV_SERVER_USER'), string(credentialsId: 'dev-server-host', variable: 'DEV_SERVER_HOST'), - sshUserPrivateKey(credentialsId: 'dev-server-ssh', keyFileVariable: 'DEV_SERVER_KEY') + sshUserPrivateKey( + credentialsId: 'dev-server-ssh', + keyFileVariable: 'DEV_SERVER_KEY', + usernameVariable: 'DEV_SERVER_SSH_USER' + ) ]) { script { def remote = [ name: 'dev-server', host: env.DEV_SERVER_HOST, - user: env.DEV_SERVER_USER, + user: env.DEV_SERVER_SSH_USER, identityFile: env.DEV_SERVER_KEY, allowAnyHosts: true ] - sshCommand remote: remote, command: """ + sshCommand remote: remote, command: ''' set -euo pipefail cd ~/projects/anexo76 - echo '${env.HARBOR_PASSWORD}' | docker login '${env.REGISTRY}' -u '${env.HARBOR_USERNAME}' --password-stdin + # Asume docker login ya configurado en el host destino para evitar exponer secretos por interpolación. docker compose -f docker-compose.prod.yml pull docker compose -f docker-compose.prod.yml up -d docker image prune -f - """ + ''' } } } From 6d7020b5c4cfa8277027cef38e048981778e6529 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 10:52:55 -0500 Subject: [PATCH 24/32] Add DEV_SERVER_USER credential handling in Jenkinsfile Updated the Jenkinsfile to include a new credential for the development server user, enhancing the flexibility of user management during deployment. The SSH user variable now defaults to this new credential if available, improving security and maintainability. --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca8b37fd..109c43f5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -219,6 +219,7 @@ pipeline { steps { withCredentials([ usernamePassword(credentialsId: 'harbor-credentials', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD'), + string(credentialsId: 'dev-server-user', variable: 'DEV_SERVER_USER'), string(credentialsId: 'dev-server-host', variable: 'DEV_SERVER_HOST'), sshUserPrivateKey( credentialsId: 'dev-server-ssh', @@ -230,7 +231,7 @@ pipeline { def remote = [ name: 'dev-server', host: env.DEV_SERVER_HOST, - user: env.DEV_SERVER_SSH_USER, + user: env.DEV_SERVER_USER ?: env.DEV_SERVER_SSH_USER, identityFile: env.DEV_SERVER_KEY, allowAnyHosts: true ] From 36424a2e04e19cbf88890e9de047998c109ffba6 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 11:00:37 -0500 Subject: [PATCH 25/32] Enhance Jenkinsfile to include passphrase handling for SSH key authentication. Added a passphrase variable for the development server SSH key, improving security during deployment processes. --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 109c43f5..6512be28 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -224,7 +224,8 @@ pipeline { sshUserPrivateKey( credentialsId: 'dev-server-ssh', keyFileVariable: 'DEV_SERVER_KEY', - usernameVariable: 'DEV_SERVER_SSH_USER' + usernameVariable: 'DEV_SERVER_SSH_USER', + passphraseVariable: 'DEV_SERVER_KEY_PASSPHRASE' ) ]) { script { @@ -233,6 +234,7 @@ pipeline { host: env.DEV_SERVER_HOST, user: env.DEV_SERVER_USER ?: env.DEV_SERVER_SSH_USER, identityFile: env.DEV_SERVER_KEY, + passphrase: env.DEV_SERVER_KEY_PASSPHRASE, allowAnyHosts: true ] sshCommand remote: remote, command: ''' From b784ea1cd536bb0ba2309816677c8467c6189103 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 11:18:18 -0500 Subject: [PATCH 26/32] Refactor Jenkinsfile to enhance build and deployment processes. Updated agent configuration to use a specific Docker label, improved database readiness checks with error handling, and implemented retry logic for Docker builds and pushes. Additionally, refined SSH key handling and ensured proper tagging for Docker images during the build process. --- Jenkinsfile | 76 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6512be28..8a5be402 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,7 @@ pipeline { - agent any + agent { + label 'docker' + } options { timestamps() disableConcurrentBuilds() @@ -57,7 +59,6 @@ pipeline { DB_NAME="anexo76_test" DB_USER="anexo76" DB_PASS="anexo76" - DB_PORT="55432" export TEST_DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@${DB_CONTAINER}:5432/${DB_NAME}" PY_CONTAINER="anexo76-test-py-${BUILD_NUMBER}" TEST_NETWORK="anexo76-test-net-${BUILD_NUMBER}" @@ -80,13 +81,19 @@ pipeline { -e POSTGRES_PASSWORD="$DB_PASS" \ postgres:16-alpine - # Espera a que Postgres acepte conexiones + # Espera a que Postgres acepte conexiones (hasta ~60s) + READY=0 for i in $(seq 1 30); do if docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + READY=1 break fi sleep 2 done + if [ "$READY" != "1" ]; then + echo "ERROR: Postgres no quedó listo a tiempo (30 intentos × 2s)." + exit 1 + fi docker exec "$DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -v ON_ERROR_STOP=1 -c " CREATE SCHEMA IF NOT EXISTS core; @@ -165,33 +172,47 @@ pipeline { stage('Build + push backend') { steps { - sh ''' - set -euo pipefail - docker build \ - --build-arg APP_VERSION="${APP_VERSION}" \ - -t "${REGISTRY}/${IMAGE_NAMESPACE}/backend:latest" \ - -f ./backend/Dockerfile \ - ./backend + script { + retry(3) { + sh ''' + set -euo pipefail + export DOCKER_BUILDKIT=1 + docker build \ + --build-arg APP_VERSION="${APP_VERSION}" \ + -t "${REGISTRY}/${IMAGE_NAMESPACE}/backend:${APP_VERSION}" \ + -t "${REGISTRY}/${IMAGE_NAMESPACE}/backend:latest" \ + -f ./backend/Dockerfile \ + ./backend - docker push "${REGISTRY}/${IMAGE_NAMESPACE}/backend:latest" - ''' + docker push "${REGISTRY}/${IMAGE_NAMESPACE}/backend:${APP_VERSION}" + docker push "${REGISTRY}/${IMAGE_NAMESPACE}/backend:latest" + ''' + } + } } } stage('Build + push frontend') { steps { - sh ''' - set -euo pipefail - docker build \ - --build-arg VITE_API_URL=https://anexo76-dev.aduanasoft.com/api/ \ - --build-arg VITE_KEYCLOAK_URL=https://anexo76-dev.aduanasoft.com/kcauth/ \ - --build-arg INTERNAL_API_URL=http://backend:3467/api/ \ - -t "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:latest" \ - -f ./frontend/Dockerfile.prod \ - ./frontend + script { + retry(3) { + sh ''' + set -euo pipefail + export DOCKER_BUILDKIT=1 + docker build \ + --build-arg VITE_API_URL=https://anexo76-dev.aduanasoft.com/api/ \ + --build-arg VITE_KEYCLOAK_URL=https://anexo76-dev.aduanasoft.com/kcauth/ \ + --build-arg INTERNAL_API_URL=http://backend:3467/api/ \ + -t "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:${APP_VERSION}" \ + -t "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:latest" \ + -f ./frontend/Dockerfile.prod \ + ./frontend - docker push "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:latest" - ''' + docker push "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:${APP_VERSION}" + docker push "${REGISTRY}/${IMAGE_NAMESPACE}/frontend:latest" + ''' + } + } } } @@ -229,13 +250,19 @@ pipeline { ) ]) { script { + sh """ + set -euo pipefail + mkdir -p ~/.ssh + chmod 700 ~/.ssh + ssh-keyscan -T 15 -H '${env.DEV_SERVER_HOST}' >> ~/.ssh/known_hosts + """ def remote = [ name: 'dev-server', host: env.DEV_SERVER_HOST, user: env.DEV_SERVER_USER ?: env.DEV_SERVER_SSH_USER, identityFile: env.DEV_SERVER_KEY, passphrase: env.DEV_SERVER_KEY_PASSPHRASE, - allowAnyHosts: true + allowAnyHosts: false ] sshCommand remote: remote, command: ''' set -euo pipefail @@ -243,6 +270,7 @@ pipeline { # Asume docker login ya configurado en el host destino para evitar exponer secretos por interpolación. docker compose -f docker-compose.prod.yml pull docker compose -f docker-compose.prod.yml up -d + # Solo imágenes colgantes (sin tag); no borra imágenes en uso por otros contenedores. docker image prune -f ''' } From 5c18108e792cc7daf510e7a2b6eeff6f594dcc62 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 11:20:56 -0500 Subject: [PATCH 27/32] Refactor Jenkinsfile to simplify agent configuration by changing from a specific Docker label to 'any', allowing for more flexible build agent selection. --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8a5be402..956c454c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,5 @@ pipeline { - agent { - label 'docker' - } + agent any options { timestamps() disableConcurrentBuilds() From 49d635b79a2cd907a6335c7618ca9e1d51cf7270 Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 11:24:04 -0500 Subject: [PATCH 28/32] Refactor Jenkinsfile to improve SSH known hosts handling. Introduced a variable for the development server host and updated the known hosts file path, enhancing security and maintainability during deployment processes. --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 956c454c..321fead5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -248,19 +248,20 @@ pipeline { ) ]) { script { + def devHost = env.DEV_SERVER_HOST?.trim() + def knownHostsFile = "${env.WORKSPACE}/.jenkins-anexo76-known_hosts" sh """ set -euo pipefail - mkdir -p ~/.ssh - chmod 700 ~/.ssh - ssh-keyscan -T 15 -H '${env.DEV_SERVER_HOST}' >> ~/.ssh/known_hosts + ssh-keyscan -T 15 -H '${devHost}' > '${knownHostsFile}' """ def remote = [ name: 'dev-server', - host: env.DEV_SERVER_HOST, + host: devHost, user: env.DEV_SERVER_USER ?: env.DEV_SERVER_SSH_USER, identityFile: env.DEV_SERVER_KEY, passphrase: env.DEV_SERVER_KEY_PASSPHRASE, - allowAnyHosts: false + allowAnyHosts: false, + knownHosts: knownHostsFile ] sshCommand remote: remote, command: ''' set -euo pipefail From 4bc38b691f59d665adf9dde00751d28ad582f58b Mon Sep 17 00:00:00 2001 From: AlexeerCT Date: Thu, 2 Apr 2026 11:31:00 -0500 Subject: [PATCH 29/32] Add APP_VERSION handling in Dockerfile and config.py Updated the Dockerfile to include an ARG and ENV for APP_VERSION, allowing for version specification during build time. Modified config.py to set a default APP_VERSION, which can be overridden by the environment, enhancing flexibility for deployment and runtime configuration. --- backend/Dockerfile | 4 ++++ backend/core/config.py | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 6f6a9c20..84d0d41d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -45,6 +45,10 @@ RUN pip install --no-cache-dir -r requirements.txt # Copiar código COPY . . +# Jenkins / CI pasan --build-arg APP_VERSION=…; debe quedar en ENV para runtime (API /version, OpenAPI, etc.) +ARG APP_VERSION=dev-local +ENV APP_VERSION=${APP_VERSION} + # Exponer puerto EXPOSE 8000 diff --git a/backend/core/config.py b/backend/core/config.py index eaabdb82..3fad873f 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -2,7 +2,6 @@ Configuración centralizada de la aplicación usando Pydantic Settings """ -import os from typing import List from pydantic import field_validator @@ -14,9 +13,8 @@ class Settings(BaseSettings): # Application APP_NAME: str = "Anexo76" - # La versión se obtiene de la variable de entorno APP_VERSION que se pasa desde Docker - # Si no existe, usa un valor por defecto de desarrollo - APP_VERSION: str = os.getenv("APP_VERSION", "dev-local") + # Sobreescribible con APP_VERSION (Dockerfile/Jenkins: build-arg + ENV) o entorno en runtime + APP_VERSION: str = "dev-local" DEBUG: bool = True ENVIRONMENT: str = "development" From 9439a6a77520f61622fd5063ec80929a5a98a764 Mon Sep 17 00:00:00 2001 From: hreyes Date: Thu, 2 Apr 2026 13:35:18 -0600 Subject: [PATCH 30/32] feature/minio-s3-integration --- .env.example | 19 + ...d4e5f6a7b8c9_company_logo_s3_key_length.py | 39 +++ .../v1/modules/a76/customs_brokers/routes.py | 303 +++++++++++++++- .../modules/a76/customs_brokers/services.py | 20 ++ .../a76/general_catalogs/company/dto.py | 2 +- .../a76/general_catalogs/company/models.py | 2 +- .../a76/general_catalogs/company/routes.py | 145 +++++--- .../a76/general_catalogs/company/service.py | 5 + .../v1/modules/a76/layouts_csv/boms/routes.py | 37 +- .../cambio_regimen_regularizacion/routes.py | 31 +- .../modules/a76/layouts_csv/classes/routes.py | 37 +- .../clients_and_providers/routes.py | 37 +- .../modules/a76/layouts_csv/common/storage.py | 118 ++++++- .../a76/layouts_csv/customs_brokers/routes.py | 37 +- .../modules/a76/layouts_csv/drivers/routes.py | 38 +- .../a76/layouts_csv/exchange_rate/routes.py | 37 +- .../a76/layouts_csv/exportacion/routes.py | 31 +- .../a76/layouts_csv/facturas/routes.py | 42 +-- .../modules/a76/layouts_csv/parts/routes.py | 37 +- .../a76/layouts_csv/pedmientos/routes.py | 37 +- .../a76/layouts_csv/trailers/routes.py | 38 +- .../a76/layouts_csv/transportistas/routes.py | 38 +- .../layouts_csv/us_tariff_fractions/routes.py | 37 +- .../a76/layouts_csv/vehicles/routes.py | 38 +- .../api/v1/modules/core/help_center/routes.py | 108 ++++-- .../api/v1/modules/core/help_center/utils.py | 129 ++++--- backend/api/v1/modules/core/users/routes.py | 135 +++++++- backend/api/v1/modules/core/users/service.py | 17 +- backend/core/config.py | 22 +- backend/core/middleware.py | 2 + backend/core/s3_keys.py | 325 ++++++++++++++++++ backend/core/storage_s3.py | 120 +++++++ backend/main.py | 5 +- backend/requirements.txt | 1 + docker-compose.prod.yml | 56 +++ docker-compose.yml | 56 +++ .../lib/api/dashboard/a76/customs-brokers.ts | 56 ++- .../dashboard/a76/general_catalogs/company.ts | 13 +- frontend/src/lib/utils.ts | 43 ++- .../customs_brokers/edit/[[id]]/+page.svelte | 154 ++++++++- .../edit/[[id]]/+page.svelte | 36 +- .../help-center/editor/[uuid]/+page.svelte | 5 +- 42 files changed, 1930 insertions(+), 558 deletions(-) create mode 100644 backend/alembic/versions/d4e5f6a7b8c9_company_logo_s3_key_length.py create mode 100644 backend/core/s3_keys.py create mode 100644 backend/core/storage_s3.py diff --git a/.env.example b/.env.example index 46bb47bc..1ef60c19 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,25 @@ VITE_KEYCLOAK_CLIENT_ID=anexo76-frontend #------ Celery / Valkey ---------- VALKEY_URL=redis://valkey:6379/0 +# ----- MinIO (S3-compatible) ----- +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=minioadmin +MINIO_API_PORT=9100 +MINIO_CONSOLE_PORT=9101 + +# ----- Imports CSV (layouts_csv): redis | minio ----- +CSV_IMPORT_STORAGE=minio +S3_ENDPOINT_URL=http://minio:9000 +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +S3_BUCKET=anexo76 +S3_REGION=us-east-1 +S3_USE_SSL=false +# Logos, certificados, help: mismo bucket. Si CSV_IMPORT_STORAGE=minio, también se usa MinIO aquí +# (use_s3_object_storage = minio CSV o S3_FILE_STORAGE=true). +S3_FILE_STORAGE=true +S3_PRESIGNED_EXPIRES_SECONDS=3600 + # ----- Sitar API ----- SITAR_API_URL=http://api.sitar.aduanasoft.com SITAR_API_USER=user_sitar_api diff --git a/backend/alembic/versions/d4e5f6a7b8c9_company_logo_s3_key_length.py b/backend/alembic/versions/d4e5f6a7b8c9_company_logo_s3_key_length.py new file mode 100644 index 00000000..2b5bc3f6 --- /dev/null +++ b/backend/alembic/versions/d4e5f6a7b8c9_company_logo_s3_key_length.py @@ -0,0 +1,39 @@ +"""extend company.logo for S3 keys + +Revision ID: d4e5f6a7b8c9 +Revises: ca7d3c4e8b2a +Create Date: 2026-04-02 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "d4e5f6a7b8c9" +down_revision: Union[str, None] = "ca7d3c4e8b2a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "company", + "logo", + existing_type=sa.String(length=255), + type_=sa.String(length=512), + existing_nullable=True, + schema="a76", + ) + + +def downgrade() -> None: + op.alter_column( + "company", + "logo", + existing_type=sa.String(length=512), + type_=sa.String(length=255), + existing_nullable=True, + schema="a76", + ) diff --git a/backend/api/v1/modules/a76/customs_brokers/routes.py b/backend/api/v1/modules/a76/customs_brokers/routes.py index 960fb1f9..8635ff2e 100644 --- a/backend/api/v1/modules/a76/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/customs_brokers/routes.py @@ -1,16 +1,69 @@ -from typing import Dict, Any -from fastapi import APIRouter, Depends, HTTPException, Query +import logging +import mimetypes +import os +from datetime import datetime +from typing import Any, Dict, Optional + +from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status from sqlalchemy.orm import Session +from core.config import settings from core.database import get_core_db -from core.security import get_current_user, validate_access_to_resource +from core.s3_keys import ( + customs_broker_vu_certificate_key, + customs_broker_vu_cove_key, + customs_broker_vu_doda_certificate_key, + customs_broker_vu_doda_cove_key, + customs_broker_vu_doda_private_key_key, + customs_broker_vu_private_key_key, +) +from core.security import get_current_user, get_tenant_from_token, validate_access_to_resource +from core.storage_s3 import delete_object_if_exists, put_object_bytes from api.v1.common.tenant_crud_routes import TenantCRUDRoutes from . import dto, services from ..layouts_csv.customs_brokers.routes import router as imports_router +logger = logging.getLogger(__name__) + router = APIRouter() +MAX_VU_CER_KEY_BYTES = 5 * 1024 * 1024 # 5 MB +MAX_COVE_BYTES = 15 * 1024 * 1024 # 15 MB (xml/zip) + + +def _resolve_tenant_id_int(current_user: dict) -> int: + tid = get_tenant_from_token(current_user) + if tid is not None: + return int(tid) + raw = current_user.get("tenant_id") + if isinstance(raw, list) and raw: + raw = raw[0] + if raw is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Tenant ID not found in user data", + ) + try: + return int(raw) + except (TypeError, ValueError): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid tenant ID in token", + ) + + +def _remove_stored_vu_path(ref: Optional[str]) -> None: + if not ref: + return + if ref.startswith("tenants/"): + delete_object_if_exists(ref) + elif os.path.isfile(ref): + try: + os.remove(ref) + except OSError: + pass + # CSV import (mismo flujo que a76.imports: upload → scan → commit) router.include_router(imports_router, prefix="/customs-brokers/imports", tags=["customs_brokers / csv_import"]) @@ -112,4 +165,246 @@ def update_customs_broker_personnel( raise HTTPException( status_code=404, detail="Customs Broker Personnel not found" ) - return updated_personnel \ No newline at end of file + return updated_personnel + + +@router.post( + "/customs-brokers/{broker_key}/vu/upload", + summary="Sube VU/DODA (CER, KEY, COVE) al bucket bajo tenants/.../customs_brokers/{id}/...", +) +async def upload_customs_broker_vu_file( + broker_key: str, + file_kind: str = Query( + ..., + description=( + "certificate (.cer), key (.key), cove (xml/zip/txt/pdf/json), " + "doda_certificate (.cer), doda_key (.key), doda_cove (xml/zip/txt/pdf/json)" + ), + ), + company_id: int = Query(..., description="Company ID"), + file: UploadFile = File(...), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Persiste el archivo bajo la misma jerarquía que logos/avatares (tenant/company/...). + Guarda la clave S3 o ruta local en: + - VU: certificate_path, key_path, xml_files_path + - DODA: doda_certificate_path, doda_key_path, doda_xml_files_path + """ + validate_access_to_resource(db, company_id, current_user) + tenant_id = _resolve_tenant_id_int(current_user) + + broker = services.CustomsBrokerService.get_by_id(db, broker_key, tenant_id, company_id) + if not broker: + raise HTTPException(status_code=404, detail="Customs Broker not found") + + fk = file_kind.lower().strip() + if fk not in ( + "certificate", + "key", + "cove", + "doda_certificate", + "doda_key", + "doda_cove", + ): + raise HTTPException( + status_code=400, + detail=( + "file_kind must be certificate, key, cove, " + "doda_certificate, doda_key, or doda_cove" + ), + ) + + content = await file.read() + max_bytes = MAX_COVE_BYTES if fk in ("cove", "doda_cove") else MAX_VU_CER_KEY_BYTES + if len(content) > max_bytes: + raise HTTPException( + status_code=400, + detail=f"File too large (max {max_bytes // (1024 * 1024)} MB)", + ) + + file_ext = os.path.splitext(file.filename or "")[1].lower() + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + vu = services.CustomsBrokerVUService.ensure_vu_for_broker(db, broker) + broker_id = broker.id + + field_name: str + stored: str + + try: + if settings.use_s3_object_storage: + if fk == "certificate": + if file_ext != ".cer": + raise HTTPException(status_code=400, detail="certificate must be .cer") + key = customs_broker_vu_certificate_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + ct = "application/x-x509-ca-cert" + field_name = "certificate_path" + elif fk == "key": + if file_ext != ".key": + raise HTTPException(status_code=400, detail="key must be .key") + key = customs_broker_vu_private_key_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + ct = "application/pkcs8" + field_name = "key_path" + elif fk == "cove": + key = customs_broker_vu_cove_key( + tenant_id, + company_id, + broker_id, + timestamp, + file.filename or "cove.xml", + ) + ct = ( + file.content_type + or mimetypes.guess_type(file.filename or "")[0] + or "application/octet-stream" + ) + field_name = "xml_files_path" + elif fk == "doda_certificate": + if file_ext != ".cer": + raise HTTPException(status_code=400, detail="doda_certificate must be .cer") + key = customs_broker_vu_doda_certificate_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + ct = "application/x-x509-ca-cert" + field_name = "doda_certificate_path" + elif fk == "doda_key": + if file_ext != ".key": + raise HTTPException(status_code=400, detail="doda_key must be .key") + key = customs_broker_vu_doda_private_key_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + ct = "application/pkcs8" + field_name = "doda_key_path" + else: + key = customs_broker_vu_doda_cove_key( + tenant_id, + company_id, + broker_id, + timestamp, + file.filename or "doda.xml", + ) + ct = ( + file.content_type + or mimetypes.guess_type(file.filename or "")[0] + or "application/octet-stream" + ) + field_name = "doda_xml_files_path" + + old = getattr(vu, field_name) + _remove_stored_vu_path(old) + put_object_bytes(key, content, content_type=ct) + logger.info( + "Customs broker VU upload kind=%s key=%s bytes=%s", + fk, + key, + len(content), + ) + stored = key + else: + base = os.path.join( + "uploads", "customs_brokers", str(company_id), str(broker_id) + ) + if fk == "certificate": + if file_ext != ".cer": + raise HTTPException(status_code=400, detail="certificate must be .cer") + key = customs_broker_vu_certificate_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + field_name = "certificate_path" + subdir = "certificates" + elif fk == "key": + if file_ext != ".key": + raise HTTPException(status_code=400, detail="key must be .key") + key = customs_broker_vu_private_key_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + field_name = "key_path" + subdir = "keys" + elif fk == "cove": + try: + key = customs_broker_vu_cove_key( + tenant_id, + company_id, + broker_id, + timestamp, + file.filename or "cove.xml", + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + field_name = "xml_files_path" + subdir = "cove" + elif fk == "doda_certificate": + if file_ext != ".cer": + raise HTTPException(status_code=400, detail="doda_certificate must be .cer") + key = customs_broker_vu_doda_certificate_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + field_name = "doda_certificate_path" + subdir = "doda/certificates" + elif fk == "doda_key": + if file_ext != ".key": + raise HTTPException(status_code=400, detail="doda_key must be .key") + key = customs_broker_vu_doda_private_key_key( + tenant_id, company_id, broker_id, timestamp, file_ext + ) + field_name = "doda_key_path" + subdir = "doda/keys" + else: + try: + key = customs_broker_vu_doda_cove_key( + tenant_id, + company_id, + broker_id, + timestamp, + file.filename or "doda.xml", + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + field_name = "doda_xml_files_path" + subdir = "doda/cove" + + fname = key.rsplit("/", 1)[-1] + dest_dir = os.path.join(base, subdir) + os.makedirs(dest_dir, exist_ok=True) + path = os.path.join(dest_dir, fname) + old = getattr(vu, field_name) + _remove_stored_vu_path(old) + with open(path, "wb") as f: + f.write(content) + logger.info( + "Customs broker VU upload kind=%s path=%s bytes=%s", + fk, + path, + len(content), + ) + stored = path + + setattr(vu, field_name, stored) + db.add(vu) + db.commit() + db.refresh(vu) + except HTTPException: + db.rollback() + raise + except ValueError as e: + db.rollback() + raise HTTPException(status_code=400, detail=str(e)) from e + except Exception as e: + db.rollback() + raise HTTPException( + status_code=500, detail=f"Error saving file: {str(e)}" + ) from e + + return { + "message": "File uploaded successfully", + "file_kind": fk, + "field": field_name, + "path": stored, + "broker_key": broker_key, + "company_id": company_id, + } \ No newline at end of file diff --git a/backend/api/v1/modules/a76/customs_brokers/services.py b/backend/api/v1/modules/a76/customs_brokers/services.py index 6e62e896..32ae8469 100644 --- a/backend/api/v1/modules/a76/customs_brokers/services.py +++ b/backend/api/v1/modules/a76/customs_brokers/services.py @@ -160,6 +160,26 @@ class CustomsBrokerVUService: db.commit() return vu + @staticmethod + def ensure_vu_for_broker(db: Session, broker: models.CustomsBroker) -> models.CustomsBrokerVU: + """Crea fila VU vacía si no existe (p. ej. antes de subir CER/KEY/COVE al bucket).""" + vu = ( + db.query(models.CustomsBrokerVU) + .filter(models.CustomsBrokerVU.customs_broker_id == broker.id) + .first() + ) + if vu: + return vu + vu = models.CustomsBrokerVU( + customs_broker_id=broker.id, + tenant_id=broker.tenant_id, + company_id=broker.company_id, + ) + db.add(vu) + db.commit() + db.refresh(vu) + return vu + class CustomsBrokerPersonnelService: @staticmethod diff --git a/backend/api/v1/modules/a76/general_catalogs/company/dto.py b/backend/api/v1/modules/a76/general_catalogs/company/dto.py index a83faf81..d96f582a 100644 --- a/backend/api/v1/modules/a76/general_catalogs/company/dto.py +++ b/backend/api/v1/modules/a76/general_catalogs/company/dto.py @@ -57,7 +57,7 @@ class CompanyCreateDTO(BaseModel): ) # Configuration - logo: Optional[str] = Field(None, max_length=255, description="Company logo") + logo: Optional[str] = Field(None, max_length=512, description="Company logo (path local o clave S3)") has_express_line: Optional[bool] = Field(None, description="Has express line") order_format_type: Optional[str] = Field( None, max_length=19, description="Order format type" diff --git a/backend/api/v1/modules/a76/general_catalogs/company/models.py b/backend/api/v1/modules/a76/general_catalogs/company/models.py index 4914f791..739078ea 100644 --- a/backend/api/v1/modules/a76/general_catalogs/company/models.py +++ b/backend/api/v1/modules/a76/general_catalogs/company/models.py @@ -60,7 +60,7 @@ class Company(Base, TimestampMixin): position: Mapped[Optional[str]] = mapped_column(String(30)) # Configuración básica - logo: Mapped[Optional[str]] = mapped_column(String(255)) + logo: Mapped[Optional[str]] = mapped_column(String(512)) has_express_line: Mapped[Optional[bool]] = mapped_column(Boolean, default=False, server_default="false") order_format_type: Mapped[Optional[str]] = mapped_column(String(19)) is_service_company: Mapped[Optional[bool]] = mapped_column(Boolean, default=False, server_default="false") diff --git a/backend/api/v1/modules/a76/general_catalogs/company/routes.py b/backend/api/v1/modules/a76/general_catalogs/company/routes.py index 5dff4d54..d6871d64 100644 --- a/backend/api/v1/modules/a76/general_catalogs/company/routes.py +++ b/backend/api/v1/modules/a76/general_catalogs/company/routes.py @@ -2,20 +2,23 @@ Rutas para gestión de empresa """ +import logging +import mimetypes import os import shutil from typing import List, Optional -import os -import shutil from pathlib import Path from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, status, File, UploadFile -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, Response from sqlalchemy.orm import Session +from core.config import settings from core.database import get_core_db -from core.security import get_current_user, validate_access_to_resource +from core.s3_keys import company_certificate_key, company_logo_key +from core.storage_s3 import delete_object_if_exists, get_object_bytes, put_object_bytes +from core.security import get_current_user, get_tenant_from_token, validate_access_to_resource from .....common.tenant_crud_routes import TenantCRUDRoutes from .dto import CompanyCreateDTO, CompanyResponseDTO, CompanyUpdateDTO from .models import Company @@ -26,6 +29,44 @@ UPLOAD_DIR = "uploads/companies" ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp"} MAX_FILE_SIZE = 5 * 1024 * 1024 # 5MB +logger = logging.getLogger(__name__) + + +def _resolve_tenant_id_int(current_user: dict) -> int: + """Misma lógica que validate_access_to_resource: entero estable para BD y claves S3.""" + tid = get_tenant_from_token(current_user) + if tid is not None: + return int(tid) + raw = current_user.get("tenant_id") + if isinstance(raw, list) and raw: + raw = raw[0] + if raw is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Tenant ID not found in user data", + ) + try: + return int(raw) + except (TypeError, ValueError): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid tenant ID in token", + ) + + +def _is_s3_object_key(ref: Optional[str]) -> bool: + return bool(ref and ref.startswith("tenants/")) + + +def _remove_stored_file(ref: str) -> None: + if _is_s3_object_key(ref): + delete_object_if_exists(ref) + elif ref and os.path.isfile(ref): + try: + os.remove(ref) + except OSError: + pass + # Main router that includes base CRUD router = APIRouter(prefix="/company") @@ -219,14 +260,20 @@ async def get_company_logo_image( if not company or not company.logo: raise HTTPException(status_code=404, detail="Logo not found") + if _is_s3_object_key(company.logo): + try: + data = get_object_bytes(company.logo) + except Exception: + raise HTTPException(status_code=404, detail="Logo file not found on server") + media = mimetypes.guess_type(company.logo)[0] or "image/jpeg" + return Response(content=data, media_type=media) + file_path = Path(company.logo) if not file_path.exists(): - # Fallback for old paths or moved files - # Check if it exists in the 'standard' location even if DB thinks otherwise standard_path = Path(f"app_data/logos/{company_id}") / file_path.name if standard_path.exists(): return FileResponse(standard_path) - + raise HTTPException(status_code=404, detail="Logo file not found on server") return FileResponse(file_path) @@ -272,12 +319,7 @@ async def upload_company_logo( current_user: dict = Depends(get_current_user), ): """Upload a logo for a company""" - tenant_id = current_user.get("tenant_id") - if not tenant_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Tenant ID not found in user data", - ) + tenant_id = _resolve_tenant_id_int(current_user) # Validar que la empresa existe company = CompanyService.get_by_id(db, company_id, tenant_id, 0) @@ -303,35 +345,34 @@ async def upload_company_logo( detail=f"File too large. Maximum size: {MAX_FILE_SIZE / 1024 / 1024}MB", ) - # Crear directorio si no existe - os.makedirs(UPLOAD_DIR, exist_ok=True) - - # Eliminar logo anterior si existe if company.logo: - old_logo_path = company.logo - if os.path.exists(old_logo_path): - try: - os.remove(old_logo_path) - except Exception: - pass # No es crítico si falla + _remove_stored_file(company.logo) - # Generar nombre único timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"company_{company_id}_{timestamp}{file_ext}" - file_path = os.path.join(UPLOAD_DIR, filename) + filename = f"logo_{company_id}_{timestamp}{file_ext}" - # Guardar archivo try: - await file.seek(0) - with open(file_path, "wb") as buffer: - shutil.copyfileobj(file.file, buffer) + if settings.use_s3_object_storage: + ct = mimetypes.guess_type(filename)[0] or "image/jpeg" + key = company_logo_key(tenant_id, company_id, filename) + put_object_bytes(key, content, content_type=ct) + logger.info( + "Company logo stored in S3 key=%s bytes=%s", + key, + len(content), + ) + file_path = key + else: + os.makedirs(UPLOAD_DIR, exist_ok=True) + file_path = os.path.join(UPLOAD_DIR, filename) + with open(file_path, "wb") as f: + f.write(content) except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error saving file: {str(e)}", ) - # Actualizar la empresa con la ruta del logo update_data = CompanyUpdateDTO(logo=file_path) service = CompanyService(db) updated_company = service.update(db, company_id, tenant_id, 0, update_data) @@ -359,12 +400,7 @@ async def upload_company_certificate( Upload a certificate for a company certificate_type: fiel_cer, fiel_key, cfdi_cert_cer, cfdi_cert_key, cancel_cer, cancel_key """ - tenant_id = current_user.get("tenant_id") - if not tenant_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Tenant ID not found in user data", - ) + tenant_id = _resolve_tenant_id_int(current_user) # Validar que la empresa existe service = CompanyService(db) @@ -410,27 +446,38 @@ async def upload_company_certificate( detail=f"File too large. Maximum size: {MAX_FILE_SIZE / 1024 / 1024}MB", ) - # Crear directorio si no existe - certs_dir = os.path.join(UPLOAD_DIR, str(company_id), "certificates") - os.makedirs(certs_dir, exist_ok=True) - - # Generar nombre único timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = f"{certificate_type}_{timestamp}{file_ext}" - file_path = os.path.join(certs_dir, filename) - # Guardar archivo try: - await file.seek(0) - with open(file_path, "wb") as buffer: - shutil.copyfileobj(file.file, buffer) + if settings.use_s3_object_storage: + key = company_certificate_key( + tenant_id, company_id, certificate_type, timestamp, file_ext + ) + ct = ( + "application/x-x509-ca-cert" + if file_ext == ".cer" + else "application/pkcs8" + ) + put_object_bytes(key, content, content_type=ct) + logger.info( + "Company certificate stored in S3 key=%s bytes=%s", + key, + len(content), + ) + file_path = key + else: + certs_dir = os.path.join(UPLOAD_DIR, str(company_id), "certificates") + os.makedirs(certs_dir, exist_ok=True) + filename = f"{certificate_type}_{timestamp}{file_ext}" + file_path = os.path.join(certs_dir, filename) + with open(file_path, "wb") as f: + f.write(content) except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error saving file: {str(e)}", ) - # Actualizar la base de datos service.upload_certificate(company_id, certificate_type, file_path, tenant_id) return { diff --git a/backend/api/v1/modules/a76/general_catalogs/company/service.py b/backend/api/v1/modules/a76/general_catalogs/company/service.py index 37158c12..21502ebc 100644 --- a/backend/api/v1/modules/a76/general_catalogs/company/service.py +++ b/backend/api/v1/modules/a76/general_catalogs/company/service.py @@ -678,6 +678,11 @@ class CompanyService: try: if target_cert: + old_path = getattr(target_cert, field_to_update, None) + if old_path and str(old_path).startswith("tenants/"): + from core.storage_s3 import delete_object_if_exists + + delete_object_if_exists(str(old_path)) # Si existe, actualizamos setattr(target_cert, field_to_update, file_path) else: diff --git a/backend/api/v1/modules/a76/layouts_csv/boms/routes.py b/backend/api/v1/modules/a76/layouts_csv/boms/routes.py index f7d46af7..d86f62cf 100644 --- a/backend/api/v1/modules/a76/layouts_csv/boms/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/boms/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para BOMs. Flujo: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -22,10 +20,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - BOM_IMPORT_FILE_PREFIX, + JOB_TYPE, BOM_IMPORT_META_PREFIX, BOM_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit from ..common.responses import normalize_commit_status_payload @@ -67,31 +66,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{BOM_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=BOM_IMPORT_REDIS_TTL, - ) - r.set( - f"{BOM_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=BOM_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=BOM_IMPORT_REDIS_TTL, + log_label="BOMs import", ) + except common_storage.ImportStoreError as e: + logger.error(f"BOMs import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"BOMs import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"bom_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"bom_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"BOMs import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py index d8612d2e..89147746 100644 --- a/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/cambio_regimen_regularizacion/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Cambio de régimen y Regularización (encabezado y partidas). Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Literal, Optional, Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -65,7 +63,6 @@ async def upload_import_file( job_id = str(uuid4()) contents = await file.read() - file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id) meta_data = { "tenant_id": tenant_id, "company_id": company_id, @@ -76,25 +73,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set(file_key, base64.b64encode(contents), ex=CRREG_IMPORT_REDIS_TTL) - r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=CRREG_IMPORT_REDIS_TTL) + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=CRREG_IMPORT_REDIS_TTL, + log_label="Cambio régimen/Regularización import", + ) + except common_storage.ImportStoreError as e: + logger.error("Cambio régimen/Regularización import: store error: %s", e) + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error("Cambio régimen/Regularización import: Redis store error: %s", e) raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id) - with open(csv_path, "wb") as f: - f.write(contents) - meta_path = csv_path.replace(".csv", ".meta.json") - with open(meta_path, "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning("Cambio régimen/Regularización import: local file save failed: %s", e) - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/classes/routes.py b/backend/api/v1/modules/a76/layouts_csv/classes/routes.py index 0d711a72..2fa3347b 100644 --- a/backend/api/v1/modules/a76/layouts_csv/classes/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/classes/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Clases de Materiales. Flujo: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,10 +22,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - CLS_IMPORT_FILE_PREFIX, + JOB_TYPE, CLS_IMPORT_META_PREFIX, CLS_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream router = APIRouter() @@ -71,31 +70,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{CLS_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=CLS_IMPORT_REDIS_TTL, - ) - r.set( - f"{CLS_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=CLS_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=CLS_IMPORT_REDIS_TTL, + log_label="Classes import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Classes import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"Classes import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"cls_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"cls_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Classes import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py index a9f14507..ef6453ec 100644 --- a/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/clients_and_providers/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Clientes y Proveedores. Mismo flujo que a76.imports: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,10 +22,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - CP_IMPORT_FILE_PREFIX, + JOB_TYPE, CP_IMPORT_META_PREFIX, CP_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream router = APIRouter() @@ -70,31 +69,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{CP_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=CP_IMPORT_REDIS_TTL, - ) - r.set( - f"{CP_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=CP_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=CP_IMPORT_REDIS_TTL, + log_label="CP import", ) + except common_storage.ImportStoreError as e: + logger.error(f"CP import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"CP import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"cp_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"cp_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"CP import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/common/storage.py b/backend/api/v1/modules/a76/layouts_csv/common/storage.py index f2875d5c..f737441a 100644 --- a/backend/api/v1/modules/a76/layouts_csv/common/storage.py +++ b/backend/api/v1/modules/a76/layouts_csv/common/storage.py @@ -15,6 +15,12 @@ logger = logging.getLogger(__name__) IMPORT_REDIS_TTL = 3600 +class ImportStoreError(Exception): + """Fallo al guardar CSV en Redis/MinIO.""" + + pass + + def _get_redis(): import redis url = os.getenv("VALKEY_URL", os.getenv("REDIS_URL", "redis://valkey:6379/0")) @@ -60,23 +66,103 @@ def error_path_for_job(job_type: str, job_id: str) -> str: return os.path.join(error_dir(), f"{job_type}_{job_id}.jsonl") +def store_import_file( + job_type: str, + job_id: str, + raw_bytes: bytes, + meta_dict: dict, + tenant_id, + company_id: int, + ttl: int = IMPORT_REDIS_TTL, + log_label: str = "", +) -> None: + """ + Guarda CSV y meta en Redis. Si CSV_IMPORT_STORAGE=minio, sube el CSV a S3 y en Redis + guarda JSON {"v":2,"s3_key":...}; si no, base64 en Redis (comportamiento anterior). + En modo redis opcionalmente escribe copia local bajo layouts/imports/temp (debug). + """ + from core.config import settings + + file_key, meta_key, _ = storage_keys(job_type, job_id) + r = _get_redis() + meta_bytes = json.dumps(meta_dict).encode("utf-8") + + if settings.CSV_IMPORT_STORAGE == "minio": + from core.storage_s3 import put_csv_object, s3_key_for_csv_import + + key = s3_key_for_csv_import(tenant_id, company_id, job_type, job_id) + try: + put_csv_object(key, raw_bytes) + except Exception as e: + logger.exception("%s MinIO put failed: %s", log_label or job_type, e) + raise ImportStoreError(str(e)) from e + payload = json.dumps({"v": 2, "s3_key": key}).encode("utf-8") + r.set(file_key, payload, ex=ttl) + else: + r.set(file_key, base64.b64encode(raw_bytes), ex=ttl) + + r.set(meta_key, meta_bytes, ex=ttl) + + if settings.CSV_IMPORT_STORAGE != "minio": + try: + path = file_path_for_job(job_type, job_id) + os.makedirs(upload_dir(), exist_ok=True) + with open(path, "wb") as f: + f.write(raw_bytes) + meta_path = path.replace(".csv", ".meta.json") + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta_dict, f) + except Exception as e: + logger.warning("%s local file save failed: %s", log_label or job_type, e) + + def ensure_file_from_redis(job_type: str, job_id: str, log_prefix: str = "") -> Optional[str]: """ - Descarga contenido del CSV desde Redis y lo escribe en disco. + Obtiene el CSV desde Redis (referencia MinIO o base64 legacy) y lo escribe en disco. Devuelve la ruta del archivo o None si no hay datos o falla. """ + from core.config import settings + file_key, _, _ = storage_keys(job_type, job_id) r = _get_redis() data = r.get(file_key) if not data: return None + + path = file_path_for_job(job_type, job_id) + os.makedirs(upload_dir(), exist_ok=True) + + if data.startswith(b"{"): + try: + obj = json.loads(data.decode("utf-8")) + if isinstance(obj, dict) and obj.get("v") == 2 and obj.get("s3_key"): + if settings.CSV_IMPORT_STORAGE != "minio": + logger.warning( + "%s Redis has MinIO ref but CSV_IMPORT_STORAGE=%s", + log_prefix or job_type, + settings.CSV_IMPORT_STORAGE, + ) + from core.storage_s3 import get_object_bytes + + try: + raw = get_object_bytes(obj["s3_key"]) + except Exception as e: + logger.warning("%s MinIO get failed: %s", log_prefix or job_type, e) + return None + with open(path, "wb") as f: + f.write(raw) + return path + except json.JSONDecodeError: + pass + except Exception as e: + logger.warning("%s failed to parse MinIO ref from Redis: %s", log_prefix or job_type, e) + return None + try: raw = base64.b64decode(data) except Exception as e: logger.warning("%s failed to decode file from Redis: %s", log_prefix or job_type, e) return None - path = file_path_for_job(job_type, job_id) - os.makedirs(upload_dir(), exist_ok=True) with open(path, "wb") as f: f.write(raw) return path @@ -152,7 +238,31 @@ def cleanup_import_job( error_path: Optional[str] = None, meta_path: Optional[str] = None, ) -> None: - """Elimina archivos locales y claves Redis del job.""" + """Elimina archivos locales, objeto MinIO si aplica, y claves Redis del job.""" + from core.config import settings + + if settings.CSV_IMPORT_STORAGE == "minio": + from core.storage_s3 import delete_object_if_exists + from core.s3_keys import legacy_csv_import_key + + file_key, _, _ = storage_keys(job_type, job_id) + try: + r = _get_redis() + data = r.get(file_key) + deleted = False + if data and data.startswith(b"{"): + try: + obj = json.loads(data.decode("utf-8")) + if isinstance(obj, dict) and obj.get("v") == 2 and obj.get("s3_key"): + delete_object_if_exists(obj["s3_key"]) + deleted = True + except Exception as e: + logger.warning("Cleanup: parse Redis file ref: %s", e) + if not deleted: + delete_object_if_exists(legacy_csv_import_key(job_type, job_id)) + except Exception as e: + logger.warning("Cleanup: MinIO delete failed: %s", e) + if file_path and os.path.exists(file_path): try: os.remove(file_path) diff --git a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py index dc17a477..36534ef0 100644 --- a/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/customs_brokers/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Agentes Aduanales. Mismo flujo que a76.imports: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,10 +22,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - CB_IMPORT_FILE_PREFIX, + JOB_TYPE, CB_IMPORT_META_PREFIX, CB_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream router = APIRouter() @@ -70,31 +69,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{CB_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=CB_IMPORT_REDIS_TTL, - ) - r.set( - f"{CB_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=CB_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=CB_IMPORT_REDIS_TTL, + log_label="CB import", ) + except common_storage.ImportStoreError as e: + logger.error(f"CB import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"CB import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"cb_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"cb_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"CB import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py b/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py index bb118f17..9d28625a 100644 --- a/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/drivers/routes.py @@ -2,7 +2,6 @@ Rutas de importacion CSV para Conductores. Flujo: upload -> scan -> status (polling) -> commit. """ -import base64 import json import logging import os @@ -15,18 +14,17 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from .schemas import ImportJobResponse from .tasks import ( run_scan_sync, run_commit_sync, - DRV_IMPORT_FILE_PREFIX, - DRV_IMPORT_META_PREFIX, + JOB_TYPE, DRV_IMPORT_STATUS_PREFIX, DRV_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream from ..common.responses import normalize_commit_status_payload @@ -67,31 +65,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{DRV_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=DRV_IMPORT_REDIS_TTL, - ) - r.set( - f"{DRV_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=DRV_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=DRV_IMPORT_REDIS_TTL, + log_label="Drivers import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Drivers import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"Drivers import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"drv_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"drv_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Drivers import: local file save failed: {e}") - def run_scan_background(): try: run_scan_sync(job_id) diff --git a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py index 4887092f..0ac58d66 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/exchange_rate/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Tipos de Cambio. Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any, Optional from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,10 +22,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - ER_IMPORT_FILE_PREFIX, + JOB_TYPE, ER_IMPORT_META_PREFIX, ER_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream router = APIRouter() @@ -81,31 +80,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{ER_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=ER_IMPORT_REDIS_TTL, - ) - r.set( - f"{ER_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=ER_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=ER_IMPORT_REDIS_TTL, + log_label="ER import", ) + except common_storage.ImportStoreError as e: + logger.error(f"ER import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"ER import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"er_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"er_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"ER import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py b/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py index 49eba2b8..ec4f7fc9 100644 --- a/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/exportacion/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Exportación (encabezado y partidas). Flujo: upload → scan → status (polling) → commit. Sin validaciones ni inserción aún. """ -import base64 import json import logging import os @@ -17,7 +16,6 @@ from typing import Literal, Optional, Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -61,7 +59,6 @@ async def upload_import_file( job_id = str(uuid4()) contents = await file.read() - file_key, meta_key, _ = common_storage.storage_keys(JOB_TYPE, job_id) default_template = ( "exp_def_header" if model_target == "invoice_header" else "exp_def_series" if model_target == "invoice_series" @@ -83,25 +80,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set(file_key, base64.b64encode(contents), ex=EXP_IMPORT_REDIS_TTL) - r.set(meta_key, json.dumps(meta_data).encode("utf-8"), ex=EXP_IMPORT_REDIS_TTL) + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=EXP_IMPORT_REDIS_TTL, + log_label="Exportación import", + ) + except common_storage.ImportStoreError as e: + logger.error("Exportación import: store error: %s", e) + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error("Exportación import: Redis store error: %s", e) raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - csv_path = common_storage.file_path_for_job(JOB_TYPE, job_id) - with open(csv_path, "wb") as f: - f.write(contents) - meta_path = csv_path.replace(".csv", ".meta.json") - with open(meta_path, "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning("Exportación import: local file save failed: %s", e) - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py b/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py index 0ae9556f..8cb0df25 100644 --- a/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/facturas/routes.py @@ -1,6 +1,5 @@ from datetime import datetime from uuid import uuid4 -import base64 import os import json import logging @@ -14,14 +13,13 @@ from typing import Optional, Literal, Dict, Any from core.celery_app import celery_app from core.config import settings from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch from .tasks import ( scan_file, insert_valid_rows, - IMPORT_FILE_KEY_PREFIX, + JOB_TYPE, IMPORT_META_KEY_PREFIX, IMPORT_REDIS_TTL, ) @@ -83,37 +81,25 @@ async def upload_import_file( "template_id": template_id, } - # Store file and meta in Redis so the Celery worker can read them (no shared filesystem needed) try: - redis_client = _get_redis() - redis_client.set( - f"{IMPORT_FILE_KEY_PREFIX}{job_id}", - base64.b64encode(contents), - ex=IMPORT_REDIS_TTL, - ) - redis_client.set( - f"{IMPORT_META_KEY_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=IMPORT_REDIS_TTL, + log_label="Facturas import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Import store error: {e}") + raise HTTPException(status_code=500, detail="Failed to queue file for processing.") except Exception as e: logger.error(f"Redis store error: {e}") raise HTTPException(status_code=500, detail="Failed to queue file for processing.") - # Optional: also write to local disk (e.g. for same-machine worker or debugging) - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - file_path = os.path.join(upload_dir, f"{job_id}.csv") - meta_path = os.path.join(upload_dir, f"{job_id}.meta.json") - with open(file_path, "wb") as f: - f.write(contents) - with open(meta_path, "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Local file save failed (worker will use Redis): {e}") - - # Trigger Celery Task (Async). Worker loads file from Redis. + # Trigger Celery Task (Async). Worker loads file from Redis / MinIO. track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/parts/routes.py b/backend/api/v1/modules/a76/layouts_csv/parts/routes.py index 6e1fb792..2c479dfb 100644 --- a/backend/api/v1/modules/a76/layouts_csv/parts/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/parts/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Números de Parte. Flujo: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -22,10 +20,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - PART_IMPORT_FILE_PREFIX, + JOB_TYPE, PART_IMPORT_META_PREFIX, PART_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream from ..common.track_commit_dispatch import dispatch_tracked_layouts_csv_commit from ..common.responses import normalize_commit_status_payload @@ -71,31 +70,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{PART_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=PART_IMPORT_REDIS_TTL, - ) - r.set( - f"{PART_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=PART_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=PART_IMPORT_REDIS_TTL, + log_label="Parts import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Parts import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"Parts import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"part_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"part_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Parts import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py b/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py index 69e26d65..0031f9de 100644 --- a/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/pedmientos/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Pedimentos. Mismo flujo que customs_brokers/imports: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any, Optional from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -64,7 +62,6 @@ async def upload_import_file( job_id = str(uuid4()) contents = await file.read() - file_key, meta_key, _ = common_storage.storage_keys(PED_JOB_TYPE, job_id) meta_data: Dict[str, Any] = { "tenant_id": tenant_id, "company_id": company_id, @@ -76,33 +73,23 @@ async def upload_import_file( meta_data["dateFormat"] = dateFormat try: - r = _get_redis() - r.set( - file_key, - base64.b64encode(contents), - ex=PED_IMPORT_REDIS_TTL, - ) - r.set( - meta_key, - json.dumps(meta_data).encode("utf-8"), - ex=PED_IMPORT_REDIS_TTL, + common_storage.store_import_file( + PED_JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=PED_IMPORT_REDIS_TTL, + log_label="Pedimentos import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Pedimentos import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"Pedimentos import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - csv_path = common_storage.file_path_for_job(PED_JOB_TYPE, job_id) - with open(csv_path, "wb") as f: - f.write(contents) - meta_path = csv_path.replace(".csv", ".meta.json") - with open(meta_path, "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Pedimentos import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py b/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py index 73547beb..fc46a9af 100644 --- a/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/trailers/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Trailers y Cajas. Flujo: upload -> scan -> status (polling) -> commit. """ -import base64 import json import logging import os @@ -15,7 +14,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,11 +22,11 @@ from .tasks import ( scan_file, run_scan_sync, run_commit_sync, - TRL_IMPORT_FILE_PREFIX, - TRL_IMPORT_META_PREFIX, + JOB_TYPE, TRL_IMPORT_STATUS_PREFIX, TRL_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream from ..common.responses import normalize_commit_status_payload @@ -71,31 +69,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{TRL_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=TRL_IMPORT_REDIS_TTL, - ) - r.set( - f"{TRL_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=TRL_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=TRL_IMPORT_REDIS_TTL, + log_label="Trailers import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Trailers import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"Trailers import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"trl_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"trl_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Trailers import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py b/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py index 518b3d3f..4191393b 100644 --- a/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/transportistas/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Transportistas. Flujo: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -15,7 +14,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,11 +22,11 @@ from .tasks import ( scan_file, run_scan_sync, run_commit_sync, - TRP_IMPORT_FILE_PREFIX, - TRP_IMPORT_META_PREFIX, + JOB_TYPE, TRP_IMPORT_STATUS_PREFIX, TRP_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream from ..common.responses import normalize_commit_status_payload @@ -71,31 +69,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{TRP_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=TRP_IMPORT_REDIS_TTL, - ) - r.set( - f"{TRP_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=TRP_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=TRP_IMPORT_REDIS_TTL, + log_label="Transportistas import", ) + except common_storage.ImportStoreError as e: + logger.error("Transportistas import: store error: %s", e) + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error("Transportistas import: Redis store error: %s", e) raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"trp_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"trp_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning("Transportistas import: local file save failed: %s", e) - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py index b35846f5..4e641f17 100644 --- a/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/us_tariff_fractions/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Fracción Americana (US Tariff Fractions). Mismo flujo que exchange_rate/imports: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -14,7 +13,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,10 +22,11 @@ from .schemas import ImportJobResponse from .tasks import ( scan_file, insert_valid_rows, - FA_IMPORT_FILE_PREFIX, + JOB_TYPE, FA_IMPORT_META_PREFIX, FA_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream router = APIRouter() @@ -73,31 +72,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{FA_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=FA_IMPORT_REDIS_TTL, - ) - r.set( - f"{FA_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=FA_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=FA_IMPORT_REDIS_TTL, + log_label="FA import", ) + except common_storage.ImportStoreError as e: + logger.error(f"FA import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"FA import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"fa_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"fa_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"FA import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py b/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py index 95539a68..aff5f682 100644 --- a/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py +++ b/backend/api/v1/modules/a76/layouts_csv/vehicles/routes.py @@ -2,7 +2,6 @@ Rutas de importación CSV para Vehículos (Transportes). Flujo: upload → scan → status (polling) → commit. """ -import base64 import json import logging import os @@ -15,7 +14,6 @@ from typing import Dict, Any from core.celery_app import celery_app from core.database import get_core_db -from core.paths import layout_path from core.security import get_current_user, validate_access_to_resource from api.v1.modules.core.tasks_tracking import track_and_dispatch @@ -24,11 +22,11 @@ from .tasks import ( scan_file, run_scan_sync, run_commit_sync, - VEHL_IMPORT_FILE_PREFIX, - VEHL_IMPORT_META_PREFIX, + JOB_TYPE, VEHL_IMPORT_STATUS_PREFIX, VEHL_IMPORT_REDIS_TTL, ) +from ..common import storage as common_storage from ..common.error_csv import download_scan_errors_csv_stream from ..common.responses import normalize_commit_status_payload @@ -71,31 +69,23 @@ async def upload_import_file( } try: - r = _get_redis() - r.set( - f"{VEHL_IMPORT_FILE_PREFIX}{job_id}", - base64.b64encode(contents), - ex=VEHL_IMPORT_REDIS_TTL, - ) - r.set( - f"{VEHL_IMPORT_META_PREFIX}{job_id}", - json.dumps(meta_data).encode("utf-8"), - ex=VEHL_IMPORT_REDIS_TTL, + common_storage.store_import_file( + JOB_TYPE, + job_id, + contents, + meta_data, + tenant_id=int(tenant_id), + company_id=company_id, + ttl=VEHL_IMPORT_REDIS_TTL, + log_label="Vehicles import", ) + except common_storage.ImportStoreError as e: + logger.error(f"Vehicles import: store error: {e}") + raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") except Exception as e: logger.error(f"Vehicles import: Redis store error: {e}") raise HTTPException(status_code=500, detail="No se pudo encolar el archivo.") - try: - upload_dir = layout_path("imports", "temp") - os.makedirs(upload_dir, exist_ok=True) - with open(os.path.join(upload_dir, f"veh_{job_id}.csv"), "wb") as f: - f.write(contents) - with open(os.path.join(upload_dir, f"veh_{job_id}.meta.json"), "w") as f: - json.dump(meta_data, f) - except Exception as e: - logger.warning(f"Vehicles import: local file save failed: {e}") - track_and_dispatch( db=db, task=scan_file, diff --git a/backend/api/v1/modules/core/help_center/routes.py b/backend/api/v1/modules/core/help_center/routes.py index 29f576b5..c76d697a 100644 --- a/backend/api/v1/modules/core/help_center/routes.py +++ b/backend/api/v1/modules/core/help_center/routes.py @@ -1,13 +1,24 @@ -import shutil +import mimetypes import os +import shutil import uuid from datetime import datetime from typing import List, Optional, Dict, Any from uuid import UUID + from fastapi import APIRouter, Depends, HTTPException, Header, status, UploadFile, File +from fastapi.responses import Response from sqlalchemy.orm import Session -from core.database import get_core_db + from core.config import settings +from core.database import get_core_db +from core.s3_keys import ( + help_asset_key, + help_public_api_path, + help_s3_key_to_public_relative_path, + system_help_object_key, +) +from core.storage_s3 import get_object_bytes, put_object_bytes from core.security import get_current_user, has_role from .schemas import HelpArticleInDB, HelpArticleCreate, HelpArticleUpdate, HelpSyncRequest, HelpSyncResponse from .services import HelpCenterService @@ -80,60 +91,95 @@ def sync_help_article(sync_data: HelpSyncRequest, db: Session = Depends(get_core return result +@router.get("/files/{file_path:path}") +def serve_help_file(file_path: str): + """Sirve un objeto bajo system/help/ (público vía middleware).""" + if ".." in file_path or file_path.startswith("/"): + raise HTTPException(status_code=404, detail="Not found") + try: + key = system_help_object_key(file_path) + except ValueError: + raise HTTPException(status_code=404, detail="Not found") + if not settings.use_s3_object_storage: + legacy = os.path.join("uploads", "help", file_path) + if not os.path.isfile(legacy): + raise HTTPException(status_code=404, detail="Not found") + with open(legacy, "rb") as f: + data = f.read() + media = mimetypes.guess_type(file_path)[0] or "application/octet-stream" + return Response(content=data, media_type=media) + try: + data = get_object_bytes(key) + except Exception: + raise HTTPException(status_code=404, detail="Not found") + media = mimetypes.guess_type(file_path)[0] or "application/octet-stream" + return Response(content=data, media_type=media) + + @router.post("/upload-image/") -def upload_help_image( +async def upload_help_image( file: UploadFile = File(...), current_user: Dict[str, Any] = Depends(has_role("admin")) ): """Sube una imagen para usar en los artículos.""" try: - file_ext = os.path.splitext(file.filename)[1] + file_ext = os.path.splitext(file.filename or "")[1] or ".png" new_filename = f"{uuid.uuid4()}{file_ext}" - file_location = f"uploads/help/{new_filename}" - - # Ensure directory exists + body = await file.read() + if settings.use_s3_object_storage: + key = help_asset_key("", new_filename) + ct = mimetypes.guess_type(new_filename)[0] or "image/png" + put_object_bytes(key, body, content_type=ct) + rel = help_s3_key_to_public_relative_path(key) + return {"url": help_public_api_path(rel)} os.makedirs("uploads/help", exist_ok=True) - - with open(file_location, "wb+") as buffer: - shutil.copyfileobj(file.file, buffer) - + file_location = f"uploads/help/{new_filename}" + with open(file_location, "wb") as f: + f.write(body) return {"url": f"/api/uploads/help/{new_filename}"} except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @router.post("/upload-asset/") -def upload_help_asset( +async def upload_help_asset( file: UploadFile = File(...), current_user: Dict[str, Any] = Depends(has_role("admin")) ): """Sube cualquier tipo de archivo (PDF, Video, etc.) para la biblioteca.""" try: - file_ext = os.path.splitext(file.filename)[1].lower() + file_ext = os.path.splitext(file.filename or "")[1].lower() new_filename = f"{uuid.uuid4()}{file_ext}" - - # Guardar en una carpeta segun el tipo o general - folder = "uploads/help/assets" - if file_ext in ['.pdf']: - folder = "uploads/help/pdfs" - elif file_ext in ['.mp4', '.mov', '.avi']: - folder = "uploads/help/videos" - - file_location = f"{folder}/{new_filename}" - - # Ensure directory exists + subfolder = "assets" + if file_ext == ".pdf": + subfolder = "pdfs" + elif file_ext in [".mp4", ".mov", ".avi"]: + subfolder = "videos" + + if settings.use_s3_object_storage: + body = await file.read() + key = help_asset_key(subfolder, new_filename) + ct = file.content_type or mimetypes.guess_type(new_filename)[0] or "application/octet-stream" + put_object_bytes(key, body, content_type=ct) + rel = help_s3_key_to_public_relative_path(key) + return { + "url": help_public_api_path(rel), + "filename": file.filename, + "size": len(body), + "mime_type": file.content_type, + } + + folder = f"uploads/help/{subfolder}" os.makedirs(folder, exist_ok=True) - - with open(file_location, "wb+") as buffer: - shutil.copyfileobj(file.file, buffer) - - # Get file size + file_location = f"{folder}/{new_filename}" + body = await file.read() + with open(file_location, "wb") as f: + f.write(body) file_size = os.path.getsize(file_location) - return { "url": f"/api/{file_location}", "filename": file.filename, "size": file_size, - "mime_type": file.content_type + "mime_type": file.content_type, } except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/api/v1/modules/core/help_center/utils.py b/backend/api/v1/modules/core/help_center/utils.py index b5fe4c58..8d9b7854 100644 --- a/backend/api/v1/modules/core/help_center/utils.py +++ b/backend/api/v1/modules/core/help_center/utils.py @@ -1,76 +1,111 @@ +import logging +import mimetypes import os import re -import httpx -import logging -import uuid from pathlib import Path +from typing import Optional + +import httpx + from core.config import settings +from core.s3_keys import SYSTEM_HELP_PREFIX +from core.storage_s3 import object_exists, put_object_bytes logger = logging.getLogger(__name__) + +def _asset_url_to_s3_key(asset_url: str) -> Optional[str]: + """Deriva la clave S3 bajo system/help/ a partir de una URL de artículo.""" + if "/help-center/files/" in asset_url: + rel = asset_url.split("/help-center/files/", 1)[1].lstrip("/") + if ".." in rel: + return None + return f"{SYSTEM_HELP_PREFIX}{rel}" + u = asset_url.replace("/api/uploads/", "uploads/") + if u.startswith("/"): + u = u[1:] + if u.startswith("uploads/help/"): + return f"{SYSTEM_HELP_PREFIX}{u[len('uploads/help/') :]}" + return None + + def download_file_from_hub(relative_path: str) -> bool: """ - Downloads a file from the Hub to the local storage. - relative_path: e.g., 'uploads/help/pdfs/myfile.pdf' or '/api/uploads/help/image.png' + Descarga un asset del Hub y lo guarda en MinIO (system/help/...) o en disco si no hay almacenamiento S3 activo. + relative_path: URL parcial, p. ej. '/api/uploads/help/x.png' o '/api/v1/core/help-center/files/pdfs/x.pdf' """ if not settings.CENTRAL_SERVER_URL or settings.CENTRAL_SERVER_URL == '""': return False - # Clean the path - clean_path = relative_path.replace("/api/uploads/", "uploads/") - if clean_path.startswith("/"): - clean_path = clean_path[1:] - - # Check if it starts with uploads - if not clean_path.startswith("uploads/"): - # If it doesn't start with uploads, it might just be the filename or a subpath - # We assume it's relative to /app/ - pass + key = _asset_url_to_s3_key(relative_path) + if not key: + logger.warning("download_file_from_hub: could not map URL to S3 key: %s", relative_path) + return False - local_path = Path(clean_path) - if local_path.exists(): - logger.info(f"File {clean_path} already exists, skipping download.") + if settings.use_s3_object_storage and object_exists(key): + logger.info("S3 object %s already exists, skipping download.", key) return True - # Ensure directories exist - local_path.parent.mkdir(parents=True, exist_ok=True) - - # Resolve Hub Base URL - # CENTRAL_SERVER_URL is usually http://hub:8000/api/v1/core/help-center/sync/ - # We want http://hub:8000/api/ base_url = settings.CENTRAL_SERVER_URL.split("/v1/")[0] - # The file in backend is served usually under /api/uploads/... - # But clean_path is just "uploads/...". So the Hub route is base_url + "/" + clean_path - hub_file_url = f"{base_url}/{clean_path}" + if "/help-center/files/" in relative_path: + rel = relative_path.split("/help-center/files/", 1)[1].lstrip("/") + hub_file_url = f"{base_url.rstrip('/')}/api/v1/core/help-center/files/{rel}" + else: + clean_path = relative_path.replace("/api/uploads/", "uploads/") + if clean_path.startswith("/"): + clean_path = clean_path[1:] + hub_file_url = f"{base_url.rstrip('/')}/{clean_path}" + + logger.info("Downloading asset from Hub: %s", hub_file_url) - logger.info(f"Downloading asset from Hub: {hub_file_url} -> {local_path}") - try: with httpx.Client() as client: response = client.get(hub_file_url, timeout=30.0) - if response.status_code == 200: - with open(local_path, "wb") as f: - f.write(response.content) - logger.info(f"Successfully downloaded {clean_path}") - return True - else: - logger.warning(f"Failed to download {clean_path}: Status {response.status_code} URL: {hub_file_url}") + if response.status_code != 200: + logger.warning( + "Failed to download %s: Status %s URL: %s", + relative_path, + response.status_code, + hub_file_url, + ) return False + body = response.content except Exception as e: - logger.error(f"Error downloading {clean_path}: {str(e)}") + logger.error("Error downloading %s: %s", relative_path, str(e)) return False + if settings.use_s3_object_storage: + rel = key[len(SYSTEM_HELP_PREFIX) :] + ct = mimetypes.guess_type(rel)[0] or "application/octet-stream" + try: + put_object_bytes(key, body, content_type=ct) + logger.info("Stored hub asset in S3: %s", key) + return True + except Exception as e: + logger.error("S3 put failed for %s: %s", key, e) + return False + + rel = key[len(SYSTEM_HELP_PREFIX) :] + local_path = Path("uploads/help") / rel + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.write_bytes(body) + logger.info("Stored hub asset locally: %s", local_path) + return True + + def sync_assets_from_content(content: str): - """ - Parses markdown content for image URLs and downloads them if they are local references. - Example: ![alt text](/api/uploads/help/uuid.png) - """ + """Parsea markdown y descarga imágenes referenciadas (rutas legacy y nuevas).""" if not content: return - # Regex for markdown images: ![...](/api/uploads/...) - image_pattern = r'!\[.*?\]\((/api/uploads/.*?)\)' - matches = re.findall(image_pattern, content) - - for asset_url in matches: - download_file_from_hub(asset_url) + patterns = [ + r'!\[.*?\]\((/api/uploads/.*?)\)', + r'!\[.*?\]\((/api/v1/core/help-center/files/.*?)\)', + ] + seen = set() + for pattern in patterns: + for asset_url in re.findall(pattern, content): + if asset_url in seen: + continue + seen.add(asset_url) + download_file_from_hub(asset_url) diff --git a/backend/api/v1/modules/core/users/routes.py b/backend/api/v1/modules/core/users/routes.py index 594355f5..6ea6893a 100644 --- a/backend/api/v1/modules/core/users/routes.py +++ b/backend/api/v1/modules/core/users/routes.py @@ -2,14 +2,20 @@ Rutas para gestión de usuarios de Keycloak """ +import logging +import mimetypes from typing import Optional import os import uuid from pathlib import Path +from core.config import settings from core.database import get_core_db +from core.s3_keys import public_user_avatar_api_path, user_avatar_key +from core.storage_s3 import delete_object_if_exists, get_object_bytes, put_object_bytes from core.security import get_current_user, validate_access_to_resource from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile +from fastapi.responses import Response from sqlalchemy.orm import Session from ..user_tenant.models import UserTenant @@ -25,6 +31,10 @@ from .service import UserService router = APIRouter(prefix="/users", tags=["Users"]) +logger = logging.getLogger(__name__) + +_AVATAR_EXT = {".jpg", ".jpeg", ".png", ".gif", ".webp"} + @router.get("/stats", response_model=UserStatsDTO) def get_user_statistics( @@ -67,6 +77,52 @@ def list_users( return result +@router.get("/avatar/{tenant_id}/{keycloak_user_id}") +def get_user_avatar_image( + tenant_id: int, + keycloak_user_id: str, + db: Session = Depends(get_core_db), +): + """ + Sirve la imagen de avatar (público para poder usarla en sin Bearer). + El almacenamiento interno puede ser clave S3 o ruta bajo uploads/. + """ + ut = ( + db.query(UserTenant) + .filter( + UserTenant.tenant_id == tenant_id, + UserTenant.keycloak_user_id == keycloak_user_id, + UserTenant.is_active == True, + ) + .first() + ) + if not ut or not ut.avatar_url: + raise HTTPException(status_code=404, detail="Avatar not found") + + raw = ut.avatar_url + if raw.startswith("tenants/"): + try: + data = get_object_bytes(raw) + except Exception: + raise HTTPException(status_code=404, detail="Avatar not found") + media = mimetypes.guess_type(raw)[0] or "image/jpeg" + return Response(content=data, media_type=media) + + rel = raw.lstrip("/") + path = Path(rel) + if not path.is_file(): + path = Path.cwd() / rel + if not path.is_file(): + alt = Path("/app") / rel + if alt.is_file(): + path = alt + if not path.is_file(): + raise HTTPException(status_code=404, detail="Avatar file not found") + data = path.read_bytes() + media = mimetypes.guess_type(str(path))[0] or "image/jpeg" + return Response(content=data, media_type=media) + + @router.get("/{user_id}", response_model=UserResponseDTO) def get_user( user_id: str, @@ -282,33 +338,74 @@ async def upload_avatar( db: Session = Depends(get_core_db), ): """ - Sube un avatar para el usuario actual - Retorna la URL del avatar subido + Sube un avatar para el usuario actual. + Con MinIO guarda en tenants/{tid}/users/{sub}/avatar.{ext} y persiste la clave en UserTenant. + Retorna URL pública para (GET /users/avatar/...). """ - # Validar tipo de archivo if not file.content_type or not file.content_type.startswith("image/"): raise HTTPException(status_code=400, detail="El archivo debe ser una imagen") - # Validar tamaño (max 2MB) + keycloak_user_id = current_user.get("sub") + if not keycloak_user_id: + raise HTTPException(status_code=400, detail="User ID not found in token") + + user_tenant = ( + db.query(UserTenant) + .filter( + UserTenant.keycloak_user_id == keycloak_user_id, + UserTenant.is_active == True, + ) + .first() + ) + if not user_tenant: + raise HTTPException( + status_code=400, detail="User does not belong to any tenant" + ) + + ext = Path(file.filename or "image.jpg").suffix.lower() or ".jpg" + if ext not in _AVATAR_EXT: + raise HTTPException( + status_code=400, + detail=f"Extensión no permitida. Use: {', '.join(sorted(_AVATAR_EXT))}", + ) + contents = await file.read() if len(contents) > 2 * 1024 * 1024: raise HTTPException(status_code=400, detail="La imagen debe ser menor a 2MB") - # Crear directorio si no existe - upload_dir = Path("/app/uploads/avatars") - upload_dir.mkdir(parents=True, exist_ok=True) + tenant_id = user_tenant.tenant_id - # Generar nombre con keycloak_user_id (sobrescribe si existe) - keycloak_user_id = current_user.get("sub") - ext = Path(file.filename or "image.jpg").suffix - filename = f"{keycloak_user_id}{ext}" - file_path = upload_dir / filename + try: + if settings.use_s3_object_storage: + if user_tenant.avatar_url and str(user_tenant.avatar_url).startswith( + "tenants/" + ): + delete_object_if_exists(str(user_tenant.avatar_url)) + key = user_avatar_key(tenant_id, keycloak_user_id, ext) + ct = file.content_type or mimetypes.guess_type(f"x{ext}")[0] or "image/jpeg" + put_object_bytes(key, contents, content_type=ct) + user_tenant.avatar_url = key + logger.info( + "User avatar stored in S3 key=%s bytes=%s", key, len(contents) + ) + else: + upload_dir = Path("uploads/avatars") + upload_dir.mkdir(parents=True, exist_ok=True) + filename = f"{keycloak_user_id}{ext}" + file_path = upload_dir / filename + with open(file_path, "wb") as f: + f.write(contents) + user_tenant.avatar_url = f"/uploads/avatars/{filename}" - # Guardar archivo - with open(file_path, "wb") as f: - f.write(contents) + db.add(user_tenant) + db.commit() + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + except Exception as e: + db.rollback() + raise HTTPException( + status_code=500, detail=f"Error al guardar el avatar: {str(e)}" + ) from e - # Retornar URL relativa - avatar_url = f"/uploads/avatars/{filename}" - - return {"avatar_url": avatar_url} + public_url = public_user_avatar_api_path(tenant_id, keycloak_user_id) + return {"avatar_url": public_url} diff --git a/backend/api/v1/modules/core/users/service.py b/backend/api/v1/modules/core/users/service.py index b7b313ff..38464160 100644 --- a/backend/api/v1/modules/core/users/service.py +++ b/backend/api/v1/modules/core/users/service.py @@ -43,9 +43,16 @@ def _normalize_keycloak_user( # Agregar campos de perfil si user_tenant está disponible if user_tenant: + avatar_out = user_tenant.avatar_url + if avatar_out and str(avatar_out).startswith("tenants/"): + from core.s3_keys import public_user_avatar_api_path + + avatar_out = public_user_avatar_api_path( + user_tenant.tenant_id, user_tenant.keycloak_user_id + ) normalized.update( { - "avatar_url": user_tenant.avatar_url, + "avatar_url": avatar_out, "phone": user_tenant.phone, "bio": user_tenant.bio, "preferences": user_tenant.preferences or {}, @@ -435,7 +442,9 @@ class UserService: # Actualizar campos en UserTenant if role is not None: user_tenant.role = role - if avatar_url is not None: + if avatar_url is not None and not str(avatar_url).startswith( + "/api/v1/core/users/avatar/" + ): user_tenant.avatar_url = avatar_url if phone is not None: user_tenant.phone = phone @@ -641,7 +650,9 @@ class UserService: self.keycloak_admin.update_user(keycloak_user_id, update_data) # Actualizar campos de perfil en UserTenant - if avatar_url is not None: + if avatar_url is not None and not str(avatar_url).startswith( + "/api/v1/core/users/avatar/" + ): user_tenant.avatar_url = avatar_url if phone is not None: user_tenant.phone = phone diff --git a/backend/core/config.py b/backend/core/config.py index eaabdb82..e356a397 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -3,7 +3,7 @@ Configuración centralizada de la aplicación usando Pydantic Settings """ import os -from typing import List +from typing import List, Literal from pydantic import field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -66,6 +66,18 @@ class Settings(BaseSettings): SMTP_FROM_NAME: str = "Sistema Anexo76" SMTP_USE_TLS: bool = True + # CSV imports (layouts_csv): redis = base64 en Valkey; minio = S3 + referencia en Redis + CSV_IMPORT_STORAGE: Literal["redis", "minio"] = "minio" + S3_ENDPOINT_URL: str = "http://minio:9000" + S3_ACCESS_KEY: str = "" + S3_SECRET_KEY: str = "" + S3_BUCKET: str = "anexo76" + S3_REGION: str = "us-east-1" + S3_USE_SSL: bool = False + # Logos, certificados, help (si no quieres MinIO aquí, pon false Y CSV_IMPORT_STORAGE=redis) + S3_FILE_STORAGE: bool = True + S3_PRESIGNED_EXPIRES_SECONDS: int = 3600 + model_config = SettingsConfigDict( env_file=[".env", "../.env"], case_sensitive=True, @@ -95,6 +107,14 @@ class Settings(BaseSettings): """Lista de orígenes CORS permitidos""" return [origin.strip() for origin in self.CORS_ORIGINS.split(",")] + @property + def use_s3_object_storage(self) -> bool: + """ + Usar MinIO para logos, certificados y Help (mismo bucket que CSV). + True si los imports CSV ya usan MinIO o si S3_FILE_STORAGE está activo. + """ + return self.CSV_IMPORT_STORAGE == "minio" or self.S3_FILE_STORAGE + # Instancia global de configuración settings = Settings() diff --git a/backend/core/middleware.py b/backend/core/middleware.py index edd39f75..9893b57f 100644 --- a/backend/core/middleware.py +++ b/backend/core/middleware.py @@ -22,6 +22,7 @@ class TenantMiddleware(BaseHTTPMiddleware): "/api/", "/uploads", "/api/v1/core/help-center", + "/api/v1/core/users/avatar", ] path = request.url.path @@ -88,6 +89,7 @@ class LicenseValidationMiddleware(BaseHTTPMiddleware): "/api/health", "/api/", "/api/v1/core/help-center", + "/api/v1/core/users/avatar", ] # Verificar si la ruta está exenta (comparación exacta o prefijo) diff --git a/backend/core/s3_keys.py b/backend/core/s3_keys.py new file mode 100644 index 00000000..1108b02f --- /dev/null +++ b/backend/core/s3_keys.py @@ -0,0 +1,325 @@ +""" +Convención de claves S3/MinIO para objetos persistidos. + +Todas las cargas que usen ``put_object_bytes`` deben obtener la clave mediante +funciones de este módulo (no construir ``tenants/...`` a mano en las rutas HTTP). + +Árbol canónico +-------------- + +**Multi-tenant** (datos de clientes), siempre bajo ``tenants/{tenant_id}/``: + +- ``tenants/{tid}/users/{keycloak_sub}/`` + Perfil de usuario (avatar). Ver ``tenant_user_prefix``, ``user_avatar_key``. + +- ``tenants/{tid}/companies/{company_id}/`` + Recursos ligados a una empresa: + + - ``.../branding/{filename}`` — logo. ``company_logo_key``. + - ``.../certificates/{tipo}_{timestamp}.{cer|key}`` — CER/KEY FIEL, CFDI, cancelación. + ``company_certificate_key``. + - ``.../imports/csv/{job_type}/{job_id}.csv`` — CSV de layouts (import jobs). + ``csv_import_key`` (usado por ``storage_s3.s3_key_for_csv_import``). + - ``.../customs_brokers/{broker_id}/certificates/`` — CER del VU (``.cer``). + ``customs_broker_vu_certificate_key``. + - ``.../customs_brokers/{broker_id}/keys/`` — llave privada VU (``.key``). + ``customs_broker_vu_private_key_key``. + - ``.../customs_brokers/{broker_id}/cove/`` — archivos COVE (xml, zip, etc.). + ``customs_broker_vu_cove_key``. + +**Sistema global** (no por tenant): + +- ``system/help/{carpeta opcional}/{archivo}`` — biblioteca de ayuda (imágenes, PDFs, vídeos). + ``help_asset_key``, ``global_system_prefix``. Lectura HTTP mapea a este prefijo. + +**Legado / migración**: + +- ``imports/csv/{job_type}/{job_id}.csv`` — sin tenant/company. Solo ``legacy_csv_import_key`` + (cleanup o compatibilidad). + +Constantes públicas +------------------- + +``SYSTEM_HELP_PREFIX`` — prefijo literal ``system/help/`` para lecturas y utilidades +que no pasan por ``help_asset_key``. +""" +import re +from typing import Union + +# Segmentos permitidos en claves (evita path traversal) +_SAFE_SEGMENT = re.compile(r"^[a-zA-Z0-9._\-]+$") + +# Prefijo fijo para objetos de Help Center (debe coincidir con help_asset_key / GET /files/) +SYSTEM_HELP_PREFIX = "system/help/" + + +def _segment(value: Union[int, str], label: str) -> str: + s = str(value).strip() + if not s or "/" in s or ".." in s: + raise ValueError(f"invalid {label} segment") + if not _SAFE_SEGMENT.match(s): + raise ValueError(f"invalid {label} characters") + return s + + +def tenant_company_prefix(tenant_id: Union[int, str], company_id: int) -> str: + """Prefijo `tenants/{tid}/companies/{cid}/` (termina en /).""" + tid = _segment(tenant_id, "tenant_id") + cid = _segment(company_id, "company_id") + return f"tenants/{tid}/companies/{cid}/" + + +def tenant_user_prefix(tenant_id: Union[int, str], keycloak_user_id: str) -> str: + """Prefijo `tenants/{tid}/users/{keycloak_sub}/` (avatar de perfil, sin company).""" + tid = _segment(tenant_id, "tenant_id") + kid = _segment(keycloak_user_id, "keycloak_user_id") + return f"tenants/{tid}/users/{kid}/" + + +def user_avatar_key( + tenant_id: Union[int, str], + keycloak_user_id: str, + ext: str, +) -> str: + ext = ext.lower() if ext.startswith(".") else f".{ext}" + allowed = (".jpg", ".jpeg", ".png", ".gif", ".webp") + if ext not in allowed: + raise ValueError("invalid avatar extension") + return f"{tenant_user_prefix(tenant_id, keycloak_user_id)}avatar{ext}" + + +def public_user_avatar_api_path(tenant_id: int, keycloak_user_id: str) -> str: + """Ruta GET pública para servir la imagen (sin host).""" + return f"/api/v1/core/users/avatar/{tenant_id}/{keycloak_user_id}" + + +def global_system_prefix(subpath: str = "help") -> str: + """Prefijo bajo `system/` para contenido global (p. ej. help). Termina en /.""" + sub = subpath.strip().strip("/") + if not sub: + return SYSTEM_HELP_PREFIX + parts = sub.split("/") + for p in parts: + _segment(p, "system_subpath") + return f"system/{sub}/" + + +def customs_broker_vu_prefix( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, +) -> str: + """ + Prefijo para el bloque VU del agente aduanal. + + Forma: ``tenants/{tid}/companies/{cid}/customs_brokers/{broker_id}/`` + """ + bid = _segment(str(broker_id), "broker_id") + return f"{tenant_company_prefix(tenant_id, company_id)}customs_brokers/{bid}/" + + +def customs_broker_vu_certificate_key( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, + timestamp: str, + file_ext: str, +) -> str: + """CER del VU bajo ``.../customs_brokers/{id}/certificates/vu_cer_{timestamp}.cer``.""" + ts = _segment(timestamp, "timestamp") + ext = file_ext.lower() if str(file_ext).startswith(".") else f".{file_ext}" + if ext != ".cer": + raise ValueError("VU certificate must be .cer") + base = f"vu_cer_{ts}{ext}" + return f"{customs_broker_vu_prefix(tenant_id, company_id, broker_id)}certificates/{base}" + + +def customs_broker_vu_private_key_key( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, + timestamp: str, + file_ext: str, +) -> str: + """Llave privada del VU bajo ``.../customs_brokers/{id}/keys/vu_key_{timestamp}.key``.""" + ts = _segment(timestamp, "timestamp") + ext = file_ext.lower() if str(file_ext).startswith(".") else f".{file_ext}" + if ext != ".key": + raise ValueError("VU private key must be .key") + base = f"vu_key_{ts}{ext}" + return f"{customs_broker_vu_prefix(tenant_id, company_id, broker_id)}keys/{base}" + + +def customs_broker_vu_cove_key( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, + timestamp: str, + original_filename: str, +) -> str: + """ + Archivos COVE bajo ``.../customs_brokers/{id}/cove/cove_{timestamp}_{filename}``. + Extensiones típicas: .xml, .zip, .txt, .pdf, .json + """ + ts = _segment(timestamp, "timestamp") + fn = safe_filename(original_filename) + parts = fn.rsplit(".", 1) + if len(parts) < 2: + raise ValueError("COVE file must have an extension") + ext = "." + parts[1].lower() + allowed = (".xml", ".zip", ".txt", ".pdf", ".json") + if ext not in allowed: + raise ValueError(f"COVE extension not allowed: {ext}") + base = f"cove_{ts}_{fn}" + return f"{customs_broker_vu_prefix(tenant_id, company_id, broker_id)}cove/{base}" + + +def customs_broker_vu_doda_certificate_key( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, + timestamp: str, + file_ext: str, +) -> str: + """CER DODA bajo ``.../customs_brokers/{id}/doda/certificates/doda_cer_{timestamp}.cer``.""" + ts = _segment(timestamp, "timestamp") + ext = file_ext.lower() if str(file_ext).startswith(".") else f".{file_ext}" + if ext != ".cer": + raise ValueError("DODA certificate must be .cer") + base = f"doda_cer_{ts}{ext}" + return f"{customs_broker_vu_prefix(tenant_id, company_id, broker_id)}doda/certificates/{base}" + + +def customs_broker_vu_doda_private_key_key( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, + timestamp: str, + file_ext: str, +) -> str: + """Llave DODA bajo ``.../customs_brokers/{id}/doda/keys/doda_key_{timestamp}.key``.""" + ts = _segment(timestamp, "timestamp") + ext = file_ext.lower() if str(file_ext).startswith(".") else f".{file_ext}" + if ext != ".key": + raise ValueError("DODA private key must be .key") + base = f"doda_key_{ts}{ext}" + return f"{customs_broker_vu_prefix(tenant_id, company_id, broker_id)}doda/keys/{base}" + + +def customs_broker_vu_doda_cove_key( + tenant_id: Union[int, str], + company_id: int, + broker_id: int, + timestamp: str, + original_filename: str, +) -> str: + """ + Archivos DODA XML bajo ``.../customs_brokers/{id}/doda/cove/doda_cove_{timestamp}_{filename}``. + Extensiones permitidas: .xml, .zip, .txt, .pdf, .json + """ + ts = _segment(timestamp, "timestamp") + fn = safe_filename(original_filename) + parts = fn.rsplit(".", 1) + if len(parts) < 2: + raise ValueError("DODA file must have an extension") + ext = "." + parts[1].lower() + allowed = (".xml", ".zip", ".txt", ".pdf", ".json") + if ext not in allowed: + raise ValueError(f"DODA extension not allowed: {ext}") + base = f"doda_cove_{ts}_{fn}" + return f"{customs_broker_vu_prefix(tenant_id, company_id, broker_id)}doda/cove/{base}" + + +def job_type_segment(job_type: str) -> str: + if job_type == "" or job_type == "invoice": + return "invoice" + return job_type + + +def csv_import_key( + tenant_id: Union[int, str], + company_id: int, + job_type: str, + job_id: str, +) -> str: + _segment(job_id, "job_id") + return ( + f"{tenant_company_prefix(tenant_id, company_id)}" + f"imports/csv/{job_type_segment(job_type)}/{job_id}.csv" + ) + + +def legacy_csv_import_key(job_type: str, job_id: str) -> str: + """Clave antigua sin tenant/company (solo migración / cleanup).""" + _segment(job_id, "job_id") + return f"imports/csv/{job_type_segment(job_type)}/{job_id}.csv" + + +def safe_filename(filename: str) -> str: + """Nombre de archivo final sin separadores.""" + base = filename.rsplit("/", 1)[-1].rsplit("\\", 1)[-1] + if not base or ".." in base: + raise ValueError("invalid filename") + return base + + +def company_logo_key( + tenant_id: Union[int, str], + company_id: int, + filename: str, +) -> str: + fn = safe_filename(filename) + return f"{tenant_company_prefix(tenant_id, company_id)}branding/{fn}" + + +def company_certificate_key( + tenant_id: Union[int, str], + company_id: int, + certificate_type: str, + timestamp: str, + file_ext: str, +) -> str: + ct = _segment(certificate_type.replace(".", "_"), "certificate_type") + ts = _segment(timestamp, "timestamp") + ext = file_ext.lower() if file_ext.startswith(".") else f".{file_ext}" + if ext not in (".cer", ".key"): + raise ValueError("certificate file must be .cer or .key") + base = f"{ct}_{ts}{ext}" + return f"{tenant_company_prefix(tenant_id, company_id)}certificates/{base}" + + +def help_asset_key(folder: str, new_filename: str) -> str: + """ + folder: '', 'pdfs', 'videos', 'assets' relativo a system/help/ + """ + folder = folder.strip().strip("/") + fn = safe_filename(new_filename) + if folder: + for p in folder.split("/"): + _segment(p, "help_folder") + return f"{global_system_prefix('help')}{folder}/{fn}" + return f"{global_system_prefix('help')}{fn}" + + +def help_s3_key_to_public_relative_path(key: str) -> str: + """Parte tras `system/help/` para el path del endpoint público.""" + if not key.startswith(SYSTEM_HELP_PREFIX): + raise ValueError("key is not under system/help/") + return key[len(SYSTEM_HELP_PREFIX) :] + + +def help_public_api_path(relative_under_help: str) -> str: + """URL de lectura pública bajo el router help-center (sin host).""" + rel = relative_under_help.lstrip("/") + return f"/api/v1/core/help-center/files/{rel}" + + +def system_help_object_key(relative_path: str) -> str: + """ + Clave S3 completa bajo ``system/help/`` para un path relativo (p. ej. GET /files/...). + ``relative_path`` no debe empezar por / ni contener '..'. + """ + rel = relative_path.strip().lstrip("/") + if ".." in rel or not rel: + raise ValueError("invalid help object path") + return f"{SYSTEM_HELP_PREFIX}{rel}" diff --git a/backend/core/storage_s3.py b/backend/core/storage_s3.py new file mode 100644 index 00000000..555d6ab3 --- /dev/null +++ b/backend/core/storage_s3.py @@ -0,0 +1,120 @@ +""" +Cliente S3 (MinIO): bucket, objetos genéricos, presign, imports CSV. + +Las claves de objeto deben generarse con ``core.s3_keys`` (p. ej. ``csv_import_key`` vía +``s3_key_for_csv_import``); no construir prefijos ``tenants/...`` aquí. +""" +import logging +from typing import Optional + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from core.config import settings +from core.s3_keys import csv_import_key + +logger = logging.getLogger(__name__) + + +def _client(): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT_URL, + aws_access_key_id=settings.S3_ACCESS_KEY, + aws_secret_access_key=settings.S3_SECRET_KEY, + region_name=settings.S3_REGION, + use_ssl=settings.S3_USE_SSL, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), + ) + + +def should_ensure_s3_bucket() -> bool: + return settings.use_s3_object_storage + + +def ensure_s3_bucket() -> None: + """Crea el bucket si no existe (idempotente).""" + if not should_ensure_s3_bucket(): + return + bucket = settings.S3_BUCKET + client = _client() + try: + client.head_bucket(Bucket=bucket) + logger.info("S3 bucket %s exists", bucket) + return + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code not in ("404", "NoSuchBucket", "403"): + logger.warning("head_bucket %s: %s", bucket, e) + try: + if settings.S3_REGION == "us-east-1": + client.create_bucket(Bucket=bucket) + else: + client.create_bucket( + Bucket=bucket, + CreateBucketConfiguration={"LocationConstraint": settings.S3_REGION}, + ) + logger.info("S3 bucket %s created", bucket) + except ClientError as e: + logger.error("create_bucket %s failed: %s", bucket, e) + raise + + +# Alias para código existente +def ensure_csv_import_bucket() -> None: + ensure_s3_bucket() + + +def put_object_bytes(key: str, body: bytes, content_type: str = "application/octet-stream") -> None: + _client().put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentType=content_type, + ) + + +def put_csv_object(key: str, body: bytes, content_type: str = "text/csv") -> None: + put_object_bytes(key, body, content_type=content_type) + + +def get_object_bytes(key: str) -> bytes: + resp = _client().get_object(Bucket=settings.S3_BUCKET, Key=key) + return resp["Body"].read() + + +def delete_object_if_exists(key: str) -> None: + try: + _client().delete_object(Bucket=settings.S3_BUCKET, Key=key) + except ClientError as e: + logger.warning("delete_object %s: %s", key, e) + + +def object_exists(key: str) -> bool: + try: + _client().head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError: + return False + + +def presigned_get_url(key: str, expires_in: Optional[int] = None) -> str: + sec = expires_in if expires_in is not None else settings.S3_PRESIGNED_EXPIRES_SECONDS + return _client().generate_presigned_url( + "get_object", + Params={"Bucket": settings.S3_BUCKET, "Key": key}, + ExpiresIn=sec, + ) + + +def s3_key_for_csv_import( + tenant_id, + company_id: int, + job_type: str, + job_id: str, +) -> str: + return csv_import_key(tenant_id, company_id, job_type, job_id) diff --git a/backend/main.py b/backend/main.py index c2d060ac..54685087 100644 --- a/backend/main.py +++ b/backend/main.py @@ -17,6 +17,7 @@ from api.v1.modules.a76.audit_log.middleware import UserContextMiddleware # Midd from api.v1.modules.a76.audit_log.register import register_audit from api.v1.router import router as api_v1_router from core.config import settings +from core.storage_s3 import ensure_s3_bucket from core.paths import layout_path from core.error_handlers import register_exception_handlers from core.middleware import ( @@ -54,7 +55,9 @@ async def on_startup(): """Evento de inicio de la aplicación""" logger.info("Iniciando la aplicación Anexo76...") #init_db() - run_migrations() + run_migrations() + if settings.use_s3_object_storage: + ensure_s3_bucket() logger.info("Base de datos inicializada correctamente.") diff --git a/backend/requirements.txt b/backend/requirements.txt index 7480be9f..86175896 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -20,6 +20,7 @@ passlib[bcrypt]==1.7.4 # HTTP & API httpx==0.28.1 requests==2.32.5 +boto3==1.35.36 # Utilities diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e823b73e..75af28a0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -178,6 +178,15 @@ services: - CENTRAL_SERVER_URL=${CENTRAL_SERVER_URL:-""} - SYNC_SECRET_TOKEN=${SYNC_SECRET_TOKEN:-change-this-sync-token-in-production} - SPOKE_URLS=${SPOKE_URLS:-""} + - CSV_IMPORT_STORAGE=${CSV_IMPORT_STORAGE:-minio} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + - S3_SECRET_KEY=${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} + - S3_BUCKET=${S3_BUCKET:-anexo76} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_USE_SSL=${S3_USE_SSL:-false} + - S3_FILE_STORAGE=${S3_FILE_STORAGE:-true} + - S3_PRESIGNED_EXPIRES_SECONDS=${S3_PRESIGNED_EXPIRES_SECONDS:-3600} ports: - "3467:8000" depends_on: @@ -185,6 +194,8 @@ services: condition: service_healthy keycloak: condition: service_healthy + minio: + condition: service_healthy volumes: - backend_uploads:/app/uploads - backend_layouts:/app/layouts @@ -231,6 +242,14 @@ services: - SITAR_API_URL=${SITAR_API_URL} - SITAR_API_USER=${SITAR_API_USER} - SITAR_API_PASSWORD=${SITAR_API_PASSWORD} + - CSV_IMPORT_STORAGE=${CSV_IMPORT_STORAGE:-minio} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + - S3_SECRET_KEY=${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} + - S3_BUCKET=${S3_BUCKET:-anexo76} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_USE_SSL=${S3_USE_SSL:-false} + - S3_FILE_STORAGE=${S3_FILE_STORAGE:-true} depends_on: - backend - valkey @@ -256,6 +275,14 @@ services: - SITAR_API_URL=${SITAR_API_URL} - SITAR_API_USER=${SITAR_API_USER} - SITAR_API_PASSWORD=${SITAR_API_PASSWORD} + - CSV_IMPORT_STORAGE=${CSV_IMPORT_STORAGE:-minio} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + - S3_SECRET_KEY=${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} + - S3_BUCKET=${S3_BUCKET:-anexo76} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_USE_SSL=${S3_USE_SSL:-false} + - S3_FILE_STORAGE=${S3_FILE_STORAGE:-true} depends_on: - backend - valkey @@ -274,6 +301,33 @@ services: networks: - backend-net + minio: + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: anexo76-minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + ports: + - "${MINIO_API_PORT:-9100}:9000" + - "${MINIO_CONSOLE_PORT:-9101}:9001" + volumes: + - minio_data:/data + networks: + - backend-net + restart: unless-stopped + healthcheck: + test: [ "CMD-SHELL", "curl -f http://127.0.0.1:9000/minio/health/live || exit 1" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 15s + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + # Frontend - SvelteKit frontend: image: dev.aduanasoft.com/anexo76/frontend:latest @@ -336,6 +390,8 @@ volumes: driver: local backend_layouts: driver: local + minio_data: + driver: local networks: backend-net: diff --git a/docker-compose.yml b/docker-compose.yml index 6bb012de..68d14cd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -182,6 +182,15 @@ services: - CENTRAL_SERVER_URL=${CENTRAL_SERVER_URL:-""} - SYNC_SECRET_TOKEN=${SYNC_SECRET_TOKEN:-change-this-sync-token-in-production} - SPOKE_URLS=${SPOKE_URLS:-""} + - CSV_IMPORT_STORAGE=${CSV_IMPORT_STORAGE:-minio} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + - S3_SECRET_KEY=${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} + - S3_BUCKET=${S3_BUCKET:-anexo76} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_USE_SSL=${S3_USE_SSL:-false} + - S3_FILE_STORAGE=${S3_FILE_STORAGE:-true} + - S3_PRESIGNED_EXPIRES_SECONDS=${S3_PRESIGNED_EXPIRES_SECONDS:-3600} ports: - "8000:8000" depends_on: @@ -189,6 +198,8 @@ services: condition: service_healthy keycloak: condition: service_healthy + minio: + condition: service_healthy volumes: - ./backend:/app - backend_cache:/app/__pycache__ @@ -293,6 +304,14 @@ services: - SITAR_API_URL=${SITAR_API_URL} - SITAR_API_USER=${SITAR_API_USER} - SITAR_API_PASSWORD=${SITAR_API_PASSWORD} + - CSV_IMPORT_STORAGE=${CSV_IMPORT_STORAGE:-minio} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + - S3_SECRET_KEY=${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} + - S3_BUCKET=${S3_BUCKET:-anexo76} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_USE_SSL=${S3_USE_SSL:-false} + - S3_FILE_STORAGE=${S3_FILE_STORAGE:-true} depends_on: - backend - valkey @@ -317,6 +336,14 @@ services: - CORE_DB_NAME=${CORE_DB_NAME:-anexo76_core} - CORE_DB_USER=${CORE_DB_USER:-postgres} - CORE_DB_PASSWORD=${POSTGRES_APP_PASSWORD:-postgres} + - CSV_IMPORT_STORAGE=${CSV_IMPORT_STORAGE:-minio} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} + - S3_SECRET_KEY=${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}} + - S3_BUCKET=${S3_BUCKET:-anexo76} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_USE_SSL=${S3_USE_SSL:-false} + - S3_FILE_STORAGE=${S3_FILE_STORAGE:-true} depends_on: - backend - valkey @@ -336,6 +363,33 @@ services: networks: - backend-net + minio: + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: anexo76-minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + ports: + - "${MINIO_API_PORT:-9100}:9000" + - "${MINIO_CONSOLE_PORT:-9101}:9001" + volumes: + - minio_data:/data + networks: + - backend-net + restart: unless-stopped + healthcheck: + test: [ "CMD-SHELL", "curl -f http://127.0.0.1:9000/minio/health/live || exit 1" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 15s + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + volumes: postgres_app_data: driver: local @@ -349,6 +403,8 @@ volumes: driver: local backend_uploads: driver: local + minio_data: + driver: local networks: backend-net: diff --git a/frontend/src/lib/api/dashboard/a76/customs-brokers.ts b/frontend/src/lib/api/dashboard/a76/customs-brokers.ts index 1362a11a..4dc83595 100644 --- a/frontend/src/lib/api/dashboard/a76/customs-brokers.ts +++ b/frontend/src/lib/api/dashboard/a76/customs-brokers.ts @@ -1,6 +1,24 @@ import { api } from '$lib/api'; import { companyStore } from '$lib/stores/company.svelte'; // <--- NUEVO: Importamos el store para el fallback import type { ApiResponse } from '$lib/api'; +import { getToken } from '$lib/auth'; + +export type VuUploadFileKind = + | 'certificate' + | 'key' + | 'cove' + | 'doda_certificate' + | 'doda_key' + | 'doda_cove'; + +export interface CustomsBrokerVuUploadResult { + message: string; + file_kind: string; + field: string; + path: string; + broker_key: string; + company_id: number; +} export interface CustomsBroker { id: number; @@ -165,4 +183,40 @@ export const customsBrokersApi = { data ); } -}; \ No newline at end of file +}; + +/** + * Sube CER, KEY o COVE del VU al bucket (MinIO: tenants/.../customs_brokers/...). + */ +export async function uploadCustomsBrokerVuFile( + brokerKey: string, + companyId: string, + fileKind: VuUploadFileKind, + file: File +): Promise> { + const formData = new FormData(); + formData.append('file', file); + const token = getToken(); + const API_BASE_URL = (import.meta.env.VITE_API_URL || '').replace(/\/+$/, ''); + const q = new URLSearchParams({ + company_id: companyId, + file_kind: fileKind + }); + const response = await fetch( + `${API_BASE_URL}/v1/a76/customs-brokers/${encodeURIComponent(brokerKey)}/vu/upload?${q.toString()}`, + { + method: 'POST', + headers: token ? { Authorization: `Bearer ${token}` } : {}, + body: formData, + credentials: 'include' + } + ); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + return { + error: (data as { detail?: string }).detail || (data as { message?: string }).message || 'Error al subir el archivo VU', + status: response.status + }; + } + return { data: data as CustomsBrokerVuUploadResult, status: response.status }; +} \ No newline at end of file diff --git a/frontend/src/lib/api/dashboard/a76/general_catalogs/company.ts b/frontend/src/lib/api/dashboard/a76/general_catalogs/company.ts index 11b037a1..f5814474 100644 --- a/frontend/src/lib/api/dashboard/a76/general_catalogs/company.ts +++ b/frontend/src/lib/api/dashboard/a76/general_catalogs/company.ts @@ -1,5 +1,6 @@ import { api } from '$lib/api'; import type { ApiResponse } from '$lib/api'; +import { getToken } from '$lib/auth'; export interface CompanyAddress { id?: number; @@ -450,14 +451,12 @@ export async function uploadCompanyLogo(id: number, file: File): Promise = T extends { child?: any } ? Omit : T; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte b/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte index 0c001f08..43de3704 100644 --- a/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte +++ b/frontend/src/routes/dashboard/customs_brokers/edit/[[id]]/+page.svelte @@ -4,6 +4,7 @@ import { companyStore } from '$lib/stores/company.svelte'; import { customsBrokersApi, + uploadCustomsBrokerVuFile, type CreateCustomsBrokerData } from '$lib/api/dashboard/a76/customs-brokers'; @@ -41,6 +42,7 @@ ShieldCheck } from 'lucide-svelte'; import { toast } from 'svelte-sonner'; + import { getFileDisplayName } from '$lib/utils'; import { useShortcuts } from '$lib/hooks/use-shortcuts'; import { obtenerAtajosEdicionAgente } from '$lib/config/shortcuts/dashboard/customs_brokers/edit'; @@ -100,6 +102,15 @@ archive_path: '' }); + let pendingVuFiles = $state<{ + certificate?: File; + key?: File; + cove?: File; + dodaCertificate?: File; + dodaKey?: File; + dodaCove?: File; + }>({}); + let brokerKeyError = $state(false); let licenseError = $state(false); let brokerKeyTimeout: ReturnType; @@ -121,10 +132,88 @@ const file = input.files?.[0]; if (file) { vuData[targetKey] = file.name; + if (targetKey === 'certificate_path') pendingVuFiles.certificate = file; + if (targetKey === 'key_path') pendingVuFiles.key = file; + if (targetKey === 'xml_files_path') pendingVuFiles.cove = file; + if (targetKey === 'doda_certificate_path') pendingVuFiles.dodaCertificate = file; + if (targetKey === 'doda_key_path') pendingVuFiles.dodaKey = file; + if (targetKey === 'doda_xml_files_path') pendingVuFiles.dodaCove = file; toast.success(`Archivo ${file.name} seleccionado`); } } + async function uploadPendingVuFiles(companyId: string, brokerKey: string) { + if (pendingVuFiles.certificate) { + const res = await uploadCustomsBrokerVuFile( + brokerKey, + companyId, + 'certificate', + pendingVuFiles.certificate + ); + if ((res as any).error || !(res as any).data?.path) { + throw new Error((res as any).error || 'Error al subir certificado VU (.cer)'); + } + vuData.certificate_path = (res as any).data.path; + } + + if (pendingVuFiles.key) { + const res = await uploadCustomsBrokerVuFile(brokerKey, companyId, 'key', pendingVuFiles.key); + if ((res as any).error || !(res as any).data?.path) { + throw new Error((res as any).error || 'Error al subir llave VU (.key)'); + } + vuData.key_path = (res as any).data.path; + } + + if (pendingVuFiles.cove) { + const res = await uploadCustomsBrokerVuFile(brokerKey, companyId, 'cove', pendingVuFiles.cove); + if ((res as any).error || !(res as any).data?.path) { + throw new Error((res as any).error || 'Error al subir archivo COVE'); + } + vuData.xml_files_path = (res as any).data.path; + } + + if (pendingVuFiles.dodaCertificate) { + const res = await uploadCustomsBrokerVuFile( + brokerKey, + companyId, + 'doda_certificate', + pendingVuFiles.dodaCertificate + ); + if ((res as any).error || !(res as any).data?.path) { + throw new Error((res as any).error || 'Error al subir certificado DODA (.cer)'); + } + vuData.doda_certificate_path = (res as any).data.path; + } + + if (pendingVuFiles.dodaKey) { + const res = await uploadCustomsBrokerVuFile( + brokerKey, + companyId, + 'doda_key', + pendingVuFiles.dodaKey + ); + if ((res as any).error || !(res as any).data?.path) { + throw new Error((res as any).error || 'Error al subir llave DODA (.key)'); + } + vuData.doda_key_path = (res as any).data.path; + } + + if (pendingVuFiles.dodaCove) { + const res = await uploadCustomsBrokerVuFile( + brokerKey, + companyId, + 'doda_cove', + pendingVuFiles.dodaCove + ); + if ((res as any).error || !(res as any).data?.path) { + throw new Error((res as any).error || 'Error al subir archivo DODA'); + } + vuData.doda_xml_files_path = (res as any).data.path; + } + + pendingVuFiles = {}; + } + async function loadBrokerData(key: string, cId: string) { if (!key || key === 'undefined') return; loading = true; @@ -279,6 +368,9 @@ if ((res as any).error) throw new Error((res as any).error); + // Primero sube archivos CER/KEY/COVE al bucket y guarda la ruta real en vuData. + await uploadPendingVuFiles(cId, formData.broker_key); + // UPSERT VU try { const vuRes = await customsBrokersApi.updateVU(formData.broker_key, vuData, cId); @@ -694,15 +786,20 @@
@@ -726,15 +823,20 @@
@@ -836,8 +938,13 @@ >
@@ -950,15 +1057,20 @@
@@ -982,15 +1094,20 @@
@@ -1039,8 +1156,13 @@ >
diff --git a/frontend/src/routes/dashboard/general_catalogs/company_information/edit/[[id]]/+page.svelte b/frontend/src/routes/dashboard/general_catalogs/company_information/edit/[[id]]/+page.svelte index 09f7ced6..1401d421 100644 --- a/frontend/src/routes/dashboard/general_catalogs/company_information/edit/[[id]]/+page.svelte +++ b/frontend/src/routes/dashboard/general_catalogs/company_information/edit/[[id]]/+page.svelte @@ -16,7 +16,7 @@ type Company } from '$lib/api/dashboard/a76/general_catalogs/company'; import { companyStore } from '$lib/stores/company.svelte'; - import { getBackendAssetUrl } from '$lib/utils'; + import { getBackendAssetUrl, getFileDisplayName } from '$lib/utils'; import { ArrowLeft, LoaderCircle, @@ -498,7 +498,7 @@ successMessage = null; try { - const response = await uploadCompanyCertificate(Number(id), file, type); + const response = await uploadCompanyCertificate(Number(id), type, file); if (response.error) { throw new Error(response.error); @@ -1614,9 +1614,7 @@ > - {formData.fiel_cer - ? formData.fiel_cer.split('/').pop() - : 'Seleccionar archivo .cer'} + {getFileDisplayName(formData.fiel_cer, 'CER', 'Seleccionar archivo .cer')}
@@ -1638,9 +1636,7 @@ > - {formData.fiel_key - ? formData.fiel_key.split('/').pop() - : 'Seleccionar archivo .key'} + {getFileDisplayName(formData.fiel_key, 'KEY', 'Seleccionar archivo .key')}
@@ -1688,9 +1684,11 @@ > - {formData.cfdi_cert_cer - ? formData.cfdi_cert_cer.split('/').pop() - : 'Seleccionar archivo .cer'} + {getFileDisplayName( + formData.cfdi_cert_cer, + 'CER', + 'Seleccionar archivo .cer' + )}
@@ -1712,9 +1710,11 @@ > - {formData.cfdi_cert_key - ? formData.cfdi_cert_key.split('/').pop() - : 'Seleccionar archivo .key'} + {getFileDisplayName( + formData.cfdi_cert_key, + 'KEY', + 'Seleccionar archivo .key' + )}
@@ -1774,9 +1774,7 @@ > - {formData.cancel_cer - ? formData.cancel_cer.split('/').pop() - : 'Seleccionar archivo .cer'} + {getFileDisplayName(formData.cancel_cer, 'CER', 'Seleccionar archivo .cer')}
@@ -1798,9 +1796,7 @@ > - {formData.cancel_key - ? formData.cancel_key.split('/').pop() - : 'Seleccionar archivo .key'} + {getFileDisplayName(formData.cancel_key, 'KEY', 'Seleccionar archivo .key')}
diff --git a/frontend/src/routes/dashboard/help-center/editor/[uuid]/+page.svelte b/frontend/src/routes/dashboard/help-center/editor/[uuid]/+page.svelte index 2455fea8..d8ea7064 100644 --- a/frontend/src/routes/dashboard/help-center/editor/[uuid]/+page.svelte +++ b/frontend/src/routes/dashboard/help-center/editor/[uuid]/+page.svelte @@ -26,6 +26,7 @@ import { toast } from 'svelte-sonner'; import { marked } from 'marked'; import DOMPurify from 'dompurify'; + import { getFileNameFromPath } from '$lib/utils'; // Helper to resolve an API path from the DB (e.g., /api/uploads/...) to a full URL pointing to the local backend. function resolveAssetUrl(url: string | undefined): string { @@ -439,7 +440,9 @@

Archivo Cargado

-

{file_url}

+

+ {getFileNameFromPath(file_url)} +

{mime_type} • {file_size ? (file_size / 1024 / 1024).toFixed(2) : '??'} MB

From 1355bf34c4470ebbd673d6dc86fb771a0ceecc2b Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 6 Apr 2026 07:43:43 -0600 Subject: [PATCH 31/32] feature/file-manager-user-only-read --- .../api/v1/modules/a76/audit_log/router.py | 234 +++++++++++++++++- .../api/v1/modules/a76/audit_log/schemas.py | 26 ++ backend/core/storage_s3.py | 55 +++- frontend/messages/en.json | 13 + frontend/messages/es.json | 13 + frontend/src/lib/api.ts | 1 + .../src/lib/api/dashboard/a76/audit_files.ts | 53 ++++ .../routes/dashboard/audit_logs/+page.svelte | 24 +- .../src/routes/dashboard/audit_logs/+page.ts | 2 +- .../dashboard/audit_logs/files-tab.svelte | 205 +++++++++++++++ 10 files changed, 614 insertions(+), 12 deletions(-) create mode 100644 frontend/src/lib/api/dashboard/a76/audit_files.ts create mode 100644 frontend/src/routes/dashboard/audit_logs/files-tab.svelte diff --git a/backend/api/v1/modules/a76/audit_log/router.py b/backend/api/v1/modules/a76/audit_log/router.py index 6bd03187..46d9bb11 100644 --- a/backend/api/v1/modules/a76/audit_log/router.py +++ b/backend/api/v1/modules/a76/audit_log/router.py @@ -1,19 +1,147 @@ """ Audit Log Router """ -from typing import List, Optional from datetime import date -from fastapi import APIRouter, Depends, Query, HTTPException +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session from sqlalchemy import or_, desc, distinct from core.database import get_core_db -from core.security import get_current_user # Assuming this exists +from core.security import get_current_user, get_tenant_from_token +from core.storage_s3 import get_object_bytes, list_objects_tree, should_ensure_s3_bucket from .models import AuditLog -from .schemas import AuditLogListResponse, AuditLogResponse, AuditLogDetailResponse +from .schemas import ( + AuditFileBreadcrumb, + AuditFileBrowserResponse, + AuditFileFolderItem, + AuditFileObjectItem, + AuditLogDetailResponse, + AuditLogListResponse, +) +from api.v1.modules.a76.general_catalogs.company.models import Company router = APIRouter() +_SEGMENT_LABELS = { + "tenants": "Espacio", + "companies": "Companias", + "users": "Usuarios", + "imports": "Importaciones", + "csv": "Archivos CSV", + "branding": "Logotipos", + "certificates": "Certificados", + "customs_brokers": "Agentes aduanales", + "keys": "Llaves", + "cove": "COVE", + "doda": "DODA", + "system": "Sistema", + "help": "Ayuda", +} + + +def _tenant_id_from_user(current_user: Dict[str, Any]) -> int: + tenant_id = get_tenant_from_token(current_user) or current_user.get("tenant_id") + if not tenant_id: + raise HTTPException(status_code=401, detail="User context is invalid") + return int(tenant_id) + + +def _normalize_relative_path(raw: Optional[str]) -> str: + if not raw: + return "" + val = raw.strip().strip("/") + if not val: + return "" + if ".." in val or "\\" in val: + raise HTTPException(status_code=400, detail="Invalid path") + parts = [p for p in val.split("/") if p] + for part in parts: + if part in (".", ".."): + raise HTTPException(status_code=400, detail="Invalid path segment") + return "/".join(parts) + + +def _tenant_prefix(tenant_id: int) -> str: + return f"tenants/{tenant_id}/" + + +def _relative_from_tenant_prefix(key: str, tenant_prefix: str) -> str: + if not key.startswith(tenant_prefix): + raise HTTPException(status_code=403, detail="Access denied to object key") + return key[len(tenant_prefix) :].strip("/") + + +def _companies_map(db: Session, tenant_id: int) -> Dict[str, str]: + rows = ( + db.query(Company.id, Company.name) + .filter(Company.tenant_id == tenant_id, Company.deleted_at.is_(None)) + .all() + ) + out: Dict[str, str] = {} + for company_id, company_name in rows: + if company_id is None: + continue + safe_name = (company_name or "").strip() + out[str(company_id)] = safe_name or "Compania" + return out + + +def _display_segment(part: str, prev_part: Optional[str], company_names: Dict[str, str]) -> str: + if prev_part == "companies": + return company_names.get(part, "Compania") + if part in _SEGMENT_LABELS: + return _SEGMENT_LABELS[part] + # Evita exponer IDs puros en UI. + if part.isdigit(): + return "Elemento" + return part.replace("_", " ").strip().title() or "Elemento" + + +def _display_path(rel_path: str, company_names: Dict[str, str]) -> str: + if not rel_path: + return "Raiz de archivos" + parts = [p for p in rel_path.split("/") if p] + labels: List[str] = [] + prev: Optional[str] = None + for part in parts: + labels.append(_display_segment(part, prev, company_names)) + prev = part + return " / ".join(labels) + + +def _display_file_name(filename: str) -> str: + stem, dot, ext = filename.rpartition(".") + if not dot: + stem = filename + ext = "" + if stem.isdigit(): + return f"Archivo{f'.{ext}' if ext else ''}" + return filename + + +def _build_breadcrumbs(rel_path: str, company_names: Dict[str, str]) -> List[AuditFileBreadcrumb]: + breadcrumbs: List[AuditFileBreadcrumb] = [ + AuditFileBreadcrumb(path="", display_name="Raiz de archivos") + ] + if not rel_path: + return breadcrumbs + parts = [p for p in rel_path.split("/") if p] + prev: Optional[str] = None + acc: List[str] = [] + for part in parts: + acc.append(part) + breadcrumbs.append( + AuditFileBreadcrumb( + path="/".join(acc), + display_name=_display_segment(part, prev, company_names), + ) + ) + prev = part + return breadcrumbs + @router.get("/bitacora", response_model=AuditLogListResponse) async def get_bitacora( page: int = Query(1, ge=1), @@ -91,3 +219,101 @@ async def get_audit_detail(spec_id: int, db: Session = Depends(get_core_db)): if not log: raise HTTPException(status_code=404, detail="Log entry not found") return log + + +@router.get("/files", response_model=AuditFileBrowserResponse) +async def list_tenant_files( + path: Optional[str] = Query(default="", description="Ruta relativa de navegación."), + continuation_token: Optional[str] = Query(default=None), + max_keys: int = Query(default=100, ge=1, le=500), + db: Session = Depends(get_core_db), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Explorador de archivos de solo lectura para Auditoría. + """ + if not should_ensure_s3_bucket(): + raise HTTPException(status_code=400, detail="S3 storage is disabled") + + tenant_id = _tenant_id_from_user(current_user) + tenant_prefix = _tenant_prefix(tenant_id) + rel_path = _normalize_relative_path(path) + list_prefix = f"{tenant_prefix}{rel_path}/" if rel_path else tenant_prefix + + data = list_objects_tree( + prefix=list_prefix, + delimiter="/", + max_keys=max_keys, + continuation_token=continuation_token, + ) + + company_names = _companies_map(db, tenant_id) + folders: List[AuditFileFolderItem] = [] + for prefix in data.get("prefixes", []): + rel = _relative_from_tenant_prefix(prefix, tenant_prefix) + folders.append( + AuditFileFolderItem( + path=rel, + display_name=_display_path(rel, company_names).split(" / ")[-1], + ) + ) + + files: List[AuditFileObjectItem] = [] + for obj in data.get("objects", []): + key = obj.get("key") + if not key: + continue + rel = _relative_from_tenant_prefix(key, tenant_prefix) + name = rel.rsplit("/", 1)[-1] + files.append( + AuditFileObjectItem( + path=rel, + display_name=_display_file_name(name), + size=int(obj.get("size", 0) or 0), + last_modified=obj.get("last_modified"), + ) + ) + + return AuditFileBrowserResponse( + current_path=rel_path, + display_path=_display_path(rel_path, company_names), + breadcrumbs=_build_breadcrumbs(rel_path, company_names), + folders=sorted(folders, key=lambda x: x.display_name.lower()), + files=sorted(files, key=lambda x: x.display_name.lower()), + next_token=data.get("next_continuation_token"), + ) + + +@router.get("/files/download") +async def download_tenant_file( + path: str = Query(..., description="Ruta relativa del archivo a descargar."), + current_user: Dict[str, Any] = Depends(get_current_user), +): + """ + Descarga segura (backend streaming) de archivos autorizados. + """ + if not should_ensure_s3_bucket(): + raise HTTPException(status_code=400, detail="S3 storage is disabled") + + tenant_id = _tenant_id_from_user(current_user) + tenant_prefix = _tenant_prefix(tenant_id) + rel_path = _normalize_relative_path(path) + if not rel_path or rel_path.endswith("/"): + raise HTTPException(status_code=400, detail="A file path is required") + + object_key = f"{tenant_prefix}{rel_path}" + if not object_key.startswith(tenant_prefix): + raise HTTPException(status_code=403, detail="Access denied to object key") + + try: + body = get_object_bytes(object_key) + except Exception as e: + raise HTTPException(status_code=404, detail=f"File not found: {e}") from e + + filename = rel_path.rsplit("/", 1)[-1] + headers = {"Content-Disposition": f'attachment; filename="{filename}"'} + return StreamingResponse( + iter([body]), + media_type="application/octet-stream", + headers=headers, + ) diff --git a/backend/api/v1/modules/a76/audit_log/schemas.py b/backend/api/v1/modules/a76/audit_log/schemas.py index bd803974..ae1ad0e0 100644 --- a/backend/api/v1/modules/a76/audit_log/schemas.py +++ b/backend/api/v1/modules/a76/audit_log/schemas.py @@ -50,3 +50,29 @@ class AuditLogListResponse(BaseModel): total: int page: int page_size: int + + +class AuditFileBreadcrumb(BaseModel): + path: str = Field(default="") + display_name: str + + +class AuditFileFolderItem(BaseModel): + path: str = Field(description="Ruta relativa interna del archivo, para navegación.") + display_name: str + + +class AuditFileObjectItem(BaseModel): + path: str = Field(description="Ruta relativa interna del archivo, para descarga.") + display_name: str + size: int + last_modified: Optional[datetime] = None + + +class AuditFileBrowserResponse(BaseModel): + current_path: str = Field(default="") + display_path: str = Field(default="") + breadcrumbs: List[AuditFileBreadcrumb] + folders: List[AuditFileFolderItem] + files: List[AuditFileObjectItem] + next_token: Optional[str] = None diff --git a/backend/core/storage_s3.py b/backend/core/storage_s3.py index 555d6ab3..5febded2 100644 --- a/backend/core/storage_s3.py +++ b/backend/core/storage_s3.py @@ -5,7 +5,7 @@ Las claves de objeto deben generarse con ``core.s3_keys`` (p. ej. ``csv_import_k ``s3_key_for_csv_import``); no construir prefijos ``tenants/...`` aquí. """ import logging -from typing import Optional +from typing import Any, Dict, List, Optional import boto3 from botocore.config import Config @@ -111,6 +111,59 @@ def presigned_get_url(key: str, expires_in: Optional[int] = None) -> str: ) +def list_objects_tree( + prefix: str, + delimiter: str = "/", + max_keys: int = 100, + continuation_token: Optional[str] = None, +) -> Dict[str, Any]: + """ + Lista objetos/prefijos como árbol virtual. + + Retorna: + - ``prefixes``: subcarpetas (CommonPrefixes) + - ``objects``: objetos directos bajo ``prefix`` + - ``next_continuation_token`` y ``is_truncated`` para paginación + """ + params: Dict[str, Any] = { + "Bucket": settings.S3_BUCKET, + "Prefix": prefix, + "Delimiter": delimiter, + "MaxKeys": max(1, min(int(max_keys), 500)), + } + if continuation_token: + params["ContinuationToken"] = continuation_token + + resp = _client().list_objects_v2(**params) + common_prefixes: List[str] = [ + p.get("Prefix", "") for p in (resp.get("CommonPrefixes") or []) if p.get("Prefix") + ] + objects: List[Dict[str, Any]] = [] + for obj in resp.get("Contents") or []: + key = obj.get("Key") + if not key: + continue + if key == prefix: + # Marcador de carpeta (objeto vacío con mismo nombre del prefijo). + continue + objects.append( + { + "key": key, + "size": int(obj.get("Size", 0) or 0), + "last_modified": obj.get("LastModified"), + "etag": obj.get("ETag"), + "storage_class": obj.get("StorageClass"), + } + ) + + return { + "prefixes": common_prefixes, + "objects": objects, + "next_continuation_token": resp.get("NextContinuationToken"), + "is_truncated": bool(resp.get("IsTruncated")), + } + + def s3_key_for_csv_import( tenant_id, company_id: int, diff --git a/frontend/messages/en.json b/frontend/messages/en.json index a4985a02..ed08c90a 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -119,6 +119,19 @@ "audit_logs_description": "Audit trail of operations and background task (Celery) status.", "audit_logs_tab_bitacora": "Audit trail", "audit_logs_tab_tasks": "Background tasks", + "audit_logs_tab_files": "File manager", + "audit_logs_files_title": "File manager", + "audit_logs_files_root": "Files root", + "audit_logs_files_refresh": "Refresh", + "audit_logs_files_list_title": "Contents", + "audit_logs_files_error_prefix": "Error:", + "audit_logs_files_col_name": "Name", + "audit_logs_files_col_size": "Size", + "audit_logs_files_col_modified": "Modified", + "audit_logs_files_col_actions": "Actions", + "audit_logs_files_loading": "Loading files...", + "audit_logs_files_empty": "No files or folders found in this location.", + "audit_logs_files_download": "Download", "client_provider_type": { "client_indicator": "C", "provider_indicator": "P", diff --git a/frontend/messages/es.json b/frontend/messages/es.json index 53e27d56..b0ab3e96 100644 --- a/frontend/messages/es.json +++ b/frontend/messages/es.json @@ -119,6 +119,19 @@ "audit_logs_description": "Auditoría de operaciones y seguimiento de tareas en segundo plano (Celery).", "audit_logs_tab_bitacora": "Bitácora", "audit_logs_tab_tasks": "Tareas en segundo plano", + "audit_logs_tab_files": "Gestor de archivos", + "audit_logs_files_title": "Gestor de archivos", + "audit_logs_files_root": "Raíz de archivos", + "audit_logs_files_refresh": "Actualizar", + "audit_logs_files_list_title": "Contenido", + "audit_logs_files_error_prefix": "Error:", + "audit_logs_files_col_name": "Nombre", + "audit_logs_files_col_size": "Tamaño", + "audit_logs_files_col_modified": "Modificado", + "audit_logs_files_col_actions": "Acciones", + "audit_logs_files_loading": "Cargando archivos...", + "audit_logs_files_empty": "No hay archivos o carpetas en esta ubicación.", + "audit_logs_files_download": "Descargar", "client_provider_type": { "client_indicator": "C", "provider_indicator": "P", diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 42b5290a..ce180b4f 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -456,6 +456,7 @@ async function fetchBlob(endpoint: string, options: RequestInit = {}): Promise(endpoint: string) => fetchApi(endpoint, { method: 'GET' }), + getBlob: (endpoint: string) => fetchBlob(endpoint, { method: 'GET' }), post: (endpoint: string, body: any, options: RequestInit = {}) => fetchApi(endpoint, { diff --git a/frontend/src/lib/api/dashboard/a76/audit_files.ts b/frontend/src/lib/api/dashboard/a76/audit_files.ts new file mode 100644 index 00000000..def3ba73 --- /dev/null +++ b/frontend/src/lib/api/dashboard/a76/audit_files.ts @@ -0,0 +1,53 @@ +import { api } from '$lib/api'; + +const BASE_PATH = '/v1/a76/audit-log/files'; + +export interface AuditFileBreadcrumb { + path: string; + display_name: string; +} + +export interface AuditFolderItem { + path: string; + display_name: string; +} + +export interface AuditFileItem { + path: string; + display_name: string; + size: number; + last_modified?: string | null; +} + +export interface AuditFileListResponse { + current_path: string; + display_path: string; + breadcrumbs: AuditFileBreadcrumb[]; + folders: AuditFolderItem[]; + files: AuditFileItem[]; + next_token?: string | null; +} + +export const AuditFilesAPI = { + list: async (params?: { + path?: string; + continuation_token?: string; + max_keys?: number; + }): Promise => { + const query = new URLSearchParams(); + if (params?.path) query.set('path', params.path); + if (params?.continuation_token) query.set('continuation_token', params.continuation_token); + if (params?.max_keys) query.set('max_keys', String(params.max_keys)); + + const qs = query.toString(); + const endpoint = qs ? `${BASE_PATH}?${qs}` : BASE_PATH; + const response = await api.get(endpoint); + if (response.error || !response.data) { + throw new Error(response.error || 'Failed to list tenant files'); + } + return response.data; + }, + + downloadBlob: (path: string) => + api.getBlob(`${BASE_PATH}/download?path=${encodeURIComponent(path)}`) +}; diff --git a/frontend/src/routes/dashboard/audit_logs/+page.svelte b/frontend/src/routes/dashboard/audit_logs/+page.svelte index 7de90e27..6d41408e 100644 --- a/frontend/src/routes/dashboard/audit_logs/+page.svelte +++ b/frontend/src/routes/dashboard/audit_logs/+page.svelte @@ -2,10 +2,11 @@ import { afterNavigate, goto } from '$app/navigation'; import { page } from '$app/state'; import * as Tabs from '$lib/components/ui/tabs'; - import { ScrollText, ListTodo } from 'lucide-svelte'; + import { ScrollText, ListTodo, FolderTree } from 'lucide-svelte'; import * as m from '$lib/paraglide/messages.js'; import BitacoraTab from './bitacora-tab.svelte'; import TasksTab from './tasks-tab.svelte'; + import FilesTab from './files-tab.svelte'; import type { PageData } from './$types'; let { data }: { data: PageData } = $props(); @@ -17,17 +18,19 @@ }); afterNavigate(() => { - const t = page.url.searchParams.get('tab') === 'tasks' ? 'tasks' : 'bitacora'; + const raw = page.url.searchParams.get('tab'); + const t = raw === 'tasks' || raw === 'files' ? raw : 'bitacora'; tabValue = t; }); function onTabValueChange(v: string) { - const fromUrl = page.url.searchParams.get('tab') === 'tasks' ? 'tasks' : 'bitacora'; + const raw = page.url.searchParams.get('tab'); + const fromUrl = raw === 'tasks' || raw === 'files' ? raw : 'bitacora'; if (v === fromUrl) return; const u = new URL(page.url.href); - if (v === 'tasks') { - u.searchParams.set('tab', 'tasks'); + if (v === 'tasks' || v === 'files') { + u.searchParams.set('tab', v); } else { u.searchParams.delete('tab'); } @@ -53,7 +56,7 @@ onValueChange={onTabValueChange} class="flex min-h-0 w-full flex-1 flex-col" > - + {m['sidebar.audit_logs_tab_bitacora']()} @@ -62,6 +65,10 @@ {m['sidebar.audit_logs_tab_tasks']()} + + + {m['sidebar.audit_logs_tab_files']()} + @@ -74,5 +81,10 @@ {/if} + + {#if tabValue === 'files'} + + {/if} +
diff --git a/frontend/src/routes/dashboard/audit_logs/+page.ts b/frontend/src/routes/dashboard/audit_logs/+page.ts index 1bf476b2..a653eef6 100644 --- a/frontend/src/routes/dashboard/audit_logs/+page.ts +++ b/frontend/src/routes/dashboard/audit_logs/+page.ts @@ -2,6 +2,6 @@ import type { PageLoad } from './$types'; export const load: PageLoad = ({ url }) => { const tab = url.searchParams.get('tab'); - const initialTab = tab === 'tasks' ? 'tasks' : 'bitacora'; + const initialTab = tab === 'tasks' || tab === 'files' ? tab : 'bitacora'; return { initialTab }; }; diff --git a/frontend/src/routes/dashboard/audit_logs/files-tab.svelte b/frontend/src/routes/dashboard/audit_logs/files-tab.svelte new file mode 100644 index 00000000..051df209 --- /dev/null +++ b/frontend/src/routes/dashboard/audit_logs/files-tab.svelte @@ -0,0 +1,205 @@ + + +
+
+
+

{m['sidebar.audit_logs_files_title']()}

+

{displayPath || m['sidebar.audit_logs_files_root']()}

+
+ +
+ + + +
+ {#each breadcrumbs as crumb, idx} + {#if idx > 0} + / + {/if} + {#if idx === breadcrumbs.length - 1} + + {crumb.display_name} + + {:else} + + {/if} + {/each} +
+ + + + + + {m['sidebar.audit_logs_files_list_title']()} + + + {#if error} +
+ {m['sidebar.audit_logs_files_error_prefix']()} {error} +
+ {/if} +
+ + + + + {m['sidebar.audit_logs_files_col_name']()} + + + {m['sidebar.audit_logs_files_col_size']()} + + + {m['sidebar.audit_logs_files_col_modified']()} + + + {m['sidebar.audit_logs_files_col_actions']()} + + + + + {#if loading} + + + {m['sidebar.audit_logs_files_loading']()} + + + {:else if folders.length === 0 && files.length === 0} + + + {m['sidebar.audit_logs_files_empty']()} + + + {:else} + {#each folders as folder} + void loadPath(folder.path)} + > + + + + {folder.display_name} + + + — + — + — + + {/each} + {#each files as file} + + + {file.display_name} + + {formatSize(file.size)} + {formatDate(file.last_modified)} + + + + + {/each} + {/if} + + +
+ + +
From 0c25572d7040e677217e926af41dc9d3413af065 Mon Sep 17 00:00:00 2001 From: hreyes Date: Mon, 6 Apr 2026 08:13:01 -0600 Subject: [PATCH 32/32] feature/correcion-al-borrar-datos-borrar-tambien-del-bucket --- .../api/v1/modules/a76/audit_log/router.py | 79 +++++++++++++++++-- .../a76/general_catalogs/company/service.py | 21 ++++- backend/core/storage_s3.py | 53 +++++++++++++ 3 files changed, 144 insertions(+), 9 deletions(-) diff --git a/backend/api/v1/modules/a76/audit_log/router.py b/backend/api/v1/modules/a76/audit_log/router.py index 46d9bb11..4339286f 100644 --- a/backend/api/v1/modules/a76/audit_log/router.py +++ b/backend/api/v1/modules/a76/audit_log/router.py @@ -89,9 +89,22 @@ def _companies_map(db: Session, tenant_id: int) -> Dict[str, str]: return out -def _display_segment(part: str, prev_part: Optional[str], company_names: Dict[str, str]) -> str: +def _display_segment( + part: str, + prev_part: Optional[str], + company_names: Dict[str, str], + current_user_id: Optional[str] = None, + current_user_label: Optional[str] = None, +) -> str: if prev_part == "companies": return company_names.get(part, "Compania") + if prev_part == "users": + # Para carpetas de usuarios, mostrar un nombre amigable: + # - Si es el propio usuario actual, usar preferred_username/email/nombre. + # - Para otros IDs (UUIDs) mostrar un label genérico. + if current_user_id and part == str(current_user_id): + return (current_user_label or "").strip() or "Usuario" + return "Usuario" if part in _SEGMENT_LABELS: return _SEGMENT_LABELS[part] # Evita exponer IDs puros en UI. @@ -100,14 +113,27 @@ def _display_segment(part: str, prev_part: Optional[str], company_names: Dict[st return part.replace("_", " ").strip().title() or "Elemento" -def _display_path(rel_path: str, company_names: Dict[str, str]) -> str: +def _display_path( + rel_path: str, + company_names: Dict[str, str], + current_user_id: Optional[str] = None, + current_user_label: Optional[str] = None, +) -> str: if not rel_path: return "Raiz de archivos" parts = [p for p in rel_path.split("/") if p] labels: List[str] = [] prev: Optional[str] = None for part in parts: - labels.append(_display_segment(part, prev, company_names)) + labels.append( + _display_segment( + part, + prev, + company_names, + current_user_id=current_user_id, + current_user_label=current_user_label, + ) + ) prev = part return " / ".join(labels) @@ -122,7 +148,12 @@ def _display_file_name(filename: str) -> str: return filename -def _build_breadcrumbs(rel_path: str, company_names: Dict[str, str]) -> List[AuditFileBreadcrumb]: +def _build_breadcrumbs( + rel_path: str, + company_names: Dict[str, str], + current_user_id: Optional[str] = None, + current_user_label: Optional[str] = None, +) -> List[AuditFileBreadcrumb]: breadcrumbs: List[AuditFileBreadcrumb] = [ AuditFileBreadcrumb(path="", display_name="Raiz de archivos") ] @@ -136,7 +167,13 @@ def _build_breadcrumbs(rel_path: str, company_names: Dict[str, str]) -> List[Aud breadcrumbs.append( AuditFileBreadcrumb( path="/".join(acc), - display_name=_display_segment(part, prev, company_names), + display_name=_display_segment( + part, + prev, + company_names, + current_user_id=current_user_id, + current_user_label=current_user_label, + ), ) ) prev = part @@ -240,6 +277,15 @@ async def list_tenant_files( rel_path = _normalize_relative_path(path) list_prefix = f"{tenant_prefix}{rel_path}/" if rel_path else tenant_prefix + # Datos del usuario actual para etiquetas amigables bajo /users/{id}/... + current_user_id = str(current_user.get("sub") or "") + current_user_label = ( + (current_user.get("preferred_username") or "").strip() + or (current_user.get("name") or "").strip() + or (current_user.get("email") or "").strip() + or "Usuario" + ) + data = list_objects_tree( prefix=list_prefix, delimiter="/", @@ -254,7 +300,14 @@ async def list_tenant_files( folders.append( AuditFileFolderItem( path=rel, - display_name=_display_path(rel, company_names).split(" / ")[-1], + # Para el gestor de archivos mostramos el nombre amigable del último segmento + # (empresa, usuario actual, etc.), no el ID bruto. + display_name=_display_path( + rel, + company_names, + current_user_id=current_user_id, + current_user_label=current_user_label, + ).split(" / ")[-1], ) ) @@ -276,8 +329,18 @@ async def list_tenant_files( return AuditFileBrowserResponse( current_path=rel_path, - display_path=_display_path(rel_path, company_names), - breadcrumbs=_build_breadcrumbs(rel_path, company_names), + display_path=_display_path( + rel_path, + company_names, + current_user_id=current_user_id, + current_user_label=current_user_label, + ), + breadcrumbs=_build_breadcrumbs( + rel_path, + company_names, + current_user_id=current_user_id, + current_user_label=current_user_label, + ), folders=sorted(folders, key=lambda x: x.display_name.lower()), files=sorted(files, key=lambda x: x.display_name.lower()), next_token=data.get("next_continuation_token"), diff --git a/backend/api/v1/modules/a76/general_catalogs/company/service.py b/backend/api/v1/modules/a76/general_catalogs/company/service.py index 21502ebc..29cc68e0 100644 --- a/backend/api/v1/modules/a76/general_catalogs/company/service.py +++ b/backend/api/v1/modules/a76/general_catalogs/company/service.py @@ -10,6 +10,10 @@ from fastapi import HTTPException from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session +from core.config import settings +from core.s3_keys import tenant_company_prefix +from core.storage_s3 import delete_objects_with_prefix + from .dto import CompanyCreateDTO, CompanyResponseDTO, CompanyUpdateDTO from .models import Company from ...audit_log.services.service import AuditService @@ -594,9 +598,24 @@ class CompanyService: # ---------------------- try: + # 1) Borrado lógico en base de datos company.deleted_at = datetime.utcnow() - db.flush() + + # 2) Limpieza de objetos S3/MinIO asociados a la compañía + if settings.use_s3_object_storage: + try: + prefix = tenant_company_prefix(tenant_id, company_id) + delete_objects_with_prefix(prefix) + except Exception as e: + # No bloquear la eliminación lógica si falla la limpieza de objetos + logger.error( + "Error deleting S3 objects for company %s (tenant %s): %s", + company_id, + tenant_id, + e, + ) + db.commit() # --- Audit Log --- diff --git a/backend/core/storage_s3.py b/backend/core/storage_s3.py index 5febded2..c0c7ef0a 100644 --- a/backend/core/storage_s3.py +++ b/backend/core/storage_s3.py @@ -94,6 +94,59 @@ def delete_object_if_exists(key: str) -> None: logger.warning("delete_object %s: %s", key, e) +def delete_objects_with_prefix(prefix: str, batch_size: int = 1000) -> None: + """ + Elimina en cascada todos los objetos cuyo Key empieza con `prefix`. + + Pensado para limpiar recursos ligados a una entidad (por ejemplo, + todos los objetos de una compañía bajo `tenants/{tid}/companies/{cid}/`). + """ + if not settings.use_s3_object_storage: + return + + client = _client() + continuation_token: Optional[str] = None + + while True: + params: Dict[str, Any] = { + "Bucket": settings.S3_BUCKET, + "Prefix": prefix, + "MaxKeys": max(1, min(int(batch_size), 1000)), + } + if continuation_token: + params["ContinuationToken"] = continuation_token + + try: + resp = client.list_objects_v2(**params) + except ClientError as e: + logger.warning("list_objects_v2 for prefix %s failed: %s", prefix, e) + break + + contents = resp.get("Contents") or [] + if not contents: + break + + to_delete = [{"Key": obj.get("Key")} for obj in contents if obj.get("Key")] + if to_delete: + try: + client.delete_objects( + Bucket=settings.S3_BUCKET, + Delete={"Objects": to_delete, "Quiet": True}, + ) + except ClientError as e: + logger.warning( + "delete_objects_with_prefix %s (batch_size=%s) failed: %s", + prefix, + len(to_delete), + e, + ) + + if not resp.get("IsTruncated"): + break + + continuation_token = resp.get("NextContinuationToken") + + def object_exists(key: str) -> bool: try: _client().head_object(Bucket=settings.S3_BUCKET, Key=key)