from sqlalchemy.orm import Session from datetime import datetime from typing import Optional from fastapi import HTTPException from app.modules.edos.models import EDOS from app.modules.edos.schema import EdosResponse, EdosUpload, messageResponse from app.helpers.csv_mapper import CSVMapper from app.helpers.db_adapter import DBAdapter from app.helpers.extractCsv import CSVExtractor class XMACSVService: """Orchestrator: extract, map, adapt pipeline""" @staticmethod def process_csv_from_url( csv_url: str, dry_run: bool = False, max_rows: int = None ) -> dict: """Flow: download CSV, parse, map to schema, insert into database""" try: # ========================================================= # X - EXTRACT: Download and parse CSV # ========================================================= print(f"Downloading CSV: {csv_url}") content, encoding = CSVExtractor.download_csv(csv_url) print(f" Downloaded {len(content)} bytes, encoding: {encoding}") # Debug: show first lines print("First 10 lines of CSV:") lines = content.split('\n') for i, line in enumerate(lines[:10]): print(f"Line {i}: {repr(line[:200])}") # Parse CSV to raw rows raw_rows = CSVExtractor.read_csv(content) print(f" Parsed {len(raw_rows)} rows") if not raw_rows: return { 'total_extracted': 0, 'total_mapped': 0, 'inserted': 0, 'dry_run': dry_run, 'errors': ['No data found in CSV'] } print(f" Columns: {list(raw_rows[0].keys())}") # ========================================================= # M - MAP: Transform CSV rows to database schema # ========================================================= mapped_rows = CSVMapper.map_rows(raw_rows) print(f" Mapped {len(mapped_rows)} rows to database schema") if max_rows: mapped_rows = mapped_rows[:max_rows] print(f" Limited to {max_rows} rows") # ========================================================= # A - ADAPT: Insert into database # ========================================================= inserted_count = 0 errors = [] if not dry_run: print(f" Inserting {len(mapped_rows)} rows into database") adapter = DBAdapter() result = adapter.insert_filtered_data(mapped_rows) inserted_count = result.get('inserted', 0) errors = result.get('errors', []) print(f" Inserted: {inserted_count}") print(f" Duplicates skipped: {result.get('duplicates_skiped', 0)}") if errors: print(f" Errors: {len(errors)}") else: print(f" DRY RUN - No data inserted") # ========================================================= # Return response # ========================================================= return { 'total_extracted': len(raw_rows), 'total_mapped': len(mapped_rows), 'inserted': inserted_count, 'dry_run': dry_run, 'columns': list(raw_rows[0].keys()) if raw_rows else [], 'sample': mapped_rows[:2] if mapped_rows else [], 'errors': errors[:10] if errors else [] # Limit errors in response } except Exception as e: print(f"Error: {str(e)}") import traceback traceback.print_exc() raise