98 lines
3.9 KiB
Python
98 lines
3.9 KiB
Python
from sqlalchemy.orm import Session
|
|
from datetime import datetime
|
|
from typing import Optional
|
|
from fastapi import HTTPException
|
|
from app.modules.edos.models import EDOS
|
|
from app.modules.edos.schema import EdosResponse, EdosUpload, messageResponse
|
|
from app.helpers.csv_mapper import CSVMapper
|
|
from app.helpers.db_adapter import DBAdapter
|
|
from app.helpers.extractCsv import CSVExtractor
|
|
|
|
class XMACSVService:
|
|
"""Orchestrator: extract, map, adapt pipeline"""
|
|
|
|
@staticmethod
|
|
def process_csv_from_url(
|
|
csv_url: str,
|
|
dry_run: bool = False,
|
|
max_rows: int = None
|
|
) -> dict:
|
|
"""Flow: download CSV, parse, map to schema, insert into database"""
|
|
|
|
try:
|
|
# =========================================================
|
|
# X - EXTRACT: Download and parse CSV
|
|
# =========================================================
|
|
print(f"Downloading CSV: {csv_url}")
|
|
|
|
content, encoding = CSVExtractor.download_csv(csv_url)
|
|
print(f" Downloaded {len(content)} bytes, encoding: {encoding}")
|
|
|
|
# Debug: show first lines
|
|
print("First 10 lines of CSV:")
|
|
lines = content.split('\n')
|
|
for i, line in enumerate(lines[:10]):
|
|
print(f"Line {i}: {repr(line[:200])}")
|
|
|
|
# Parse CSV to raw rows
|
|
raw_rows = CSVExtractor.read_csv(content)
|
|
print(f" Parsed {len(raw_rows)} rows")
|
|
|
|
if not raw_rows:
|
|
return {
|
|
'total_extracted': 0,
|
|
'total_mapped': 0,
|
|
'inserted': 0,
|
|
'dry_run': dry_run,
|
|
'errors': ['No data found in CSV']
|
|
}
|
|
|
|
print(f" Columns: {list(raw_rows[0].keys())}")
|
|
|
|
# =========================================================
|
|
# M - MAP: Transform CSV rows to database schema
|
|
# =========================================================
|
|
mapped_rows = CSVMapper.map_rows(raw_rows)
|
|
print(f" Mapped {len(mapped_rows)} rows to database schema")
|
|
|
|
if max_rows:
|
|
mapped_rows = mapped_rows[:max_rows]
|
|
print(f" Limited to {max_rows} rows")
|
|
|
|
# =========================================================
|
|
# A - ADAPT: Insert into database
|
|
# =========================================================
|
|
inserted_count = 0
|
|
errors = []
|
|
|
|
if not dry_run:
|
|
print(f" Inserting {len(mapped_rows)} rows into database")
|
|
adapter = DBAdapter()
|
|
result = adapter.insert_filtered_data(mapped_rows)
|
|
inserted_count = result.get('inserted', 0)
|
|
errors = result.get('errors', [])
|
|
print(f" Inserted: {inserted_count}")
|
|
print(f" Duplicates skipped: {result.get('duplicates_skiped', 0)}")
|
|
if errors:
|
|
print(f" Errors: {len(errors)}")
|
|
else:
|
|
print(f" DRY RUN - No data inserted")
|
|
|
|
# =========================================================
|
|
# Return response
|
|
# =========================================================
|
|
return {
|
|
'total_extracted': len(raw_rows),
|
|
'total_mapped': len(mapped_rows),
|
|
'inserted': inserted_count,
|
|
'dry_run': dry_run,
|
|
'columns': list(raw_rows[0].keys()) if raw_rows else [],
|
|
'sample': mapped_rows[:2] if mapped_rows else [],
|
|
'errors': errors[:10] if errors else [] # Limit errors in response
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"Error: {str(e)}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
raise |