first commit - MVE Incrementables Parser microservice with FastAPI, JWT, Celery, Redis
This commit is contained in:
Ernesto Herrera
322
app/api/v1/incrementables.py
Normal file
322
app/api/v1/incrementables.py
Normal file
@@ -0,0 +1,322 @@
|
||||
"""Incrementables parsing endpoints."""
|
||||
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, Header, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from celery.result import AsyncResult
|
||||
import hashlib
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.security import decode_access_token
|
||||
from app.core.celery_app import celery_app
|
||||
from app.tasks.parse_tasks import parse_pdf_task
|
||||
from app.services.pdf_text import extract_text_from_pdf, PDFExtractionError
|
||||
from app.services.parser import parse_incrementables, ParsingError
|
||||
from app.schemas import (
|
||||
ParseResponse, ParseAsyncResponse, TaskStatusResponse,
|
||||
DocumentInfo, IncrementablesData, ExtractionInfo
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/v1/incrementables", tags=["Incrementables"])
|
||||
security = HTTPBearer()
|
||||
|
||||
|
||||
def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security)) -> str:
|
||||
"""
|
||||
Dependency to validate JWT token and extract username.
|
||||
"""
|
||||
payload = decode_access_token(credentials.credentials)
|
||||
username = payload.get("sub")
|
||||
|
||||
if username is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid credentials"
|
||||
)
|
||||
|
||||
return username
|
||||
|
||||
|
||||
@router.post("/parse", response_model=ParseResponse)
|
||||
async def parse_pdf(
|
||||
file: UploadFile = File(..., description="PDF file to parse"),
|
||||
document_ref: Optional[str] = Form(None, description="Optional document reference or folio"),
|
||||
x_correlation_id: Optional[str] = Header(None),
|
||||
current_user: str = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Parse incrementables section from uploaded PDF.
|
||||
|
||||
- **file**: PDF file (required)
|
||||
- **document_ref**: Optional document reference or folio
|
||||
|
||||
Returns parsed incrementables data with metadata.
|
||||
|
||||
**Authentication required**: Bearer token in Authorization header.
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
||||
# Generate or use correlation ID
|
||||
correlation_id = x_correlation_id or str(uuid.uuid4())
|
||||
|
||||
logger.info(
|
||||
f"Parse request received",
|
||||
extra={
|
||||
"correlation_id": correlation_id,
|
||||
"pdf_filename": file.filename,
|
||||
"document_ref": document_ref,
|
||||
"user": current_user
|
||||
}
|
||||
)
|
||||
|
||||
# Validate file type
|
||||
if not file.filename.lower().endswith('.pdf'):
|
||||
logger.warning(f"Invalid file type: {file.filename}", extra={"correlation_id": correlation_id})
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Only PDF files are accepted"
|
||||
)
|
||||
|
||||
# Validate content type
|
||||
if file.content_type not in ["application/pdf", "application/x-pdf"]:
|
||||
logger.warning(
|
||||
f"Invalid content type: {file.content_type}",
|
||||
extra={"correlation_id": correlation_id}
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid content type. Expected application/pdf, got {file.content_type}"
|
||||
)
|
||||
|
||||
# Read file content
|
||||
try:
|
||||
pdf_bytes = await file.read()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read file: {str(e)}", extra={"correlation_id": correlation_id})
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to read uploaded file"
|
||||
)
|
||||
|
||||
# Validate file size
|
||||
file_size_mb = len(pdf_bytes) / (1024 * 1024)
|
||||
if file_size_mb > settings.max_file_mb:
|
||||
logger.warning(
|
||||
f"File too large: {file_size_mb:.2f} MB",
|
||||
extra={"correlation_id": correlation_id}
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"File size exceeds maximum allowed size of {settings.max_file_mb} MB"
|
||||
)
|
||||
|
||||
# Calculate SHA256 hash
|
||||
file_hash = hashlib.sha256(pdf_bytes).hexdigest()
|
||||
|
||||
# Extract text from PDF
|
||||
try:
|
||||
text, page_count, extraction_method = extract_text_from_pdf(pdf_bytes)
|
||||
logger.info(
|
||||
f"Text extracted: {len(text)} characters, {page_count} pages",
|
||||
extra={"correlation_id": correlation_id}
|
||||
)
|
||||
except PDFExtractionError as e:
|
||||
logger.error(
|
||||
f"PDF extraction failed: {str(e)}",
|
||||
extra={"correlation_id": correlation_id}
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Failed to extract text from PDF: {str(e)}"
|
||||
)
|
||||
|
||||
# Parse incrementables
|
||||
try:
|
||||
parsed_data = parse_incrementables(text)
|
||||
logger.info(
|
||||
f"Successfully parsed incrementables",
|
||||
extra={
|
||||
"correlation_id": correlation_id,
|
||||
"currency": parsed_data["currency"],
|
||||
"fletes": parsed_data["fletes"]
|
||||
}
|
||||
)
|
||||
except ParsingError as e:
|
||||
logger.error(
|
||||
f"Parsing failed: {str(e)}",
|
||||
extra={"correlation_id": correlation_id}
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Failed to parse incrementables section: {str(e)}"
|
||||
)
|
||||
|
||||
# Build response
|
||||
response = ParseResponse(
|
||||
correlation_id=correlation_id,
|
||||
document=DocumentInfo(
|
||||
filename=file.filename,
|
||||
pages=page_count,
|
||||
sha256=file_hash
|
||||
),
|
||||
incrementables=IncrementablesData(
|
||||
currency=parsed_data["currency"],
|
||||
fletes=parsed_data["fletes"],
|
||||
seguros=parsed_data["seguros"],
|
||||
almacenaje_consolidacion=parsed_data["almacenaje_consolidacion"],
|
||||
regalias=parsed_data["regalias"]
|
||||
),
|
||||
extraction=ExtractionInfo(
|
||||
method=extraction_method,
|
||||
anchors_found=parsed_data["anchors_found"],
|
||||
warnings=parsed_data["warnings"]
|
||||
)
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/parse/async", response_model=ParseAsyncResponse)
|
||||
async def parse_pdf_async(
|
||||
file: UploadFile = File(..., description="PDF file to parse"),
|
||||
document_ref: Optional[str] = Form(None, description="Optional document reference or folio"),
|
||||
x_correlation_id: Optional[str] = Header(None),
|
||||
current_user: str = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Parse incrementables section from uploaded PDF asynchronously using Celery.
|
||||
|
||||
- **file**: PDF file (required)
|
||||
- **document_ref**: Optional document reference or folio
|
||||
|
||||
Returns task ID for checking status later.
|
||||
|
||||
**Authentication required**: Bearer token in Authorization header.
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
||||
# Generate or use correlation ID
|
||||
correlation_id = x_correlation_id or str(uuid.uuid4())
|
||||
|
||||
logger.info(
|
||||
f"Async parse request received",
|
||||
extra={
|
||||
"correlation_id": correlation_id,
|
||||
"pdf_filename": file.filename,
|
||||
"document_ref": document_ref,
|
||||
"user": current_user
|
||||
}
|
||||
)
|
||||
|
||||
# Validate file type
|
||||
if not file.filename.lower().endswith('.pdf'):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Only PDF files are accepted"
|
||||
)
|
||||
|
||||
# Validate content type
|
||||
if file.content_type not in ["application/pdf", "application/x-pdf"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid content type. Expected application/pdf, got {file.content_type}"
|
||||
)
|
||||
|
||||
# Read file content
|
||||
try:
|
||||
pdf_bytes = await file.read()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read file: {str(e)}", extra={"correlation_id": correlation_id})
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to read uploaded file"
|
||||
)
|
||||
|
||||
# Validate file size
|
||||
file_size_mb = len(pdf_bytes) / (1024 * 1024)
|
||||
if file_size_mb > settings.max_file_mb:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"File size exceeds maximum allowed size of {settings.max_file_mb} MB"
|
||||
)
|
||||
|
||||
# Convert bytes to hex for serialization
|
||||
pdf_hex = pdf_bytes.hex()
|
||||
|
||||
# Queue task
|
||||
task = parse_pdf_task.delay(pdf_hex, file.filename, document_ref)
|
||||
|
||||
logger.info(
|
||||
f"Task queued: {task.id}",
|
||||
extra={"correlation_id": correlation_id, "task_id": task.id}
|
||||
)
|
||||
|
||||
return ParseAsyncResponse(
|
||||
task_id=task.id,
|
||||
correlation_id=correlation_id,
|
||||
status="queued",
|
||||
message="Task queued for processing. Use /v1/incrementables/status/{task_id} to check progress."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/status/{task_id}", response_model=TaskStatusResponse)
|
||||
async def get_task_status(
|
||||
task_id: str,
|
||||
current_user: str = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Get status of an async parsing task.
|
||||
|
||||
- **task_id**: Task ID returned from /parse/async
|
||||
|
||||
Returns task status and result if completed.
|
||||
|
||||
**Authentication required**: Bearer token in Authorization header.
|
||||
"""
|
||||
task_result = AsyncResult(task_id, app=celery_app)
|
||||
|
||||
if task_result.state == "PENDING":
|
||||
return TaskStatusResponse(
|
||||
task_id=task_id,
|
||||
status="pending",
|
||||
result=None,
|
||||
error=None
|
||||
)
|
||||
elif task_result.state == "STARTED":
|
||||
return TaskStatusResponse(
|
||||
task_id=task_id,
|
||||
status="started",
|
||||
result=None,
|
||||
error=None
|
||||
)
|
||||
elif task_result.state == "SUCCESS":
|
||||
result_data = task_result.result
|
||||
if result_data.get("status") == "failed":
|
||||
return TaskStatusResponse(
|
||||
task_id=task_id,
|
||||
status="failed",
|
||||
result=None,
|
||||
error=result_data.get("message", "Unknown error")
|
||||
)
|
||||
return TaskStatusResponse(
|
||||
task_id=task_id,
|
||||
status="completed",
|
||||
result=result_data,
|
||||
error=None
|
||||
)
|
||||
elif task_result.state == "FAILURE":
|
||||
return TaskStatusResponse(
|
||||
task_id=task_id,
|
||||
status="failed",
|
||||
result=None,
|
||||
error=str(task_result.info)
|
||||
)
|
||||
else:
|
||||
return TaskStatusResponse(
|
||||
task_id=task_id,
|
||||
status=task_result.state.lower(),
|
||||
result=None,
|
||||
error=None
|
||||
)
|
||||
Reference in New Issue
Block a user