first commit - MVE Incrementables Parser microservice with FastAPI, JWT, Celery, Redis

app/services/parser.py

@@ -0,0 +1,211 @@
+"""Parser service for extracting incrementables data from PDF text."""
+import re
+import logging
+from typing import Dict, Optional, List, Tuple
+from decimal import Decimal
+
+logger = logging.getLogger(__name__)
+
+
+class ParsingError(Exception):
+    """Custom exception for parsing errors."""
+    pass
+
+
+class IncrementablesParser:
+    """Parser for extracting incrementables data from PDF text."""
+    
+    # Anchor patterns to find the incrementables section
+    ANCHOR_PATTERNS = [
+        r"AJUSTE\s+DE\s+INCREMENTABLES\s+EN:",
+        r"INCREMENTABLES\s+EN:",
+        r"AJUSTE\s+INCREMENTABLES:",
+    ]
+    
+    # Field patterns
+    FIELD_PATTERNS = {
+        "fletes": r"Fletes[:\s]*\$?\s*([\d,]+\.?\d*)\s*(USD|MXN|EUR)?",
+        "seguros": r"Seguros[:\s]*(?:\$?\s*([\d,]+\.?\d*)\s*)?(USD|MXN|EUR)?",
+        "almacenaje": r"(?:Almacenaje[/\s]*(?:Consolidaci[oó]n)?)[:\s]*\$?\s*([\d,]+\.?\d*)\s*(USD|MXN|EUR)?",
+        "regalias": r"(?:Regal[ií]as?)[:\s]*(?:\$?\s*([\d,]+\.?\d*)\s*)?(USD|MXN|EUR)?",
+    }
+    
+    def __init__(self, text: str):
+        """
+        Initialize parser with PDF text.
+        
+        Args:
+            text: Extracted text from PDF
+        """
+        self.text = text
+        self.warnings: List[str] = []
+        self.anchors_found: List[str] = []
+    
+    def _find_incrementables_section(self) -> Optional[str]:
+        """
+        Find the incrementables section in the text.
+        
+        Returns:
+            Text snippet containing incrementables data, or None if not found
+        """
+        for pattern in self.ANCHOR_PATTERNS:
+            match = re.search(pattern, self.text, re.IGNORECASE | re.MULTILINE)
+            if match:
+                anchor_text = match.group(0)
+                self.anchors_found.append(anchor_text)
+                logger.info(f"Found anchor: {anchor_text}")
+                
+                # Extract the next ~500 characters after the anchor
+                start_pos = match.end()
+                section = self.text[start_pos:start_pos + 500]
+                return section
+        
+        return None
+    
+    def _extract_currency(self, section: str) -> str:
+        """
+        Extract currency from the section.
+        
+        Args:
+            section: Text section to search
+            
+        Returns:
+            Currency code (USD, MXN, EUR) or "USD" as default
+        """
+        currency_pattern = r"\b(USD|MXN|EUR)\b"
+        match = re.search(currency_pattern, section)
+        
+        if match:
+            return match.group(1)
+        
+        # Default to USD but add warning
+        self.warnings.append("Currency not explicitly found, defaulting to USD")
+        return "USD"
+    
+    def _parse_amount(self, value: Optional[str]) -> Optional[float]:
+        """
+        Parse monetary amount from string.
+        
+        Args:
+            value: String containing amount (e.g., "1,591.20" or "$1,591.20")
+            
+        Returns:
+            Float value or None if empty/invalid
+        """
+        if not value or value.strip() == "":
+            return None
+        
+        try:
+            # Remove $ and commas
+            cleaned = value.replace("$", "").replace(",", "").strip()
+            if not cleaned:
+                return None
+            
+            # Convert to Decimal for precision, then to float for JSON
+            amount = float(Decimal(cleaned))
+            return amount
+        except Exception as e:
+            logger.warning(f"Failed to parse amount '{value}': {str(e)}")
+            return None
+    
+    def _extract_field(self, section: str, field_name: str) -> Tuple[Optional[float], Optional[str]]:
+        """
+        Extract a specific field from the section.
+        
+        Args:
+            section: Text section to search
+            field_name: Name of field (fletes, seguros, almacenaje, regalias)
+            
+        Returns:
+            Tuple of (amount, currency) or (None, None)
+        """
+        pattern = self.FIELD_PATTERNS.get(field_name)
+        if not pattern:
+            return None, None
+        
+        match = re.search(pattern, section, re.IGNORECASE | re.MULTILINE)
+        if not match:
+            logger.warning(f"Field '{field_name}' not found in section")
+            return None, None
+        
+        groups = match.groups()
+        
+        # Extract amount (first group)
+        amount_str = groups[0] if len(groups) > 0 else None
+        amount = self._parse_amount(amount_str)
+        
+        # Extract currency (second group)
+        currency = groups[1] if len(groups) > 1 else None
+        
+        return amount, currency
+    
+    def parse(self) -> Dict:
+        """
+        Parse incrementables data from text.
+        
+        Returns:
+            Dictionary with parsed data including:
+            - currency
+            - fletes
+            - seguros (may be None)
+            - almacenaje_consolidacion
+            - regalias (may be None)
+            - warnings
+            - anchors_found
+            
+        Raises:
+            ParsingError: If incrementables section not found or parsing fails
+        """
+        # Find the section
+        section = self._find_incrementables_section()
+        if not section:
+            raise ParsingError(
+                "Incrementables section not found. Expected anchor like 'AJUSTE DE INCREMENTABLES EN:'"
+            )
+        
+        logger.debug(f"Found section: {section[:200]}...")
+        
+        # Extract currency
+        currency = self._extract_currency(section)
+        
+        # Extract fields
+        fletes, _ = self._extract_field(section, "fletes")
+        seguros, _ = self._extract_field(section, "seguros")
+        almacenaje, _ = self._extract_field(section, "almacenaje")
+        regalias, _ = self._extract_field(section, "regalias")
+        
+        # Validate required fields
+        if fletes is None:
+            raise ParsingError("Required field 'fletes' not found or invalid")
+        
+        if almacenaje is None:
+            raise ParsingError("Required field 'almacenaje/consolidacion' not found or invalid")
+        
+        # seguros and regalias can be None (empty)
+        
+        return {
+            "currency": currency,
+            "fletes": fletes,
+            "seguros": seguros,
+            "almacenaje_consolidacion": almacenaje,
+            "regalias": regalias,
+            "warnings": self.warnings,
+            "anchors_found": self.anchors_found,
+        }
+
+
+def parse_incrementables(text: str) -> Dict:
+    """
+    Parse incrementables data from PDF text.
+    
+    Args:
+        text: Extracted text from PDF
+        
+    Returns:
+        Dictionary with parsed incrementables data
+        
+    Raises:
+        ParsingError: If parsing fails
+    """
+    parser = IncrementablesParser(text)
+    return parser.parse()

app/services/pdf_text.py

@@ -0,0 +1,112 @@
+"""PDF text extraction service using PyMuPDF and pdfplumber."""
+import fitz  # PyMuPDF
+import pdfplumber
+import logging
+from typing import Tuple, Optional
+from io import BytesIO
+
+logger = logging.getLogger(__name__)
+
+
+class PDFExtractionError(Exception):
+    """Custom exception for PDF extraction errors."""
+    pass
+
+
+def extract_text_with_pymupdf(pdf_bytes: bytes) -> Tuple[str, int]:
+    """
+    Extract text from PDF using PyMuPDF (fitz).
+    
+    Args:
+        pdf_bytes: PDF file content as bytes
+        
+    Returns:
+        Tuple of (extracted_text, page_count)
+        
+    Raises:
+        PDFExtractionError: If extraction fails
+    """
+    try:
+        doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+        
+        if doc.is_encrypted:
+            raise PDFExtractionError("PDF is encrypted and cannot be read")
+        
+        page_count = len(doc)
+        text_parts = []
+        
+        for page in doc:
+            text_parts.append(page.get_text())
+        
+        doc.close()
+        full_text = "\n".join(text_parts)
+        
+        logger.info(f"Extracted {len(full_text)} characters using PyMuPDF from {page_count} pages")
+        return full_text, page_count
+        
+    except Exception as e:
+        logger.warning(f"PyMuPDF extraction failed: {str(e)}")
+        raise PDFExtractionError(f"PyMuPDF extraction failed: {str(e)}")
+
+
+def extract_text_with_pdfplumber(pdf_bytes: bytes) -> Tuple[str, int]:
+    """
+    Extract text from PDF using pdfplumber (fallback method).
+    
+    Args:
+        pdf_bytes: PDF file content as bytes
+        
+    Returns:
+        Tuple of (extracted_text, page_count)
+        
+    Raises:
+        PDFExtractionError: If extraction fails
+    """
+    try:
+        with pdfplumber.open(BytesIO(pdf_bytes)) as pdf:
+            page_count = len(pdf.pages)
+            text_parts = []
+            
+            for page in pdf.pages:
+                page_text = page.extract_text()
+                if page_text:
+                    text_parts.append(page_text)
+            
+            full_text = "\n".join(text_parts)
+            
+            logger.info(f"Extracted {len(full_text)} characters using pdfplumber from {page_count} pages")
+            return full_text, page_count
+            
+    except Exception as e:
+        logger.error(f"pdfplumber extraction failed: {str(e)}")
+        raise PDFExtractionError(f"pdfplumber extraction failed: {str(e)}")
+
+
+def extract_text_from_pdf(pdf_bytes: bytes) -> Tuple[str, int, str]:
+    """
+    Extract text from PDF using available methods.
+    Tries PyMuPDF first, falls back to pdfplumber.
+    
+    Args:
+        pdf_bytes: PDF file content as bytes
+        
+    Returns:
+        Tuple of (extracted_text, page_count, method_used)
+        
+    Raises:
+        PDFExtractionError: If all extraction methods fail
+    """
+    # Try PyMuPDF first
+    try:
+        text, pages = extract_text_with_pymupdf(pdf_bytes)
+        return text, pages, "text"
+    except PDFExtractionError as e:
+        logger.warning(f"PyMuPDF failed, trying pdfplumber: {str(e)}")
+    
+    # Fallback to pdfplumber
+    try:
+        text, pages = extract_text_with_pdfplumber(pdf_bytes)
+        return text, pages, "text"
+    except PDFExtractionError as e:
+        logger.error(f"All extraction methods failed: {str(e)}")
+        raise PDFExtractionError("Failed to extract text from PDF using all available methods")