feat: Agregar soporte OCR con Tesseract para PDFs escaneados
- Integrar Tesseract OCR para leer PDFs escaneados automáticamente - Detectar automáticamente si el PDF tiene texto o requiere OCR - Agregar servicio ocr_service.py con funciones de OCR - Actualizar Dockerfile con tesseract-ocr, tesseract-ocr-spa y poppler-utils - Agregar variables de configuración OCR (OCR_ENABLED, OCR_LANGUAGE, OCR_DPI, OCR_TIMEOUT) - Crear endpoint de debug para ver texto extraído (/api/v1/debug/extract-text) - Agregar scripts de instalación y prueba (install_ocr.ps1, test_ocr.py, debug_pdf.ps1) - Documentación completa (OCR_SETUP.md, DOCKER_OCR.md, COMO_PROBAR.md) - Actualizar docker-compose.yml con variables de entorno OCR - Modificar pdf_text.py para usar OCR cuando sea necesario - Actualizar requirements.txt con pytesseract, Pillow, pdf2image
This commit is contained in:
Ernesto Herrera
87
app/api/v1/debug.py
Normal file
87
app/api/v1/debug.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Debug endpoint to see extracted text from PDF."""
|
||||
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
import logging
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.security import decode_access_token
|
||||
from app.services.pdf_text import extract_text_from_pdf, PDFExtractionError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/v1/debug", tags=["Debug"])
|
||||
security = HTTPBearer()
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security)) -> str:
|
||||
"""Validate JWT token."""
|
||||
payload = decode_access_token(credentials.credentials)
|
||||
return payload.get("sub")
|
||||
|
||||
|
||||
@router.post("/extract-text")
|
||||
async def debug_extract_text(
|
||||
file: UploadFile = File(...),
|
||||
current_user: str = Depends(get_current_user)
|
||||
):
|
||||
"""
|
||||
Debug endpoint to see raw extracted text from PDF.
|
||||
Shows exactly what text was extracted and which method was used.
|
||||
"""
|
||||
# Validate file type
|
||||
if file.content_type != "application/pdf":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid file type: {file.content_type}. Only PDF files are allowed."
|
||||
)
|
||||
|
||||
# Read file
|
||||
pdf_bytes = await file.read()
|
||||
|
||||
# Validate size
|
||||
file_size_mb = len(pdf_bytes) / (1024 * 1024)
|
||||
if file_size_mb > settings.max_file_mb:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"File size ({file_size_mb:.2f} MB) exceeds maximum allowed size of {settings.max_file_mb} MB"
|
||||
)
|
||||
|
||||
# Extract text
|
||||
try:
|
||||
text, page_count, extraction_method = extract_text_from_pdf(
|
||||
pdf_bytes,
|
||||
enable_ocr=settings.ocr_enabled,
|
||||
ocr_lang=settings.ocr_language
|
||||
)
|
||||
|
||||
# Get first and last 500 characters
|
||||
text_preview_start = text[:500] if len(text) > 500 else text
|
||||
text_preview_end = text[-500:] if len(text) > 500 else ""
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"extraction_info": {
|
||||
"method": extraction_method,
|
||||
"pages": page_count,
|
||||
"total_characters": len(text),
|
||||
"total_words": len(text.split()),
|
||||
"total_lines": len(text.split('\n'))
|
||||
},
|
||||
"text_preview": {
|
||||
"first_500_chars": text_preview_start,
|
||||
"last_500_chars": text_preview_end if text_preview_end else None
|
||||
},
|
||||
"full_text": text, # Complete extracted text
|
||||
"ocr_settings": {
|
||||
"ocr_enabled": settings.ocr_enabled,
|
||||
"ocr_language": settings.ocr_language,
|
||||
"ocr_dpi": settings.ocr_dpi
|
||||
}
|
||||
}
|
||||
|
||||
except PDFExtractionError as e:
|
||||
logger.error(f"Extraction failed: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to extract text from PDF: {str(e)}"
|
||||
)
|
||||
@@ -117,9 +117,13 @@ async def parse_pdf(
|
||||
|
||||
# Extract text from PDF
|
||||
try:
|
||||
text, page_count, extraction_method = extract_text_from_pdf(pdf_bytes)
|
||||
text, page_count, extraction_method = extract_text_from_pdf(
|
||||
pdf_bytes,
|
||||
enable_ocr=settings.ocr_enabled,
|
||||
ocr_lang=settings.ocr_language
|
||||
)
|
||||
logger.info(
|
||||
f"Text extracted: {len(text)} characters, {page_count} pages",
|
||||
f"Text extracted: {len(text)} characters, {page_count} pages, method: {extraction_method}",
|
||||
extra={"correlation_id": correlation_id}
|
||||
)
|
||||
except PDFExtractionError as e:
|
||||
|
||||
@@ -20,6 +20,12 @@ class Settings(BaseSettings):
|
||||
# File Upload
|
||||
max_file_mb: int = 10
|
||||
|
||||
# OCR Settings
|
||||
ocr_enabled: bool = True
|
||||
ocr_language: str = "spa" # "spa" for Spanish, "eng" for English
|
||||
ocr_dpi: int = 300 # Higher = better quality but slower
|
||||
ocr_timeout: int = 300 # Maximum time in seconds for OCR
|
||||
|
||||
# Logging
|
||||
log_level: str = "INFO"
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from contextlib import asynccontextmanager
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.api import auth
|
||||
from app.api.v1 import incrementables
|
||||
from app.api.v1 import incrementables, debug
|
||||
from app.schemas import HealthResponse
|
||||
|
||||
# Configure logging
|
||||
@@ -83,6 +83,7 @@ async def health_check():
|
||||
# Include routers
|
||||
app.include_router(auth.router)
|
||||
app.include_router(incrementables.router)
|
||||
app.include_router(debug.router)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
197
app/services/ocr_service.py
Normal file
197
app/services/ocr_service.py
Normal file
@@ -0,0 +1,197 @@
|
||||
"""OCR service using Tesseract for scanned PDFs."""
|
||||
import pytesseract
|
||||
from pdf2image import convert_from_bytes
|
||||
from PIL import Image
|
||||
import logging
|
||||
from typing import Tuple, List, Optional
|
||||
from io import BytesIO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OCRError(Exception):
|
||||
"""Custom exception for OCR processing errors."""
|
||||
pass
|
||||
|
||||
|
||||
def is_pdf_scanned(text: str, min_char_threshold: int = 50) -> bool:
|
||||
"""
|
||||
Determine if a PDF is scanned (no selectable text) or has text.
|
||||
|
||||
Args:
|
||||
text: Extracted text from PDF
|
||||
min_char_threshold: Minimum characters to consider as "has text"
|
||||
|
||||
Returns:
|
||||
True if PDF appears to be scanned (no text), False otherwise
|
||||
"""
|
||||
# Remove whitespace and count actual characters
|
||||
cleaned_text = text.replace(" ", "").replace("\n", "").replace("\t", "")
|
||||
|
||||
if len(cleaned_text) < min_char_threshold:
|
||||
logger.info(f"PDF appears to be scanned (only {len(cleaned_text)} characters found)")
|
||||
return True
|
||||
|
||||
logger.info(f"PDF has selectable text ({len(cleaned_text)} characters)")
|
||||
return False
|
||||
|
||||
|
||||
def extract_text_with_ocr(
|
||||
pdf_bytes: bytes,
|
||||
lang: str = "spa",
|
||||
dpi: int = 300,
|
||||
timeout: int = 300
|
||||
) -> Tuple[str, int]:
|
||||
"""
|
||||
Extract text from scanned PDF using Tesseract OCR.
|
||||
|
||||
Args:
|
||||
pdf_bytes: PDF file content as bytes
|
||||
lang: Language for OCR (default: "spa" for Spanish, use "eng" for English)
|
||||
dpi: DPI for image conversion (higher = better quality but slower)
|
||||
timeout: Maximum time in seconds for OCR processing
|
||||
|
||||
Returns:
|
||||
Tuple of (extracted_text, page_count)
|
||||
|
||||
Raises:
|
||||
OCRError: If OCR processing fails
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Starting OCR extraction with language={lang}, dpi={dpi}")
|
||||
|
||||
# Convert PDF to images
|
||||
images = convert_from_bytes(
|
||||
pdf_bytes,
|
||||
dpi=dpi,
|
||||
fmt='png',
|
||||
thread_count=2,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
page_count = len(images)
|
||||
logger.info(f"Converted PDF to {page_count} images")
|
||||
|
||||
# Extract text from each page
|
||||
text_parts = []
|
||||
for i, image in enumerate(images, 1):
|
||||
try:
|
||||
# Perform OCR on the image
|
||||
page_text = pytesseract.image_to_string(
|
||||
image,
|
||||
lang=lang,
|
||||
timeout=timeout // page_count # Distribute timeout across pages
|
||||
)
|
||||
|
||||
if page_text.strip():
|
||||
text_parts.append(f"--- Página {i} ---\n{page_text}")
|
||||
logger.debug(f"Page {i}: Extracted {len(page_text)} characters")
|
||||
else:
|
||||
logger.warning(f"Page {i}: No text extracted")
|
||||
text_parts.append(f"--- Página {i} ---\n[Sin texto detectado]\n")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OCR failed on page {i}: {str(e)}")
|
||||
text_parts.append(f"--- Página {i} ---\n[Error en OCR: {str(e)}]\n")
|
||||
|
||||
full_text = "\n\n".join(text_parts)
|
||||
|
||||
logger.info(f"OCR extraction completed: {len(full_text)} characters from {page_count} pages")
|
||||
return full_text, page_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OCR extraction failed: {str(e)}")
|
||||
raise OCRError(f"OCR extraction failed: {str(e)}")
|
||||
|
||||
|
||||
def optimize_image_for_ocr(image: Image.Image) -> Image.Image:
|
||||
"""
|
||||
Optimize image for better OCR results.
|
||||
|
||||
Args:
|
||||
image: PIL Image object
|
||||
|
||||
Returns:
|
||||
Optimized PIL Image
|
||||
"""
|
||||
# Convert to grayscale
|
||||
image = image.convert('L')
|
||||
|
||||
# Optional: Apply threshold to make text clearer
|
||||
# This can be adjusted based on your PDFs
|
||||
# from PIL import ImageEnhance
|
||||
# enhancer = ImageEnhance.Contrast(image)
|
||||
# image = enhancer.enhance(2)
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def extract_text_with_ocr_optimized(
|
||||
pdf_bytes: bytes,
|
||||
lang: str = "spa",
|
||||
dpi: int = 300,
|
||||
timeout: int = 300,
|
||||
optimize: bool = True
|
||||
) -> Tuple[str, int]:
|
||||
"""
|
||||
Extract text from scanned PDF with image optimization.
|
||||
|
||||
Args:
|
||||
pdf_bytes: PDF file content as bytes
|
||||
lang: Language for OCR
|
||||
dpi: DPI for image conversion
|
||||
timeout: Maximum time in seconds
|
||||
optimize: Whether to optimize images before OCR
|
||||
|
||||
Returns:
|
||||
Tuple of (extracted_text, page_count)
|
||||
|
||||
Raises:
|
||||
OCRError: If OCR processing fails
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Starting optimized OCR extraction")
|
||||
|
||||
# Convert PDF to images
|
||||
images = convert_from_bytes(
|
||||
pdf_bytes,
|
||||
dpi=dpi,
|
||||
fmt='png',
|
||||
thread_count=2,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
page_count = len(images)
|
||||
text_parts = []
|
||||
|
||||
for i, image in enumerate(images, 1):
|
||||
try:
|
||||
# Optimize image if requested
|
||||
if optimize:
|
||||
image = optimize_image_for_ocr(image)
|
||||
|
||||
# Perform OCR
|
||||
page_text = pytesseract.image_to_string(
|
||||
image,
|
||||
lang=lang,
|
||||
config='--psm 1', # Automatic page segmentation with OSD
|
||||
timeout=timeout // page_count
|
||||
)
|
||||
|
||||
if page_text.strip():
|
||||
text_parts.append(f"--- Página {i} ---\n{page_text}")
|
||||
else:
|
||||
text_parts.append(f"--- Página {i} ---\n[Sin texto detectado]\n")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OCR failed on page {i}: {str(e)}")
|
||||
text_parts.append(f"--- Página {i} ---\n[Error en OCR]\n")
|
||||
|
||||
full_text = "\n\n".join(text_parts)
|
||||
|
||||
logger.info(f"Optimized OCR completed: {len(full_text)} characters")
|
||||
return full_text, page_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Optimized OCR failed: {str(e)}")
|
||||
raise OCRError(f"Optimized OCR failed: {str(e)}")
|
||||
@@ -1,9 +1,10 @@
|
||||
"""PDF text extraction service using PyMuPDF and pdfplumber."""
|
||||
"""PDF text extraction service using PyMuPDF, pdfplumber, and OCR."""
|
||||
import fitz # PyMuPDF
|
||||
import pdfplumber
|
||||
import logging
|
||||
from typing import Tuple, Optional
|
||||
from io import BytesIO
|
||||
from .ocr_service import extract_text_with_ocr, is_pdf_scanned, OCRError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -82,16 +83,19 @@ def extract_text_with_pdfplumber(pdf_bytes: bytes) -> Tuple[str, int]:
|
||||
raise PDFExtractionError(f"pdfplumber extraction failed: {str(e)}")
|
||||
|
||||
|
||||
def extract_text_from_pdf(pdf_bytes: bytes) -> Tuple[str, int, str]:
|
||||
def extract_text_from_pdf(pdf_bytes: bytes, enable_ocr: bool = True, ocr_lang: str = "spa") -> Tuple[str, int, str]:
|
||||
"""
|
||||
Extract text from PDF using available methods.
|
||||
Tries PyMuPDF first, falls back to pdfplumber.
|
||||
Tries PyMuPDF first, falls back to pdfplumber, and uses OCR for scanned PDFs.
|
||||
|
||||
Args:
|
||||
pdf_bytes: PDF file content as bytes
|
||||
enable_ocr: Whether to use OCR for scanned PDFs (default: True)
|
||||
ocr_lang: Language for OCR - "spa" for Spanish, "eng" for English (default: "spa")
|
||||
|
||||
Returns:
|
||||
Tuple of (extracted_text, page_count, method_used)
|
||||
method_used can be: "pymupdf", "pdfplumber", "ocr"
|
||||
|
||||
Raises:
|
||||
PDFExtractionError: If all extraction methods fail
|
||||
@@ -99,14 +103,41 @@ def extract_text_from_pdf(pdf_bytes: bytes) -> Tuple[str, int, str]:
|
||||
# Try PyMuPDF first
|
||||
try:
|
||||
text, pages = extract_text_with_pymupdf(pdf_bytes)
|
||||
return text, pages, "text"
|
||||
|
||||
# Check if the PDF is scanned (no text content)
|
||||
if enable_ocr and is_pdf_scanned(text):
|
||||
logger.info("PDF appears to be scanned, attempting OCR extraction")
|
||||
try:
|
||||
ocr_text, ocr_pages = extract_text_with_ocr(pdf_bytes, lang=ocr_lang)
|
||||
return ocr_text, ocr_pages, "ocr"
|
||||
except OCRError as ocr_e:
|
||||
logger.error(f"OCR failed: {str(ocr_e)}")
|
||||
# Return the minimal text we got, if any
|
||||
if text.strip():
|
||||
return text, pages, "pymupdf"
|
||||
raise PDFExtractionError(f"PDF appears to be scanned and OCR failed: {str(ocr_e)}")
|
||||
|
||||
return text, pages, "pymupdf"
|
||||
except PDFExtractionError as e:
|
||||
logger.warning(f"PyMuPDF failed, trying pdfplumber: {str(e)}")
|
||||
|
||||
# Fallback to pdfplumber
|
||||
try:
|
||||
text, pages = extract_text_with_pdfplumber(pdf_bytes)
|
||||
return text, pages, "text"
|
||||
|
||||
# Check if the PDF is scanned
|
||||
if enable_ocr and is_pdf_scanned(text):
|
||||
logger.info("PDF appears to be scanned (pdfplumber), attempting OCR extraction")
|
||||
try:
|
||||
ocr_text, ocr_pages = extract_text_with_ocr(pdf_bytes, lang=ocr_lang)
|
||||
return ocr_text, ocr_pages, "ocr"
|
||||
except OCRError as ocr_e:
|
||||
logger.error(f"OCR failed: {str(ocr_e)}")
|
||||
if text.strip():
|
||||
return text, pages, "pdfplumber"
|
||||
raise PDFExtractionError(f"PDF appears to be scanned and OCR failed: {str(ocr_e)}")
|
||||
|
||||
return text, pages, "pdfplumber"
|
||||
except PDFExtractionError as e:
|
||||
logger.error(f"All extraction methods failed: {str(e)}")
|
||||
raise PDFExtractionError("Failed to extract text from PDF using all available methods")
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import logging
|
||||
import hashlib
|
||||
from app.core.celery_app import celery_app
|
||||
from app.core.config import get_settings
|
||||
from app.services.pdf_text import extract_text_from_pdf, PDFExtractionError
|
||||
from app.services.parser import parse_incrementables, ParsingError
|
||||
|
||||
@@ -33,9 +34,14 @@ def parse_pdf_task(self, pdf_bytes_hex: str, filename: str, document_ref: str =
|
||||
file_hash = hashlib.sha256(pdf_bytes).hexdigest()
|
||||
|
||||
# Extract text
|
||||
settings = get_settings()
|
||||
try:
|
||||
text, page_count, extraction_method = extract_text_from_pdf(pdf_bytes)
|
||||
logger.info(f"Task {task_id}: Extracted {len(text)} chars from {page_count} pages")
|
||||
text, page_count, extraction_method = extract_text_from_pdf(
|
||||
pdf_bytes,
|
||||
enable_ocr=settings.ocr_enabled,
|
||||
ocr_lang=settings.ocr_language
|
||||
)
|
||||
logger.info(f"Task {task_id}: Extracted {len(text)} chars from {page_count} pages using {extraction_method}")
|
||||
except PDFExtractionError as e:
|
||||
logger.error(f"Task {task_id}: Extraction failed - {str(e)}")
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user