feat(pdf): #74 au complet — fix auth 500 stream + endpoint pdf/info + HTTP Range 206 + indexation incrementale PDF + config taille/timeout
This commit is contained in:
+48
-3
@@ -2,10 +2,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeout
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configurable limits (see ROADMAP #74 G3):
|
||||
# OBSIGATE_PDF_MAX_SIZE_MB — PDFs larger than this are not text-extracted (default 50)
|
||||
# OBSIGATE_PDF_EXTRACT_TIMEOUT — seconds before extraction is abandoned (default 30)
|
||||
PDF_MAX_SIZE_MB: int = int(os.environ.get("OBSIGATE_PDF_MAX_SIZE_MB", "50"))
|
||||
PDF_EXTRACT_TIMEOUT: float = float(os.environ.get("OBSIGATE_PDF_EXTRACT_TIMEOUT", "30"))
|
||||
|
||||
PDF_READER: str = "pypdf"
|
||||
PdfReader = None # type: ignore
|
||||
try:
|
||||
@@ -20,15 +29,51 @@ except ImportError:
|
||||
logger.warning("No PDF reader available — install pypdf or pymupdf")
|
||||
|
||||
|
||||
def pdf_exceeds_size_limit(file_path: Path) -> bool:
|
||||
"""True if the PDF is larger than OBSIGATE_PDF_MAX_SIZE_MB (skip text extraction)."""
|
||||
try:
|
||||
return file_path.stat().st_size > PDF_MAX_SIZE_MB * 1024 * 1024
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _run_with_timeout(fn, *args, timeout: float):
|
||||
"""Run a sync function in a worker thread with a hard timeout.
|
||||
|
||||
A timed-out extraction leaves its worker thread running (daemon-style pool
|
||||
is abandoned), but the request itself is freed — acceptable trade-off for
|
||||
pathological PDFs on a self-hosted single-user server.
|
||||
"""
|
||||
executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="pdf-extract")
|
||||
try:
|
||||
future = executor.submit(fn, *args)
|
||||
return future.result(timeout=timeout)
|
||||
finally:
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
|
||||
def extract_pdf_text(file_path: Path, max_chars: int = 100000) -> str:
|
||||
"""Extract text from a PDF file. Returns empty string on failure."""
|
||||
"""Extract text from a PDF file. Returns empty string on failure.
|
||||
|
||||
Oversized PDFs (> OBSIGATE_PDF_MAX_SIZE_MB) and extractions exceeding
|
||||
OBSIGATE_PDF_EXTRACT_TIMEOUT seconds return "" instead of blocking.
|
||||
"""
|
||||
if PdfReader is None and PDF_READER == "pypdf":
|
||||
return ""
|
||||
if pdf_exceeds_size_limit(file_path):
|
||||
logger.info("PDF too large to index (> %d MB), skipping text extraction: %s",
|
||||
PDF_MAX_SIZE_MB, file_path)
|
||||
return ""
|
||||
try:
|
||||
if PDF_READER == "pymupdf":
|
||||
return _extract_pymupdf(file_path, max_chars)
|
||||
return _run_with_timeout(_extract_pymupdf, file_path, max_chars,
|
||||
timeout=PDF_EXTRACT_TIMEOUT)
|
||||
else:
|
||||
return _extract_pypdf(file_path, max_chars)
|
||||
return _run_with_timeout(_extract_pypdf, file_path, max_chars,
|
||||
timeout=PDF_EXTRACT_TIMEOUT)
|
||||
except FuturesTimeout:
|
||||
logger.warning("PDF text extraction timed out (%ss): %s", PDF_EXTRACT_TIMEOUT, file_path)
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.warning("Failed to extract PDF text from %s: %s", file_path, e)
|
||||
return ""
|
||||
|
||||
Reference in New Issue
Block a user