feat(pdf): #74 au complet — fix auth 500 stream + endpoint pdf/info + HTTP Range 206 + indexation incrementale PDF + config taille/timeout

This commit is contained in:
2026-09-07 23:24:13 -04:00
parent f37d5fda3c
commit 7042307955
7 changed files with 320 additions and 17 deletions
+17 -9
View File
@@ -546,19 +546,27 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
stat = fpath.stat()
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
raw = fpath.read_text(encoding="utf-8", errors="replace")
# PDF handling — binary, must go through pdf_reader (same as _scan_vault)
tags: list[str] = []
title = fpath.stem.replace("-", " ").replace("_", " ")
content_preview = raw[:200].strip()
if ext == ".pdf":
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text
raw = extract_pdf_text(fpath, max_chars=SEARCH_CONTENT_LIMIT)
pdf_meta = extract_pdf_metadata(fpath)
title = pdf_meta.get("title") or title
content_preview = raw[:200].strip()
else:
raw = fpath.read_text(encoding="utf-8", errors="replace")
content_preview = raw[:200].strip()
if ext == ".md":
post = parse_markdown_file(raw)
tags = _extract_tags(post)
inline_tags = _extract_inline_tags(post.content)
tags = list(set(tags) | set(inline_tags))
title = _extract_title(post, fpath)
content_preview = post.content[:200].strip()
if ext == ".md":
post = parse_markdown_file(raw)
tags = _extract_tags(post)
inline_tags = _extract_inline_tags(post.content)
tags = list(set(tags) | set(inline_tags))
title = _extract_title(post, fpath)
content_preview = post.content[:200].strip()
return {
"path": str(relative).replace("\\", "/"),
+105 -5
View File
@@ -19,7 +19,7 @@ from typing import Any
import frontmatter
import mistune
from fastapi import Body, Depends, FastAPI, HTTPException, Query
from fastapi import Body, Depends, FastAPI, HTTPException, Query, Request
from fastapi.responses import FileResponse, HTMLResponse, Response, StreamingResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
@@ -2727,8 +2727,17 @@ async def api_file(vault_name: str, path: str = Query(..., description="Relative
@app.get("/api/file/{vault_name}/pdf/stream")
async def api_pdf_stream(vault_name: str, path: str = Query(...)):
"""Stream a PDF file with Content-Type: application/pdf for inline browser viewing."""
async def api_pdf_stream(
request: Request,
vault_name: str,
path: str = Query(...),
current_user=Depends(require_auth),
):
"""Stream a PDF file with Content-Type: application/pdf for inline browser viewing.
Supports HTTP Range requests (206 Partial Content) so browsers can
progressively render large PDFs in the native viewer.
"""
if not check_vault_access(vault_name, current_user):
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
vault_data = get_vault_data(vault_name)
@@ -2740,9 +2749,100 @@ async def api_pdf_stream(vault_name: str, path: str = Query(...)):
raise HTTPException(status_code=404, detail=f"File not found: {path}")
if file_path.suffix.lower() != ".pdf":
raise HTTPException(status_code=400, detail="Not a PDF file")
from fastapi.responses import FileResponse
file_size = file_path.stat().st_size
range_header = request.headers.get("range")
if range_header:
# Parse "bytes=start-end" (single range only; multi-range is not used by viewers)
m = re.match(r"bytes=(\d*)-(\d*)", range_header)
if not m:
raise HTTPException(status_code=416,
headers={"Content-Range": f"bytes */{file_size}"})
start_s, end_s = m.group(1), m.group(2)
if start_s == "" and end_s == "":
raise HTTPException(status_code=416,
headers={"Content-Range": f"bytes */{file_size}"})
if start_s == "":
# suffix range: last N bytes
length = min(int(end_s), file_size)
start = file_size - length
end = file_size - 1
else:
start = int(start_s)
end = int(end_s) if end_s else file_size - 1
end = min(end, file_size - 1)
if start > end or start >= file_size:
raise HTTPException(status_code=416,
headers={"Content-Range": f"bytes */{file_size}"})
chunk_size = end - start + 1
async def _partial():
# Open + reads offloaded to threads (avoid blocking the event loop — ASYNC230)
f = await asyncio.to_thread(open, str(file_path), "rb")
try:
await asyncio.to_thread(f.seek, start)
remaining = chunk_size
while remaining > 0:
data = await asyncio.to_thread(f.read, min(64 * 1024, remaining))
if not data:
break
remaining -= len(data)
yield data
finally:
await asyncio.to_thread(f.close)
return StreamingResponse(
_partial(),
status_code=206,
media_type="application/pdf",
headers={
"Content-Range": f"bytes {start}-{end}/{file_size}",
"Accept-Ranges": "bytes",
"Content-Length": str(chunk_size),
"Content-Disposition": f'inline; filename="{file_path.name}"',
},
)
return FileResponse(str(file_path), media_type="application/pdf", headers={
"Content-Disposition": f"inline; filename=\"{file_path.name}\""})
"Accept-Ranges": "bytes",
"Content-Disposition": f'inline; filename="{file_path.name}"'})
@app.get("/api/file/{vault_name}/pdf/info")
async def api_pdf_info(
vault_name: str,
path: str = Query(..., description="Relative path to PDF file"),
current_user=Depends(require_auth),
):
"""Return PDF metadata (pages, title, author, size) without the document content.
Lets the UI display file info before loading a heavy PDF into the viewer.
"""
if not check_vault_access(vault_name, current_user):
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
vault_data = get_vault_data(vault_name)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
vault_root = Path(vault_data["path"])
file_path = _resolve_safe_path(vault_root, path)
if not file_path.exists() or not file_path.is_file():
raise HTTPException(status_code=404, detail=f"File not found: {path}")
if file_path.suffix.lower() != ".pdf":
raise HTTPException(status_code=400, detail="Not a PDF file")
from backend.pdf_reader import extract_pdf_metadata
meta = extract_pdf_metadata(file_path)
stat = file_path.stat()
return {
"vault": vault_name,
"path": path,
"pages": meta.get("pages", 0),
"title": meta.get("title") or file_path.name,
"author": meta.get("author", ""),
"size_bytes": stat.st_size,
}
@app.get("/api/search", response_model=SearchResponse)
+48 -3
View File
@@ -2,10 +2,19 @@
from __future__ import annotations
import logging
import os
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from pathlib import Path
logger = logging.getLogger(__name__)
# Configurable limits (see ROADMAP #74 G3):
# OBSIGATE_PDF_MAX_SIZE_MB — PDFs larger than this are not text-extracted (default 50)
# OBSIGATE_PDF_EXTRACT_TIMEOUT — seconds before extraction is abandoned (default 30)
PDF_MAX_SIZE_MB: int = int(os.environ.get("OBSIGATE_PDF_MAX_SIZE_MB", "50"))
PDF_EXTRACT_TIMEOUT: float = float(os.environ.get("OBSIGATE_PDF_EXTRACT_TIMEOUT", "30"))
PDF_READER: str = "pypdf"
PdfReader = None # type: ignore
try:
@@ -20,15 +29,51 @@ except ImportError:
logger.warning("No PDF reader available — install pypdf or pymupdf")
def pdf_exceeds_size_limit(file_path: Path) -> bool:
"""True if the PDF is larger than OBSIGATE_PDF_MAX_SIZE_MB (skip text extraction)."""
try:
return file_path.stat().st_size > PDF_MAX_SIZE_MB * 1024 * 1024
except OSError:
return False
def _run_with_timeout(fn, *args, timeout: float):
"""Run a sync function in a worker thread with a hard timeout.
A timed-out extraction leaves its worker thread running (daemon-style pool
is abandoned), but the request itself is freed — acceptable trade-off for
pathological PDFs on a self-hosted single-user server.
"""
executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="pdf-extract")
try:
future = executor.submit(fn, *args)
return future.result(timeout=timeout)
finally:
executor.shutdown(wait=False)
def extract_pdf_text(file_path: Path, max_chars: int = 100000) -> str:
"""Extract text from a PDF file. Returns empty string on failure."""
"""Extract text from a PDF file. Returns empty string on failure.
Oversized PDFs (> OBSIGATE_PDF_MAX_SIZE_MB) and extractions exceeding
OBSIGATE_PDF_EXTRACT_TIMEOUT seconds return "" instead of blocking.
"""
if PdfReader is None and PDF_READER == "pypdf":
return ""
if pdf_exceeds_size_limit(file_path):
logger.info("PDF too large to index (> %d MB), skipping text extraction: %s",
PDF_MAX_SIZE_MB, file_path)
return ""
try:
if PDF_READER == "pymupdf":
return _extract_pymupdf(file_path, max_chars)
return _run_with_timeout(_extract_pymupdf, file_path, max_chars,
timeout=PDF_EXTRACT_TIMEOUT)
else:
return _extract_pypdf(file_path, max_chars)
return _run_with_timeout(_extract_pypdf, file_path, max_chars,
timeout=PDF_EXTRACT_TIMEOUT)
except FuturesTimeout:
logger.warning("PDF text extraction timed out (%ss): %s", PDF_EXTRACT_TIMEOUT, file_path)
return ""
except Exception as e:
logger.warning("Failed to extract PDF text from %s: %s", file_path, e)
return ""