feat(pdf): #74 au complet — fix auth 500 stream + endpoint pdf/info + HTTP Range 206 + indexation incrementale PDF + config taille/timeout
This commit is contained in:
+17
-9
@@ -546,19 +546,27 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
|
||||
|
||||
stat = fpath.stat()
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
# PDF handling — binary, must go through pdf_reader (same as _scan_vault)
|
||||
tags: list[str] = []
|
||||
title = fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
if ext == ".pdf":
|
||||
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text
|
||||
raw = extract_pdf_text(fpath, max_chars=SEARCH_CONTENT_LIMIT)
|
||||
pdf_meta = extract_pdf_metadata(fpath)
|
||||
title = pdf_meta.get("title") or title
|
||||
content_preview = raw[:200].strip()
|
||||
else:
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
content_preview = raw[:200].strip()
|
||||
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
tags = _extract_tags(post)
|
||||
inline_tags = _extract_inline_tags(post.content)
|
||||
tags = list(set(tags) | set(inline_tags))
|
||||
title = _extract_title(post, fpath)
|
||||
content_preview = post.content[:200].strip()
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
tags = _extract_tags(post)
|
||||
inline_tags = _extract_inline_tags(post.content)
|
||||
tags = list(set(tags) | set(inline_tags))
|
||||
title = _extract_title(post, fpath)
|
||||
content_preview = post.content[:200].strip()
|
||||
|
||||
return {
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
|
||||
+105
-5
@@ -19,7 +19,7 @@ from typing import Any
|
||||
|
||||
import frontmatter
|
||||
import mistune
|
||||
from fastapi import Body, Depends, FastAPI, HTTPException, Query
|
||||
from fastapi import Body, Depends, FastAPI, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, HTMLResponse, Response, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -2727,8 +2727,17 @@ async def api_file(vault_name: str, path: str = Query(..., description="Relative
|
||||
|
||||
|
||||
@app.get("/api/file/{vault_name}/pdf/stream")
|
||||
async def api_pdf_stream(vault_name: str, path: str = Query(...)):
|
||||
"""Stream a PDF file with Content-Type: application/pdf for inline browser viewing."""
|
||||
async def api_pdf_stream(
|
||||
request: Request,
|
||||
vault_name: str,
|
||||
path: str = Query(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Stream a PDF file with Content-Type: application/pdf for inline browser viewing.
|
||||
|
||||
Supports HTTP Range requests (206 Partial Content) so browsers can
|
||||
progressively render large PDFs in the native viewer.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
@@ -2740,9 +2749,100 @@ async def api_pdf_stream(vault_name: str, path: str = Query(...)):
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
if file_path.suffix.lower() != ".pdf":
|
||||
raise HTTPException(status_code=400, detail="Not a PDF file")
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
file_size = file_path.stat().st_size
|
||||
range_header = request.headers.get("range")
|
||||
|
||||
if range_header:
|
||||
# Parse "bytes=start-end" (single range only; multi-range is not used by viewers)
|
||||
m = re.match(r"bytes=(\d*)-(\d*)", range_header)
|
||||
if not m:
|
||||
raise HTTPException(status_code=416,
|
||||
headers={"Content-Range": f"bytes */{file_size}"})
|
||||
start_s, end_s = m.group(1), m.group(2)
|
||||
if start_s == "" and end_s == "":
|
||||
raise HTTPException(status_code=416,
|
||||
headers={"Content-Range": f"bytes */{file_size}"})
|
||||
if start_s == "":
|
||||
# suffix range: last N bytes
|
||||
length = min(int(end_s), file_size)
|
||||
start = file_size - length
|
||||
end = file_size - 1
|
||||
else:
|
||||
start = int(start_s)
|
||||
end = int(end_s) if end_s else file_size - 1
|
||||
end = min(end, file_size - 1)
|
||||
if start > end or start >= file_size:
|
||||
raise HTTPException(status_code=416,
|
||||
headers={"Content-Range": f"bytes */{file_size}"})
|
||||
|
||||
chunk_size = end - start + 1
|
||||
|
||||
async def _partial():
|
||||
# Open + reads offloaded to threads (avoid blocking the event loop — ASYNC230)
|
||||
f = await asyncio.to_thread(open, str(file_path), "rb")
|
||||
try:
|
||||
await asyncio.to_thread(f.seek, start)
|
||||
remaining = chunk_size
|
||||
while remaining > 0:
|
||||
data = await asyncio.to_thread(f.read, min(64 * 1024, remaining))
|
||||
if not data:
|
||||
break
|
||||
remaining -= len(data)
|
||||
yield data
|
||||
finally:
|
||||
await asyncio.to_thread(f.close)
|
||||
|
||||
return StreamingResponse(
|
||||
_partial(),
|
||||
status_code=206,
|
||||
media_type="application/pdf",
|
||||
headers={
|
||||
"Content-Range": f"bytes {start}-{end}/{file_size}",
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Length": str(chunk_size),
|
||||
"Content-Disposition": f'inline; filename="{file_path.name}"',
|
||||
},
|
||||
)
|
||||
|
||||
return FileResponse(str(file_path), media_type="application/pdf", headers={
|
||||
"Content-Disposition": f"inline; filename=\"{file_path.name}\""})
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Disposition": f'inline; filename="{file_path.name}"'})
|
||||
|
||||
|
||||
@app.get("/api/file/{vault_name}/pdf/info")
|
||||
async def api_pdf_info(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to PDF file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return PDF metadata (pages, title, author, size) without the document content.
|
||||
|
||||
Lets the UI display file info before loading a heavy PDF into the viewer.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = _resolve_safe_path(vault_root, path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
if file_path.suffix.lower() != ".pdf":
|
||||
raise HTTPException(status_code=400, detail="Not a PDF file")
|
||||
|
||||
from backend.pdf_reader import extract_pdf_metadata
|
||||
meta = extract_pdf_metadata(file_path)
|
||||
stat = file_path.stat()
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"pages": meta.get("pages", 0),
|
||||
"title": meta.get("title") or file_path.name,
|
||||
"author": meta.get("author", ""),
|
||||
"size_bytes": stat.st_size,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/search", response_model=SearchResponse)
|
||||
|
||||
+48
-3
@@ -2,10 +2,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeout
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configurable limits (see ROADMAP #74 G3):
|
||||
# OBSIGATE_PDF_MAX_SIZE_MB — PDFs larger than this are not text-extracted (default 50)
|
||||
# OBSIGATE_PDF_EXTRACT_TIMEOUT — seconds before extraction is abandoned (default 30)
|
||||
PDF_MAX_SIZE_MB: int = int(os.environ.get("OBSIGATE_PDF_MAX_SIZE_MB", "50"))
|
||||
PDF_EXTRACT_TIMEOUT: float = float(os.environ.get("OBSIGATE_PDF_EXTRACT_TIMEOUT", "30"))
|
||||
|
||||
PDF_READER: str = "pypdf"
|
||||
PdfReader = None # type: ignore
|
||||
try:
|
||||
@@ -20,15 +29,51 @@ except ImportError:
|
||||
logger.warning("No PDF reader available — install pypdf or pymupdf")
|
||||
|
||||
|
||||
def pdf_exceeds_size_limit(file_path: Path) -> bool:
|
||||
"""True if the PDF is larger than OBSIGATE_PDF_MAX_SIZE_MB (skip text extraction)."""
|
||||
try:
|
||||
return file_path.stat().st_size > PDF_MAX_SIZE_MB * 1024 * 1024
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _run_with_timeout(fn, *args, timeout: float):
|
||||
"""Run a sync function in a worker thread with a hard timeout.
|
||||
|
||||
A timed-out extraction leaves its worker thread running (daemon-style pool
|
||||
is abandoned), but the request itself is freed — acceptable trade-off for
|
||||
pathological PDFs on a self-hosted single-user server.
|
||||
"""
|
||||
executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="pdf-extract")
|
||||
try:
|
||||
future = executor.submit(fn, *args)
|
||||
return future.result(timeout=timeout)
|
||||
finally:
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
|
||||
def extract_pdf_text(file_path: Path, max_chars: int = 100000) -> str:
|
||||
"""Extract text from a PDF file. Returns empty string on failure."""
|
||||
"""Extract text from a PDF file. Returns empty string on failure.
|
||||
|
||||
Oversized PDFs (> OBSIGATE_PDF_MAX_SIZE_MB) and extractions exceeding
|
||||
OBSIGATE_PDF_EXTRACT_TIMEOUT seconds return "" instead of blocking.
|
||||
"""
|
||||
if PdfReader is None and PDF_READER == "pypdf":
|
||||
return ""
|
||||
if pdf_exceeds_size_limit(file_path):
|
||||
logger.info("PDF too large to index (> %d MB), skipping text extraction: %s",
|
||||
PDF_MAX_SIZE_MB, file_path)
|
||||
return ""
|
||||
try:
|
||||
if PDF_READER == "pymupdf":
|
||||
return _extract_pymupdf(file_path, max_chars)
|
||||
return _run_with_timeout(_extract_pymupdf, file_path, max_chars,
|
||||
timeout=PDF_EXTRACT_TIMEOUT)
|
||||
else:
|
||||
return _extract_pypdf(file_path, max_chars)
|
||||
return _run_with_timeout(_extract_pypdf, file_path, max_chars,
|
||||
timeout=PDF_EXTRACT_TIMEOUT)
|
||||
except FuturesTimeout:
|
||||
logger.warning("PDF text extraction timed out (%ss): %s", PDF_EXTRACT_TIMEOUT, file_path)
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.warning("Failed to extract PDF text from %s: %s", file_path, e)
|
||||
return ""
|
||||
|
||||
Reference in New Issue
Block a user