Add comprehensive test suite for image processing and related services
- Implement tests for database generator to ensure proper session handling. - Create tests for EXIF extraction and conversion functions. - Add tests for image-related endpoints, ensuring proper data retrieval and isolation between clients. - Develop tests for OCR functionality, including language detection and text extraction. - Introduce tests for the image processing pipeline, covering success and failure scenarios. - Validate rate limiting functionality and ensure independent counters for different clients. - Implement scraper tests to verify HTML content fetching and error handling. - Add unit tests for various services, including storage and filename generation. - Establish worker entry point for ARQ to handle background image processing tasks.
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
from app.services import storage, exif_service, ocr_service, ai_vision, scraper, pipeline
|
||||
|
||||
__all__ = ["storage", "exif_service", "ocr_service", "ai_vision", "scraper", "pipeline"]
|
||||
@@ -0,0 +1,417 @@
|
||||
"""
|
||||
Service AI Vision — description, classification et tags via Google Gemini ou OpenRouter
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import base64
|
||||
import httpx
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_client: Optional[genai.Client] = None
|
||||
|
||||
|
||||
def _get_client() -> genai.Client:
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = genai.Client(api_key=settings.GEMINI_API_KEY)
|
||||
return _client
|
||||
|
||||
|
||||
def _read_image(file_path: str) -> tuple[bytes, str]:
|
||||
"""Lit l'image en bytes et détecte le media_type."""
|
||||
path = Path(file_path)
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
mime_map = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
}
|
||||
media_type = mime_map.get(suffix, "image/jpeg")
|
||||
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
|
||||
return data, media_type
|
||||
|
||||
|
||||
def _extract_json(text: str) -> Optional[dict]:
|
||||
cleaned = re.sub(r"```json\s*|```\s*", "", (text or "")).strip()
|
||||
json_match = re.search(r"\{.*\}", cleaned, re.DOTALL)
|
||||
if not json_match:
|
||||
return None
|
||||
try:
|
||||
return json.loads(json_match.group())
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _usage_tokens_gemini(response) -> tuple[Optional[int], Optional[int]]:
|
||||
usage = getattr(response, "usage_metadata", None)
|
||||
if not usage:
|
||||
return None, None
|
||||
prompt_tokens = getattr(usage, "prompt_token_count", None)
|
||||
output_tokens = getattr(usage, "candidates_token_count", None)
|
||||
return prompt_tokens, output_tokens
|
||||
|
||||
|
||||
async def _generate_gemini(
|
||||
prompt: str,
|
||||
image_bytes: Optional[bytes] = None,
|
||||
media_type: Optional[str] = None,
|
||||
max_tokens: int = 1024
|
||||
) -> dict:
|
||||
"""Appel à Google Gemini via SDK."""
|
||||
if not settings.GEMINI_API_KEY:
|
||||
logger.warning("ai.gemini.no_key")
|
||||
return {"text": None, "usage": (None, None)}
|
||||
|
||||
client = _get_client()
|
||||
contents = []
|
||||
if image_bytes and media_type:
|
||||
contents.append(types.Part.from_bytes(data=image_bytes, mime_type=media_type))
|
||||
contents.append(prompt)
|
||||
|
||||
try:
|
||||
# Le SDK est sync, on le run dans un thread
|
||||
response = await asyncio.to_thread(
|
||||
client.models.generate_content,
|
||||
model=settings.GEMINI_MODEL,
|
||||
contents=contents,
|
||||
config=types.GenerateContentConfig(
|
||||
max_output_tokens=max_tokens,
|
||||
response_mime_type="application/json",
|
||||
),
|
||||
)
|
||||
usage = _usage_tokens_gemini(response)
|
||||
return {"text": getattr(response, "text", ""), "usage": usage}
|
||||
except Exception as e:
|
||||
logger.error("ai.gemini.error", extra={"error": str(e)})
|
||||
return {"text": None, "usage": (None, None), "error": str(e)}
|
||||
|
||||
|
||||
async def _generate_openrouter(
|
||||
prompt: str,
|
||||
image_bytes: Optional[bytes] = None,
|
||||
media_type: Optional[str] = None,
|
||||
max_tokens: int = 1024
|
||||
) -> dict:
|
||||
"""Appel à OpenRouter via HTTP."""
|
||||
if not settings.OPENROUTER_API_KEY:
|
||||
logger.warning("ai.openrouter.no_key")
|
||||
return {"text": None, "usage": (None, None)}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {settings.OPENROUTER_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
"HTTP-Referer": settings.HOST,
|
||||
"X-Title": settings.APP_NAME,
|
||||
}
|
||||
|
||||
messages = []
|
||||
content_payload = []
|
||||
|
||||
content_payload.append({"type": "text", "text": prompt})
|
||||
|
||||
if image_bytes and media_type:
|
||||
b64_img = base64.b64encode(image_bytes).decode("utf-8")
|
||||
content_payload.append({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:{media_type};base64,{b64_img}"
|
||||
}
|
||||
})
|
||||
|
||||
messages.append({"role": "user", "content": content_payload})
|
||||
|
||||
payload = {
|
||||
"model": settings.OPENROUTER_MODEL,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
# OpenRouter/OpenAI support response_format={"type": "json_object"} pour certains modèles
|
||||
# On tente le coup si le modèle est compatible, sinon le prompt engineering fait le travail
|
||||
"response_format": {"type": "json_object"}
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
response = await client.post(
|
||||
"https://openrouter.ai/api/v1/chat/completions",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=60.0
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
text = ""
|
||||
if "choices" in data and len(data["choices"]) > 0:
|
||||
text = data["choices"][0]["message"]["content"]
|
||||
|
||||
usage_data = data.get("usage", {})
|
||||
prompt_tokens = usage_data.get("prompt_tokens")
|
||||
output_tokens = usage_data.get("completion_tokens")
|
||||
|
||||
return {"text": text, "usage": (prompt_tokens, output_tokens)}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("ai.openrouter.error", extra={"error": str(e)})
|
||||
return {"text": None, "usage": (None, None), "error": str(e)}
|
||||
|
||||
|
||||
async def _generate(
|
||||
prompt: str,
|
||||
image_bytes: Optional[bytes] = None,
|
||||
media_type: Optional[str] = None,
|
||||
max_tokens: int = 1024
|
||||
) -> dict:
|
||||
"""Dispatcher vers le bon provider."""
|
||||
provider = settings.AI_PROVIDER.lower()
|
||||
logger.info("ai.generate", extra={"provider": provider})
|
||||
|
||||
if provider == "openrouter":
|
||||
return await _generate_openrouter(prompt, image_bytes, media_type, max_tokens)
|
||||
else:
|
||||
# Default to Gemini
|
||||
return await _generate_gemini(prompt, image_bytes, media_type, max_tokens)
|
||||
|
||||
|
||||
def _build_prompt(ocr_hint: Optional[str], language: str) -> str:
|
||||
ocr_section = ""
|
||||
if ocr_hint and len(ocr_hint.strip()) > 5:
|
||||
ocr_section = f"""
|
||||
Texte détecté dans l'image par OCR (utilise-le pour enrichir ta réponse) :
|
||||
\"\"\"
|
||||
{ocr_hint[:500]}
|
||||
\"\"\"
|
||||
"""
|
||||
|
||||
return f"""Analyse cette image avec précision et retourne UNIQUEMENT un objet JSON valide avec ces champs :
|
||||
|
||||
{{
|
||||
"description": "Description complète et détaillée en {language}, 2-4 phrases. Décris le sujet principal, le contexte, les couleurs, l'ambiance.",
|
||||
"tags": ["tag1", "tag2", "tag3"],
|
||||
"confidence": 0.95
|
||||
}}
|
||||
|
||||
Règles pour les tags :
|
||||
- Entre {settings.AI_TAGS_MIN} et {settings.AI_TAGS_MAX} tags
|
||||
- En minuscules, sans espaces (utiliser des tirets si nécessaire)
|
||||
- Couvrir : sujet principal, type d'image, couleurs dominantes, style, contexte
|
||||
- Exemples : portrait, paysage, architecture, nature, nourriture, texte, document, animal, sport, technologie, intérieur, extérieur
|
||||
{ocr_section}
|
||||
Réponds UNIQUEMENT avec le JSON, sans texte avant ou après, sans balises markdown."""
|
||||
|
||||
|
||||
async def analyze_image(
|
||||
file_path: str,
|
||||
ocr_hint: Optional[str] = None,
|
||||
language: str = "français",
|
||||
) -> dict:
|
||||
"""
|
||||
Envoie l'image à l'AI pour analyse (Description + Tags).
|
||||
"""
|
||||
if not settings.AI_ENABLED:
|
||||
return {}
|
||||
|
||||
result = {
|
||||
"description": None,
|
||||
"tags": [],
|
||||
"confidence": None,
|
||||
"model": settings.OPENROUTER_MODEL if settings.AI_PROVIDER == "openrouter" else settings.GEMINI_MODEL,
|
||||
"prompt_tokens": None,
|
||||
"output_tokens": None,
|
||||
}
|
||||
|
||||
try:
|
||||
image_bytes, media_type = _read_image(file_path)
|
||||
prompt = _build_prompt(ocr_hint, language)
|
||||
|
||||
response = await _generate(
|
||||
prompt=prompt,
|
||||
image_bytes=image_bytes,
|
||||
media_type=media_type,
|
||||
max_tokens=settings.GEMINI_MAX_TOKENS # Ou une config unifiée
|
||||
)
|
||||
|
||||
text = response.get("text")
|
||||
result["prompt_tokens"], result["output_tokens"] = response.get("usage")
|
||||
|
||||
if text:
|
||||
parsed = _extract_json(text)
|
||||
if parsed:
|
||||
result["description"] = parsed.get("description")
|
||||
result["tags"] = parsed.get("tags", [])
|
||||
result["confidence"] = parsed.get("confidence")
|
||||
else:
|
||||
logger.warning("ai.vision.json_parse_failed", extra={"raw": text[:100]})
|
||||
|
||||
if response.get("error"):
|
||||
logger.error("ai.vision.provider_error", extra={"error": response['error']})
|
||||
|
||||
except Exception as e:
|
||||
logger.error("ai.vision.unexpected_error", extra={"error": str(e)})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def extract_text_with_ai(file_path: str) -> dict:
|
||||
"""
|
||||
Utilise l'AI comme fallback OCR.
|
||||
"""
|
||||
result = {
|
||||
"text": None,
|
||||
"has_text": False,
|
||||
"language": "unknown",
|
||||
"confidence": 0.0,
|
||||
"method": f"ai-{settings.AI_PROVIDER}"
|
||||
}
|
||||
|
||||
if not settings.AI_ENABLED:
|
||||
return result
|
||||
|
||||
logger.info("ai.ocr.fallback_start", extra={"file": Path(file_path).name})
|
||||
|
||||
try:
|
||||
image_bytes, media_type = _read_image(file_path)
|
||||
prompt = """Agis comme un moteur OCR avancé.
|
||||
Extrais TOUT le texte visible dans cette image.
|
||||
Retourne UNIQUEMENT un objet JSON :
|
||||
{
|
||||
"text": "Le texte complet extrait ici...",
|
||||
"language": "fr" (code langue ISO 2 lettres, ex: fr, en, es),
|
||||
"confidence": 0.9 (estimation confiance 0.0 à 1.0)
|
||||
}
|
||||
Si aucun texte n'est visible, retourne : {"text": "", "has_text": false}
|
||||
"""
|
||||
|
||||
response = await _generate(
|
||||
prompt=prompt,
|
||||
image_bytes=image_bytes,
|
||||
media_type=media_type,
|
||||
max_tokens=1024
|
||||
)
|
||||
|
||||
text = response.get("text")
|
||||
if text:
|
||||
parsed = _extract_json(text)
|
||||
if parsed:
|
||||
extracted = parsed.get("text", "").strip()
|
||||
result["text"] = extracted
|
||||
result["has_text"] = bool(extracted) or parsed.get("has_text", False)
|
||||
result["language"] = parsed.get("language", "unknown")
|
||||
result["confidence"] = parsed.get("confidence", 0.0)
|
||||
logger.info("ai.ocr.success", extra={"chars": len(extracted)})
|
||||
else:
|
||||
logger.warning("ai.ocr.json_parse_failed")
|
||||
else:
|
||||
logger.info("ai.ocr.empty_response")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("ai.ocr.error", extra={"error": str(e)})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def summarize_url(url: str, content: str, language: str = "français") -> dict:
|
||||
"""Génère un résumé et des tags pour un contenu web."""
|
||||
result = {
|
||||
"summary": "",
|
||||
"tags": [],
|
||||
"model": settings.AI_PROVIDER,
|
||||
}
|
||||
|
||||
if not settings.AI_ENABLED:
|
||||
return result
|
||||
|
||||
prompt = f"""Tu reçois le contenu d'une page web. Génère un résumé et des tags en {language}.
|
||||
|
||||
URL : {url}
|
||||
|
||||
Contenu :
|
||||
\"\"\"
|
||||
{content[:3000]}
|
||||
\"\"\"
|
||||
|
||||
Retourne UNIQUEMENT ce JSON :
|
||||
{{
|
||||
"summary": "Résumé clair en 3-5 phrases en {language}",
|
||||
"tags": ["tag1", "tag2", "tag3"]
|
||||
}}"""
|
||||
|
||||
try:
|
||||
response = await _generate(
|
||||
prompt=prompt,
|
||||
max_tokens=settings.GEMINI_MAX_TOKENS
|
||||
)
|
||||
|
||||
text = response.get("text")
|
||||
if text:
|
||||
parsed = _extract_json(text)
|
||||
if parsed:
|
||||
result["summary"] = parsed.get("summary", "")
|
||||
result["tags"] = parsed.get("tags", [])
|
||||
|
||||
except Exception as e:
|
||||
logger.error("ai.summarize_url.error", extra={"error": str(e)})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def draft_task(description: str, context: Optional[str], language: str = "français") -> dict:
|
||||
"""Génère une tâche structurée à partir d'une description."""
|
||||
result = {
|
||||
"title": "",
|
||||
"description": "",
|
||||
"steps": [],
|
||||
"estimated_time": None,
|
||||
"priority": None,
|
||||
}
|
||||
|
||||
if not settings.AI_ENABLED:
|
||||
return result
|
||||
|
||||
ctx_section = f"\nContexte : {context}" if context else ""
|
||||
prompt = f"""Tu es un assistant de gestion de tâches. Génère une tâche structurée en {language}.
|
||||
|
||||
Description : {description}{ctx_section}
|
||||
|
||||
Retourne UNIQUEMENT ce JSON :
|
||||
{{
|
||||
"title": "Titre court et actionnable",
|
||||
"description": "Description complète de la tâche",
|
||||
"steps": ["Étape 1", "Étape 2", "Étape 3"],
|
||||
"estimated_time": "30 minutes",
|
||||
"priority": "haute|moyenne|basse"
|
||||
}}"""
|
||||
|
||||
try:
|
||||
response = await _generate(
|
||||
prompt=prompt,
|
||||
max_tokens=settings.GEMINI_MAX_TOKENS
|
||||
)
|
||||
|
||||
text = response.get("text")
|
||||
if text:
|
||||
parsed = _extract_json(text)
|
||||
if parsed:
|
||||
result.update(parsed)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("ai.draft_task.error", extra={"error": str(e)})
|
||||
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
Service d'extraction EXIF — Pillow + piexif
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import piexif
|
||||
from PIL import Image as PILImage
|
||||
from PIL.ExifTags import TAGS, GPSTAGS
|
||||
|
||||
|
||||
def _dms_to_decimal(dms: tuple, ref: str) -> float | None:
|
||||
"""Convertit les coordonnées GPS DMS (degrés/minutes/secondes) en décimal."""
|
||||
try:
|
||||
degrees = dms[0][0] / dms[0][1]
|
||||
minutes = dms[1][0] / dms[1][1]
|
||||
seconds = dms[2][0] / dms[2][1]
|
||||
decimal = degrees + minutes / 60 + seconds / 3600
|
||||
if ref in ("S", "W"):
|
||||
decimal = -decimal
|
||||
return round(decimal, 7)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_rational(value) -> str | None:
|
||||
"""Convertit un rationnel EXIF en chaîne lisible."""
|
||||
try:
|
||||
if isinstance(value, tuple) and len(value) == 2:
|
||||
num, den = value
|
||||
if den == 0:
|
||||
return None
|
||||
return f"{num}/{den}"
|
||||
return str(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _safe_str(value: Any) -> str | None:
|
||||
"""Décode les bytes en string si nécessaire."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="ignore").strip("\x00")
|
||||
return str(value)
|
||||
|
||||
|
||||
def extract_exif(file_path: str) -> dict:
|
||||
"""
|
||||
Extrait toutes les métadonnées EXIF d'une image.
|
||||
Retourne un dict structuré avec les données parsées.
|
||||
"""
|
||||
result = {
|
||||
"raw": {},
|
||||
"make": None,
|
||||
"model": None,
|
||||
"lens": None,
|
||||
"taken_at": None,
|
||||
"gps_lat": None,
|
||||
"gps_lon": None,
|
||||
"altitude": None,
|
||||
"iso": None,
|
||||
"aperture": None,
|
||||
"shutter": None,
|
||||
"focal": None,
|
||||
"flash": None,
|
||||
"orientation": None,
|
||||
"software": None,
|
||||
}
|
||||
|
||||
try:
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
return result
|
||||
|
||||
# ── Lecture EXIF brute via piexif ─────────────────────
|
||||
try:
|
||||
exif_data = piexif.load(str(path))
|
||||
except Exception:
|
||||
# JPEG sans EXIF, PNG, etc.
|
||||
return result
|
||||
|
||||
raw_dict = {}
|
||||
|
||||
# ── IFD 0 (Image principale) ──────────────────────────
|
||||
ifd0 = exif_data.get("0th", {})
|
||||
result["make"] = _safe_str(ifd0.get(piexif.ImageIFD.Make))
|
||||
result["model"] = _safe_str(ifd0.get(piexif.ImageIFD.Model))
|
||||
result["software"] = _safe_str(ifd0.get(piexif.ImageIFD.Software))
|
||||
result["orientation"] = ifd0.get(piexif.ImageIFD.Orientation)
|
||||
|
||||
# ── EXIF IFD ──────────────────────────────────────────
|
||||
exif = exif_data.get("Exif", {})
|
||||
|
||||
# Date de prise de vue
|
||||
taken_raw = _safe_str(exif.get(piexif.ExifIFD.DateTimeOriginal))
|
||||
if taken_raw:
|
||||
try:
|
||||
result["taken_at"] = datetime.strptime(taken_raw, "%Y:%m:%d %H:%M:%S")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Paramètres de prise de vue
|
||||
iso_val = exif.get(piexif.ExifIFD.ISOSpeedRatings)
|
||||
result["iso"] = int(iso_val) if iso_val else None
|
||||
|
||||
aperture_val = exif.get(piexif.ExifIFD.FNumber)
|
||||
if aperture_val:
|
||||
try:
|
||||
f = aperture_val[0] / aperture_val[1]
|
||||
result["aperture"] = f"f/{f:.1f}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
shutter_val = exif.get(piexif.ExifIFD.ExposureTime)
|
||||
if shutter_val:
|
||||
result["shutter"] = _parse_rational(shutter_val)
|
||||
|
||||
focal_val = exif.get(piexif.ExifIFD.FocalLength)
|
||||
if focal_val:
|
||||
try:
|
||||
f = focal_val[0] / focal_val[1]
|
||||
result["focal"] = f"{f:.0f}mm"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
flash_val = exif.get(piexif.ExifIFD.Flash)
|
||||
result["flash"] = bool(flash_val & 1) if flash_val is not None else None
|
||||
|
||||
lens_val = _safe_str(exif.get(piexif.ExifIFD.LensModel))
|
||||
result["lens"] = lens_val
|
||||
|
||||
# ── GPS IFD ───────────────────────────────────────────
|
||||
gps = exif_data.get("GPS", {})
|
||||
if gps:
|
||||
lat_val = gps.get(piexif.GPSIFD.GPSLatitude)
|
||||
lat_ref = _safe_str(gps.get(piexif.GPSIFD.GPSLatitudeRef))
|
||||
lon_val = gps.get(piexif.GPSIFD.GPSLongitude)
|
||||
lon_ref = _safe_str(gps.get(piexif.GPSIFD.GPSLongitudeRef))
|
||||
|
||||
if lat_val and lat_ref:
|
||||
result["gps_lat"] = _dms_to_decimal(lat_val, lat_ref)
|
||||
if lon_val and lon_ref:
|
||||
result["gps_lon"] = _dms_to_decimal(lon_val, lon_ref)
|
||||
|
||||
alt_val = gps.get(piexif.GPSIFD.GPSAltitude)
|
||||
if alt_val:
|
||||
try:
|
||||
result["altitude"] = round(alt_val[0] / alt_val[1], 2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ── Dict brut lisible (TAGS humains) ──────────────────
|
||||
with PILImage.open(path) as img:
|
||||
raw_exif = img._getexif()
|
||||
if raw_exif:
|
||||
for tag_id, val in raw_exif.items():
|
||||
tag = TAGS.get(tag_id, str(tag_id))
|
||||
if isinstance(val, bytes):
|
||||
val = val.decode("utf-8", errors="ignore")
|
||||
elif isinstance(val, tuple):
|
||||
val = list(val)
|
||||
raw_dict[tag] = val
|
||||
result["raw"] = raw_dict
|
||||
|
||||
except Exception as e:
|
||||
logger.error("exif.extraction_error", extra={"file": file_path, "error": str(e)})
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Service OCR — extraction de texte via Tesseract
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from PIL import Image as PILImage
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import pytesseract
|
||||
_ocr_import_error: Exception | None = None
|
||||
except Exception as e:
|
||||
pytesseract = None
|
||||
_ocr_import_error = e
|
||||
|
||||
|
||||
def _detect_language(text: str) -> str:
|
||||
"""Détection grossière de la langue à partir du texte extrait."""
|
||||
if not text:
|
||||
return "unknown"
|
||||
|
||||
# Mots communs français
|
||||
fr_words = {"le", "la", "les", "de", "du", "des", "un", "une", "et", "en", "est", "que"}
|
||||
# Mots communs anglais
|
||||
en_words = {"the", "is", "are", "and", "or", "of", "to", "in", "a", "an", "for", "with"}
|
||||
|
||||
words = set(text.lower().split())
|
||||
fr_score = len(words & fr_words)
|
||||
en_score = len(words & en_words)
|
||||
|
||||
if fr_score == 0 and en_score == 0:
|
||||
return "unknown"
|
||||
return "fr" if fr_score >= en_score else "en"
|
||||
|
||||
|
||||
def extract_text(file_path: str) -> dict:
|
||||
"""
|
||||
Extrait le texte d'une image via Tesseract OCR.
|
||||
Retourne un dict avec le texte, la langue et le score de confiance.
|
||||
"""
|
||||
result = {
|
||||
"text": None,
|
||||
"language": None,
|
||||
"confidence": None,
|
||||
"has_text": False,
|
||||
}
|
||||
|
||||
if not settings.OCR_ENABLED:
|
||||
return result
|
||||
|
||||
if pytesseract is None:
|
||||
logger.warning("ocr.unavailable", extra={"error": str(_ocr_import_error)})
|
||||
return result
|
||||
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
return result
|
||||
|
||||
try:
|
||||
# Configuration Tesseract
|
||||
if settings.TESSERACT_CMD:
|
||||
pytesseract.pytesseract.tesseract_cmd = settings.TESSERACT_CMD
|
||||
|
||||
with PILImage.open(path) as img:
|
||||
# Convertit en RGB si nécessaire
|
||||
if img.mode not in ("RGB", "L"):
|
||||
img = img.convert("RGB")
|
||||
|
||||
# Extraction avec données de confiance
|
||||
data = pytesseract.image_to_data(
|
||||
img,
|
||||
lang=settings.OCR_LANGUAGES,
|
||||
output_type=pytesseract.Output.DICT,
|
||||
)
|
||||
|
||||
# Calcul de la confiance moyenne (on ignore les -1)
|
||||
confidences = [
|
||||
int(c) for c in data["conf"]
|
||||
if str(c).strip() not in ("-1", "")
|
||||
]
|
||||
avg_confidence = (
|
||||
round(sum(confidences) / len(confidences) / 100, 3)
|
||||
if confidences else 0.0
|
||||
)
|
||||
|
||||
# Texte nettoyé
|
||||
raw_text = pytesseract.image_to_string(
|
||||
img,
|
||||
lang=settings.OCR_LANGUAGES,
|
||||
).strip()
|
||||
|
||||
if raw_text and len(raw_text) > 3:
|
||||
result["text"] = raw_text
|
||||
result["has_text"] = True
|
||||
result["confidence"] = avg_confidence
|
||||
result["language"] = _detect_language(raw_text)
|
||||
else:
|
||||
result["has_text"] = False
|
||||
|
||||
except pytesseract.TesseractNotFoundError:
|
||||
logger.warning("ocr.tesseract_not_found")
|
||||
except Exception as e:
|
||||
logger.error("ocr.extraction_error", extra={"file": file_path, "error": str(e)})
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
Pipeline de traitement AI — orchestration des 3 étapes
|
||||
Chaque étape est indépendante : un échec partiel n'arrête pas le pipeline.
|
||||
|
||||
Publie des événements Redis (si disponible) pour le suivi en temps réel.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models.image import Image, ProcessingStatus
|
||||
from app.services.exif_service import extract_exif
|
||||
from app.services.ocr_service import extract_text
|
||||
from app.services.ai_vision import analyze_image, extract_text_with_ai
|
||||
import asyncio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _run_sync_in_thread(func: Any, *args: Any) -> Any:
|
||||
"""Exécute une fonction synchrone dans un thread pour ne pas bloquer l'event loop."""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, func, *args)
|
||||
|
||||
|
||||
async def _publish_event(
|
||||
redis: Any, image_id: int, event: str, data: dict | None = None
|
||||
) -> None:
|
||||
"""Publie un événement sur le channel Redis pipeline:{image_id}."""
|
||||
if redis is None:
|
||||
return
|
||||
try:
|
||||
payload = {"event": event, "image_id": image_id, "timestamp": time.time()}
|
||||
if data:
|
||||
payload["data"] = data
|
||||
await redis.publish(f"pipeline:{image_id}", json.dumps(payload))
|
||||
except Exception:
|
||||
pass # Pub/Sub non critique — ne doit pas bloquer le pipeline
|
||||
|
||||
|
||||
async def process_image_pipeline(
|
||||
image_id: int, db: AsyncSession, redis: Any = None
|
||||
) -> None:
|
||||
"""
|
||||
Pipeline complet de traitement d'une image :
|
||||
1. Extraction EXIF (sync → thread)
|
||||
2. OCR — extraction texte (sync → thread)
|
||||
3. Vision AI — description + tags (async)
|
||||
4. Sauvegarde finale en BDD
|
||||
|
||||
Le statut est mis à jour à chaque étape pour permettre le polling.
|
||||
Publie des événements Redis sur le channel pipeline:{image_id}.
|
||||
"""
|
||||
# ── Chargement de l'image ─────────────────────────────────
|
||||
result = await db.execute(select(Image).where(Image.id == image_id))
|
||||
image = result.scalar_one_or_none()
|
||||
|
||||
if not image:
|
||||
logger.warning("pipeline.image_not_found", extra={"image_id": image_id})
|
||||
return
|
||||
|
||||
# ── Démarrage ─────────────────────────────────────────────
|
||||
image.processing_status = ProcessingStatus.PROCESSING
|
||||
image.processing_started_at = datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
await db.refresh(image)
|
||||
|
||||
await _publish_event(redis, image_id, "pipeline.started")
|
||||
|
||||
errors: list[str] = []
|
||||
file_path = image.file_path
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# ÉTAPE 1 — Extraction EXIF
|
||||
# ════════════════════════════════════════════════════════════
|
||||
try:
|
||||
logger.info("pipeline.step.start", extra={"image_id": image_id, "step": "exif", "step_num": "1/3"})
|
||||
t0 = time.time()
|
||||
exif = await _run_sync_in_thread(extract_exif, file_path)
|
||||
|
||||
image.exif_raw = exif.get("raw")
|
||||
image.exif_make = exif.get("make")
|
||||
image.exif_model = exif.get("model")
|
||||
image.exif_lens = exif.get("lens")
|
||||
image.exif_taken_at = exif.get("taken_at")
|
||||
image.exif_gps_lat = exif.get("gps_lat")
|
||||
image.exif_gps_lon = exif.get("gps_lon")
|
||||
image.exif_altitude = exif.get("altitude")
|
||||
image.exif_iso = exif.get("iso")
|
||||
image.exif_aperture = exif.get("aperture")
|
||||
image.exif_shutter = exif.get("shutter")
|
||||
image.exif_focal = exif.get("focal")
|
||||
image.exif_flash = exif.get("flash")
|
||||
image.exif_orientation = exif.get("orientation")
|
||||
image.exif_software = exif.get("software")
|
||||
|
||||
await db.commit()
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
logger.info("pipeline.step.done", extra={"image_id": image_id, "step": "exif", "duration_ms": elapsed, "camera": image.exif_make})
|
||||
await _publish_event(redis, image_id, "step.completed", {
|
||||
"step": "exif", "duration_ms": elapsed, "camera": image.exif_make,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
msg = f"EXIF : {str(e)}"
|
||||
errors.append(msg)
|
||||
logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "exif", "error": str(e)})
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# ÉTAPE 2 — OCR
|
||||
# ════════════════════════════════════════════════════════════
|
||||
try:
|
||||
logger.info("pipeline.step.start", extra={"image_id": image_id, "step": "ocr", "step_num": "2/3"})
|
||||
t0 = time.time()
|
||||
ocr = await _run_sync_in_thread(extract_text, file_path)
|
||||
|
||||
# Fallback AI si OCR classique échoue ou ne trouve rien
|
||||
if not ocr.get("has_text", False):
|
||||
logger.info("pipeline.ocr.fallback", extra={"image_id": image_id, "reason": "tesseract_empty"})
|
||||
ai_ocr = await extract_text_with_ai(file_path)
|
||||
|
||||
if ai_ocr.get("has_text"):
|
||||
ocr = ai_ocr
|
||||
logger.info("pipeline.ocr.fallback_success", extra={"image_id": image_id, "chars": len(ocr.get("text", ""))})
|
||||
else:
|
||||
logger.info("pipeline.ocr.fallback_empty", extra={"image_id": image_id})
|
||||
|
||||
image.ocr_text = ocr.get("text")
|
||||
image.ocr_language = ocr.get("language")
|
||||
image.ocr_confidence = ocr.get("confidence")
|
||||
image.ocr_has_text = ocr.get("has_text", False)
|
||||
|
||||
await db.commit()
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
logger.info("pipeline.step.done", extra={"image_id": image_id, "step": "ocr", "duration_ms": elapsed, "has_text": image.ocr_has_text})
|
||||
await _publish_event(redis, image_id, "step.completed", {
|
||||
"step": "ocr", "duration_ms": elapsed, "has_text": image.ocr_has_text,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
msg = f"OCR : {str(e)}"
|
||||
errors.append(msg)
|
||||
logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "ocr", "error": str(e)})
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# ÉTAPE 3 — Vision AI (description + tags)
|
||||
# ════════════════════════════════════════════════════════════
|
||||
try:
|
||||
logger.info("pipeline.step.start", extra={"image_id": image_id, "step": "ai", "step_num": "3/3"})
|
||||
t0 = time.time()
|
||||
ai = await analyze_image(
|
||||
file_path=file_path,
|
||||
ocr_hint=image.ocr_text,
|
||||
)
|
||||
|
||||
image.ai_description = ai.get("description")
|
||||
image.ai_tags = ai.get("tags", [])
|
||||
image.ai_confidence = ai.get("confidence")
|
||||
image.ai_model_used = ai.get("model")
|
||||
image.ai_processed_at = datetime.now(timezone.utc)
|
||||
image.ai_prompt_tokens = ai.get("prompt_tokens")
|
||||
image.ai_output_tokens = ai.get("output_tokens")
|
||||
|
||||
await db.commit()
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
logger.info("pipeline.step.done", extra={"image_id": image_id, "step": "ai", "duration_ms": elapsed, "tags_count": len(image.ai_tags or [])})
|
||||
await _publish_event(redis, image_id, "step.completed", {
|
||||
"step": "ai", "duration_ms": elapsed, "tags_count": len(image.ai_tags or []),
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
msg = f"AI Vision : {str(e)}"
|
||||
errors.append(msg)
|
||||
logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "ai", "error": str(e)})
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# FINALISATION
|
||||
# ════════════════════════════════════════════════════════════
|
||||
image.processing_done_at = datetime.now(timezone.utc)
|
||||
|
||||
if errors:
|
||||
if image.ai_description:
|
||||
image.processing_status = ProcessingStatus.DONE
|
||||
image.processing_error = f"Avertissements : {'; '.join(errors)}"
|
||||
else:
|
||||
image.processing_status = ProcessingStatus.ERROR
|
||||
image.processing_error = "; ".join(errors)
|
||||
else:
|
||||
image.processing_status = ProcessingStatus.DONE
|
||||
image.processing_error = None
|
||||
|
||||
await db.commit()
|
||||
logger.info("pipeline.completed", extra={
|
||||
"image_id": image_id,
|
||||
"status": image.processing_status.value,
|
||||
"errors": len(errors),
|
||||
})
|
||||
|
||||
if errors:
|
||||
await _publish_event(redis, image_id, "pipeline.error", {"errors": errors})
|
||||
else:
|
||||
await _publish_event(redis, image_id, "pipeline.done")
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Service de scraping — extraction de contenu web pour résumés AI
|
||||
"""
|
||||
from typing import Optional
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (compatible; ShaarliBot/1.0)"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
async def fetch_page_content(url: str) -> dict:
|
||||
"""
|
||||
Récupère le contenu d'une URL et extrait :
|
||||
- Titre de la page
|
||||
- Méta description
|
||||
- Texte principal
|
||||
"""
|
||||
result = {
|
||||
"url": url,
|
||||
"title": None,
|
||||
"description": None,
|
||||
"text": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=HEADERS,
|
||||
timeout=15.0,
|
||||
follow_redirects=True,
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
# Titre
|
||||
title_tag = soup.find("title")
|
||||
result["title"] = title_tag.get_text(strip=True) if title_tag else None
|
||||
|
||||
# Meta description
|
||||
meta_desc = soup.find("meta", attrs={"name": "description"})
|
||||
if not meta_desc:
|
||||
meta_desc = soup.find("meta", attrs={"property": "og:description"})
|
||||
result["description"] = meta_desc.get("content", "") if meta_desc else None
|
||||
|
||||
# Texte principal — on retire scripts, styles, nav
|
||||
for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
|
||||
tag.decompose()
|
||||
|
||||
# Priorité aux balises sémantiques
|
||||
main = soup.find("article") or soup.find("main") or soup.find("body")
|
||||
if main:
|
||||
paragraphs = main.find_all("p")
|
||||
text = " ".join(p.get_text(strip=True) for p in paragraphs if len(p.get_text(strip=True)) > 30)
|
||||
result["text"] = text[:5000] if text else None
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
result["error"] = f"HTTP {e.response.status_code}"
|
||||
except httpx.RequestError as e:
|
||||
result["error"] = f"Connexion impossible : {str(e)}"
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Service de stockage — sauvegarde fichiers, génération thumbnails
|
||||
Multi-tenant : les fichiers sont isolés par client_id.
|
||||
"""
|
||||
import uuid
|
||||
import logging
|
||||
import aiofiles
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from PIL import Image as PILImage
|
||||
from fastapi import UploadFile, HTTPException, status
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"image/jpeg", "image/png", "image/gif",
|
||||
"image/webp", "image/bmp", "image/tiff",
|
||||
}
|
||||
|
||||
THUMBNAIL_SIZE = (320, 320)
|
||||
|
||||
|
||||
def _generate_filename(original: str) -> tuple[str, str]:
|
||||
"""Retourne (uuid_filename, extension)."""
|
||||
suffix = Path(original).suffix.lower() or ".jpg"
|
||||
uid = str(uuid.uuid4())
|
||||
return f"{uid}{suffix}", uid
|
||||
|
||||
|
||||
def _get_client_upload_path(client_id: str) -> Path:
|
||||
"""Retourne le répertoire d'upload pour un client donné."""
|
||||
p = settings.upload_path / client_id
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _get_client_thumbnails_path(client_id: str) -> Path:
|
||||
"""Retourne le répertoire de thumbnails pour un client donné."""
|
||||
p = settings.thumbnails_path / client_id
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
async def save_upload(file: UploadFile, client_id: str) -> dict:
|
||||
"""
|
||||
Valide, sauvegarde le fichier uploadé et génère un thumbnail.
|
||||
Les fichiers sont stockés dans uploads/{client_id}/ pour l'isolation.
|
||||
Retourne un dict avec toutes les métadonnées fichier.
|
||||
"""
|
||||
# ── Validation MIME ───────────────────────────────────────
|
||||
if file.content_type not in ALLOWED_MIME_TYPES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail=f"Type non supporté : {file.content_type}. "
|
||||
f"Acceptés : {', '.join(ALLOWED_MIME_TYPES)}",
|
||||
)
|
||||
|
||||
# ── Lecture du contenu ────────────────────────────────────
|
||||
content = await file.read()
|
||||
|
||||
if len(content) > settings.max_upload_bytes:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail=f"Fichier trop volumineux. Max : {settings.MAX_UPLOAD_SIZE_MB} MB",
|
||||
)
|
||||
|
||||
# ── Nommage et chemins ────────────────────────────────────
|
||||
filename, file_uuid = _generate_filename(file.filename or "image")
|
||||
upload_dir = _get_client_upload_path(client_id)
|
||||
thumb_dir = _get_client_thumbnails_path(client_id)
|
||||
|
||||
file_path = upload_dir / filename
|
||||
thumb_filename = f"thumb_{filename}"
|
||||
thumb_path = thumb_dir / thumb_filename
|
||||
|
||||
# ── Sauvegarde fichier original ───────────────────────────
|
||||
async with aiofiles.open(file_path, "wb") as f:
|
||||
await f.write(content)
|
||||
|
||||
# ── Dimensions + thumbnail ────────────────────────────────
|
||||
width, height = None, None
|
||||
try:
|
||||
with PILImage.open(file_path) as img:
|
||||
width, height = img.size
|
||||
img.thumbnail(THUMBNAIL_SIZE, PILImage.LANCZOS)
|
||||
# Convertit en RGB si nécessaire (ex: PNG RGBA)
|
||||
if img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
img.save(thumb_path, "JPEG", quality=85)
|
||||
except Exception as e:
|
||||
# Thumbnail non bloquant
|
||||
thumb_path = None
|
||||
logger.warning("Erreur génération thumbnail : %s", e)
|
||||
|
||||
return {
|
||||
"uuid": file_uuid,
|
||||
"original_name": file.filename,
|
||||
"filename": filename,
|
||||
"file_path": str(file_path),
|
||||
"thumbnail_path": str(thumb_path) if thumb_path else None,
|
||||
"mime_type": file.content_type,
|
||||
"file_size": len(content),
|
||||
"width": width,
|
||||
"height": height,
|
||||
"uploaded_at": datetime.now(timezone.utc),
|
||||
"client_id": client_id,
|
||||
}
|
||||
|
||||
|
||||
def delete_files(file_path: str, thumbnail_path: str | None = None) -> None:
|
||||
"""Supprime le fichier original et son thumbnail du disque."""
|
||||
for path_str in [file_path, thumbnail_path]:
|
||||
if path_str:
|
||||
p = Path(path_str)
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
|
||||
|
||||
def get_image_url(filename: str, client_id: str, thumb: bool = False) -> str:
|
||||
"""Construit l'URL publique d'une image."""
|
||||
prefix = "thumbnails" if thumb else "uploads"
|
||||
return f"/static/{prefix}/{client_id}/{filename}"
|
||||
@@ -0,0 +1,227 @@
|
||||
"""
|
||||
Abstraction StorageBackend — interface commune pour le stockage de fichiers.
|
||||
|
||||
Deux implémentations :
|
||||
- LocalStorage : fichiers sur disque local + URLs signées HMAC
|
||||
- S3Storage : AWS S3 / MinIO / Cloudflare R2 via aioboto3
|
||||
|
||||
Le reste du code utilise exclusivement get_storage_backend() et l'interface
|
||||
StorageBackend — jamais les classes concrètes directement.
|
||||
"""
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# Singleton backend
|
||||
_backend: "StorageBackend | None" = None
|
||||
|
||||
|
||||
class StorageBackend(ABC):
|
||||
"""Interface abstraite pour le stockage de fichiers."""
|
||||
|
||||
@abstractmethod
|
||||
async def save(self, content: bytes, path: str, content_type: str) -> str:
|
||||
"""Sauvegarde un fichier. Retourne le chemin stocké."""
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, path: str) -> None:
|
||||
"""Supprime un fichier."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_signed_url(self, path: str, expires_in: int = 900) -> str:
|
||||
"""Retourne une URL d'accès temporaire signée."""
|
||||
|
||||
@abstractmethod
|
||||
async def exists(self, path: str) -> bool:
|
||||
"""Vérifie qu'un fichier existe."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_size(self, path: str) -> int:
|
||||
"""Retourne la taille en bytes."""
|
||||
|
||||
|
||||
class LocalStorage(StorageBackend):
|
||||
"""Stockage sur disque local avec URLs signées HMAC."""
|
||||
|
||||
def __init__(self, base_dir: str, secret: str) -> None:
|
||||
self._base_dir = Path(base_dir)
|
||||
self._serializer = URLSafeTimedSerializer(secret)
|
||||
|
||||
def _full_path(self, path: str) -> Path:
|
||||
return self._base_dir / path
|
||||
|
||||
async def save(self, content: bytes, path: str, content_type: str) -> str:
|
||||
"""Sauvegarde un fichier sur disque."""
|
||||
full = self._full_path(path)
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.open(full, "wb") as f:
|
||||
await f.write(content)
|
||||
return path
|
||||
|
||||
async def delete(self, path: str) -> None:
|
||||
"""Supprime un fichier du disque."""
|
||||
full = self._full_path(path)
|
||||
if full.exists():
|
||||
full.unlink()
|
||||
|
||||
async def get_signed_url(self, path: str, expires_in: int = 900) -> str:
|
||||
"""Génère un token HMAC signé pour accéder au fichier."""
|
||||
token = self._serializer.dumps({"path": path, "max_age": expires_in})
|
||||
return f"/files/signed/{token}"
|
||||
|
||||
def validate_token(self, token: str) -> str | None:
|
||||
"""Valide un token HMAC et retourne le path, None si invalide/expiré."""
|
||||
try:
|
||||
data = self._serializer.loads(token, max_age=3600)
|
||||
return data.get("path")
|
||||
except (BadSignature, SignatureExpired):
|
||||
return None
|
||||
|
||||
def validate_token_with_max_age(self, token: str, max_age: int = 3600) -> str | None:
|
||||
"""Valide un token HMAC avec un max_age spécifique."""
|
||||
try:
|
||||
data = self._serializer.loads(token, max_age=max_age)
|
||||
return data.get("path")
|
||||
except (BadSignature, SignatureExpired):
|
||||
return None
|
||||
|
||||
async def exists(self, path: str) -> bool:
|
||||
"""Vérifie l'existence du fichier sur disque."""
|
||||
return self._full_path(path).exists()
|
||||
|
||||
async def get_size(self, path: str) -> int:
|
||||
"""Retourne la taille du fichier."""
|
||||
full = self._full_path(path)
|
||||
if full.exists():
|
||||
return full.stat().st_size
|
||||
return 0
|
||||
|
||||
def get_absolute_path(self, path: str) -> Path:
|
||||
"""Retourne le chemin absolu d'un fichier (pour FileResponse)."""
|
||||
return self._full_path(path)
|
||||
|
||||
|
||||
class S3Storage(StorageBackend):
|
||||
"""Stockage S3/MinIO via aioboto3."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bucket: str,
|
||||
prefix: str = "",
|
||||
region: str = "us-east-1",
|
||||
endpoint_url: str = "",
|
||||
access_key: str = "",
|
||||
secret_key: str = "",
|
||||
) -> None:
|
||||
self._bucket = bucket
|
||||
self._prefix = prefix.rstrip("/")
|
||||
self._region = region
|
||||
self._endpoint_url = endpoint_url or None
|
||||
self._access_key = access_key
|
||||
self._secret_key = secret_key
|
||||
|
||||
def _s3_key(self, path: str) -> str:
|
||||
if self._prefix:
|
||||
return f"{self._prefix}/{path}"
|
||||
return path
|
||||
|
||||
def _get_session(self):
|
||||
import aioboto3
|
||||
return aioboto3.Session(
|
||||
aws_access_key_id=self._access_key,
|
||||
aws_secret_access_key=self._secret_key,
|
||||
region_name=self._region,
|
||||
)
|
||||
|
||||
async def save(self, content: bytes, path: str, content_type: str) -> str:
|
||||
"""Upload vers S3/MinIO."""
|
||||
session = self._get_session()
|
||||
async with session.client("s3", endpoint_url=self._endpoint_url) as client:
|
||||
await client.put_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._s3_key(path),
|
||||
Body=content,
|
||||
ContentType=content_type,
|
||||
)
|
||||
return path
|
||||
|
||||
async def delete(self, path: str) -> None:
|
||||
"""Supprime un objet S3."""
|
||||
session = self._get_session()
|
||||
async with session.client("s3", endpoint_url=self._endpoint_url) as client:
|
||||
await client.delete_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._s3_key(path),
|
||||
)
|
||||
|
||||
async def get_signed_url(self, path: str, expires_in: int = 900) -> str:
|
||||
"""Génère une URL présignée S3."""
|
||||
session = self._get_session()
|
||||
async with session.client("s3", endpoint_url=self._endpoint_url) as client:
|
||||
url = await client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": self._bucket, "Key": self._s3_key(path)},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
return url
|
||||
|
||||
async def exists(self, path: str) -> bool:
|
||||
"""Vérifie l'existence via head_object."""
|
||||
session = self._get_session()
|
||||
async with session.client("s3", endpoint_url=self._endpoint_url) as client:
|
||||
try:
|
||||
await client.head_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._s3_key(path),
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def get_size(self, path: str) -> int:
|
||||
"""Retourne la taille via head_object."""
|
||||
session = self._get_session()
|
||||
async with session.client("s3", endpoint_url=self._endpoint_url) as client:
|
||||
try:
|
||||
resp = await client.head_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._s3_key(path),
|
||||
)
|
||||
return resp.get("ContentLength", 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def get_storage_backend() -> StorageBackend:
|
||||
"""Factory : retourne le backend de stockage configuré (singleton)."""
|
||||
global _backend
|
||||
if _backend is not None:
|
||||
return _backend
|
||||
|
||||
if settings.STORAGE_BACKEND == "s3":
|
||||
_backend = S3Storage(
|
||||
bucket=settings.S3_BUCKET,
|
||||
prefix=settings.S3_PREFIX,
|
||||
region=settings.S3_REGION,
|
||||
endpoint_url=settings.S3_ENDPOINT_URL,
|
||||
access_key=settings.S3_ACCESS_KEY,
|
||||
secret_key=settings.S3_SECRET_KEY,
|
||||
)
|
||||
else:
|
||||
_backend = LocalStorage(
|
||||
base_dir=str(settings.upload_path.parent),
|
||||
secret=settings.SIGNED_URL_SECRET,
|
||||
)
|
||||
|
||||
return _backend
|
||||
|
||||
|
||||
def reset_storage_backend() -> None:
|
||||
"""Reset le singleton (utile pour les tests)."""
|
||||
global _backend
|
||||
_backend = None
|
||||
Reference in New Issue
Block a user