Upgrade configuration and Docker stack to v2.0.0
This commit is contained in:
+38
-146
@@ -3,7 +3,7 @@ Imago — Application principale FastAPI
|
||||
"""
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi import FastAPI, Request, APIRouter
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from slowapi import _rate_limit_exceeded_handler
|
||||
@@ -16,7 +16,8 @@ from app.routers import images_router, ai_router, auth_router, files_router, ws_
|
||||
from app.middleware import limiter
|
||||
from app.middleware.logging_middleware import LoggingMiddleware
|
||||
from app.middleware.versioning import APIVersioningMiddleware
|
||||
from app.workers.redis_client import get_redis_pool, close_redis_pool
|
||||
from app.workers.redis_client import get_redis_pool, close_redis_pool, parse_redis_url
|
||||
from app.workers.arq_fallback import _FallbackArqPool
|
||||
|
||||
# Configure le logging structuré dès l'import
|
||||
configure_logging(debug=settings.DEBUG)
|
||||
@@ -38,41 +39,10 @@ except ImportError:
|
||||
|
||||
|
||||
def _arq_redis_settings() -> "RedisSettings":
|
||||
"""Parse REDIS_URL en RedisSettings ARQ."""
|
||||
url = settings.REDIS_URL
|
||||
if url.startswith("redis://"):
|
||||
url = url[8:]
|
||||
elif url.startswith("rediss://"):
|
||||
url = url[9:]
|
||||
|
||||
password = None
|
||||
host = "localhost"
|
||||
port = 6379
|
||||
database = 0
|
||||
|
||||
if "@" in url:
|
||||
auth_part, url = url.rsplit("@", 1)
|
||||
if ":" in auth_part:
|
||||
password = auth_part.split(":", 1)[1]
|
||||
else:
|
||||
password = auth_part
|
||||
|
||||
if "/" in url:
|
||||
host_port, db_str = url.split("/", 1)
|
||||
if db_str:
|
||||
database = int(db_str)
|
||||
else:
|
||||
host_port = url
|
||||
|
||||
if ":" in host_port:
|
||||
host, port_str = host_port.rsplit(":", 1)
|
||||
if port_str:
|
||||
port = int(port_str)
|
||||
else:
|
||||
host = host_port
|
||||
|
||||
"""Parse REDIS_URL en RedisSettings ARQ via le parser centralisé."""
|
||||
host, port, password, database = parse_redis_url()
|
||||
return RedisSettings(
|
||||
host=host or "localhost",
|
||||
host=host,
|
||||
port=port,
|
||||
password=password,
|
||||
database=database,
|
||||
@@ -85,11 +55,9 @@ def _arq_redis_settings() -> "RedisSettings":
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Création des répertoires de données
|
||||
settings.upload_path
|
||||
settings.thumbnails_path
|
||||
|
||||
# Initialisation de la base de données (création des tables)
|
||||
await init_db()
|
||||
|
||||
active_model = settings.OPENROUTER_MODEL if settings.AI_PROVIDER == "openrouter" else settings.GEMINI_MODEL
|
||||
@@ -101,7 +69,6 @@ async def lifespan(app: FastAPI):
|
||||
"ocr_enabled": settings.OCR_ENABLED,
|
||||
})
|
||||
|
||||
# Initialisation Redis + ARQ pool
|
||||
try:
|
||||
app.state.redis = await get_redis_pool()
|
||||
logger.info("startup.redis_connected", extra={"url": settings.REDIS_URL})
|
||||
@@ -122,24 +89,12 @@ async def lifespan(app: FastAPI):
|
||||
|
||||
yield
|
||||
|
||||
# Fermeture propre
|
||||
if hasattr(app.state, "arq_pool") and hasattr(app.state.arq_pool, "close"):
|
||||
await app.state.arq_pool.close()
|
||||
await close_redis_pool()
|
||||
logger.info("shutdown.complete")
|
||||
|
||||
|
||||
class _FallbackArqPool:
|
||||
"""Fallback quand Redis/ARQ n'est pas disponible."""
|
||||
|
||||
async def enqueue_job(self, *args, **kwargs):
|
||||
logger.warning("arq.fallback_enqueue", extra={"args": str(args)})
|
||||
return None
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Application
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
@@ -152,31 +107,24 @@ app = FastAPI(
|
||||
|
||||
Backend de gestion d'images et fonctionnalités AI pour l'interface Shaarli.
|
||||
|
||||
> **Note de versionnement :** L'API principale est désormais servie sous le préfixe `/api/v1/`. Les anciennes routes sans ce préfixe sont dépréciées.
|
||||
> **Note de versionnement :** L'API principale est désormais servie sous le préfixe `/api/v1/`.
|
||||
|
||||
### Fonctionnalités
|
||||
- 📸 **Upload et stockage d'images** avec génération de thumbnails
|
||||
- 🔍 **Extraction EXIF** automatique (appareil, GPS, paramètres de prise de vue)
|
||||
- 📝 **OCR** — extraction de texte depuis les images (Tesseract)
|
||||
- 🤖 **Vision AI** — description et classification par tags (Gemini)
|
||||
- 🔗 **Résumé d'URL** — scraping + résumé AI de pages web
|
||||
- ✅ **Rédaction de tâches** — génération structurée via AI
|
||||
- 📋 **File de tâches ARQ** — pipeline persistant avec retry automatique
|
||||
- 📊 **Métriques Prometheus** — /metrics endpoint
|
||||
|
||||
### Pipeline de traitement
|
||||
Chaque image uploadée est automatiquement traitée via ARQ (Redis) :
|
||||
`EXIF → OCR → Vision AI → stockage BDD`
|
||||
- Upload et stockage d'images avec génération de thumbnails
|
||||
- Extraction EXIF automatique (appareil, GPS, paramètres de prise de vue)
|
||||
- OCR — extraction de texte depuis les images (Tesseract)
|
||||
- Vision AI — description et classification par tags
|
||||
- Résumé d'URL — scraping + résumé AI de pages web
|
||||
- Rédaction de tâches — génération structurée via AI
|
||||
- File de tâches ARQ — pipeline persistant avec retry automatique
|
||||
- Métriques Prometheus — /metrics endpoint
|
||||
""",
|
||||
lifespan=lifespan,
|
||||
docs_url="/docs",
|
||||
redoc_url="/redoc",
|
||||
)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Middleware CORS
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Middleware CORS ─────────────────────────────────────────
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.CORS_ORIGINS,
|
||||
@@ -185,29 +133,21 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Middleware de Versionnement API
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# ── Request ID (trace_id) ───────────────────────────────────
|
||||
from app.middleware.request_id import RequestIDMiddleware
|
||||
app.add_middleware(RequestIDMiddleware)
|
||||
|
||||
# ── API Versioning ──────────────────────────────────────────
|
||||
app.add_middleware(APIVersioningMiddleware)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Middleware Logging HTTP
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Logging HTTP ────────────────────────────────────────────
|
||||
app.add_middleware(LoggingMiddleware)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Rate Limiting (slowapi)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Rate Limiting ───────────────────────────────────────────
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Prometheus Metrics
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Prometheus Metrics ──────────────────────────────────────
|
||||
if _prometheus_available:
|
||||
Instrumentator(
|
||||
should_group_status_codes=True,
|
||||
@@ -215,39 +155,25 @@ if _prometheus_available:
|
||||
excluded_handlers=["/health", "/health/detailed", "/metrics"],
|
||||
).instrument(app).expose(app, endpoint="/metrics", tags=["Observabilité"])
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Fichiers statiques / URLs signées
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Fichiers statiques ──────────────────────────────────────
|
||||
if settings.STORAGE_BACKEND == "local":
|
||||
app.include_router(files_router)
|
||||
app.mount("/static/uploads", StaticFiles(directory=str(settings.upload_path)), name="uploads")
|
||||
app.mount("/static/thumbnails", StaticFiles(directory=str(settings.thumbnails_path)), name="thumbnails")
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Routers Versionnés (/api/v1/)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
from fastapi import APIRouter
|
||||
|
||||
# ── Routers versionnés (/api/v1/) ───────────────────────────
|
||||
api_v1_router = APIRouter(prefix="/api/v1")
|
||||
api_v1_router.include_router(images_router)
|
||||
api_v1_router.include_router(ai_router)
|
||||
api_v1_router.include_router(auth_router)
|
||||
|
||||
# Mount the versioned router on the app
|
||||
app.include_router(api_v1_router)
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Routers Non-Versionnés (WebSockets, Fichiers, Santé, Metrics)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Routers non-versionnés ──────────────────────────────────
|
||||
app.include_router(ws_router)
|
||||
app.include_router(admin_router)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Routes utilitaires
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# ── Routes utilitaires ──────────────────────────────────────
|
||||
|
||||
@app.get("/", tags=["Santé"])
|
||||
async def root():
|
||||
@@ -266,7 +192,6 @@ async def health():
|
||||
(settings.AI_PROVIDER == "openrouter" and bool(settings.OPENROUTER_API_KEY))
|
||||
)
|
||||
active_model = settings.OPENROUTER_MODEL if settings.AI_PROVIDER == "openrouter" else settings.GEMINI_MODEL
|
||||
|
||||
return {
|
||||
"status": "healthy",
|
||||
"ai_enabled": settings.AI_ENABLED,
|
||||
@@ -282,14 +207,12 @@ async def health_detailed(request: Request):
|
||||
"""Endpoint de santé détaillé pour monitoring avancé."""
|
||||
checks = {}
|
||||
|
||||
# 1. Backend check
|
||||
checks["backend"] = {
|
||||
"status": "ok",
|
||||
"version": settings.APP_VERSION,
|
||||
"uptime": "N/A" # Optionnel: ajouter un tracker d'uptime si nécessaire
|
||||
"uptime": "N/A",
|
||||
}
|
||||
|
||||
# 2. Database check
|
||||
try:
|
||||
from app.database import AsyncSessionLocal
|
||||
from sqlalchemy import text
|
||||
@@ -299,7 +222,6 @@ async def health_detailed(request: Request):
|
||||
except Exception as e:
|
||||
checks["database"] = {"status": "error", "error": str(e)}
|
||||
|
||||
# 3. Redis check
|
||||
redis = getattr(request.app.state, "redis", None)
|
||||
if redis:
|
||||
try:
|
||||
@@ -310,29 +232,17 @@ async def health_detailed(request: Request):
|
||||
else:
|
||||
checks["redis"] = {"status": "not_configured"}
|
||||
|
||||
# 4. Worker & Queue check (ARQ)
|
||||
arq_pool = getattr(request.app.state, "arq_pool", None)
|
||||
if arq_pool and not isinstance(arq_pool, _FallbackArqPool):
|
||||
try:
|
||||
# On essaye de voir le nombre de jobs en attente
|
||||
queued_jobs = await arq_pool.zcard("arq:queue")
|
||||
checks["queue"] = {
|
||||
"status": "ok",
|
||||
"pending_jobs": queued_jobs
|
||||
}
|
||||
|
||||
# Pour le worker, on vérifie la clé de health check déposée par ARQ
|
||||
# Par défaut c'est <queue_name>:health-check
|
||||
hc_key = f"{settings.S3_PREFIX or 'arq'}:queue:health-check"
|
||||
# On ré-essaye avec la queue par défaut si le préfixe est différent
|
||||
if not await redis.exists(hc_key):
|
||||
hc_key = "arq:queue:health-check"
|
||||
|
||||
worker_active = await redis.exists(hc_key)
|
||||
checks["queue"] = {"status": "ok", "pending_jobs": queued_jobs}
|
||||
hc_key = "arq:queue:health-check"
|
||||
worker_active = await redis.exists(hc_key) if redis else False
|
||||
checks["worker"] = {
|
||||
"status": "ok" if worker_active else "error",
|
||||
"active_workers": 1 if worker_active else 0,
|
||||
"detail": "Worker opérationnel" if worker_active else "Aucun worker détecté"
|
||||
"detail": "Worker opérationnel" if worker_active else "Aucun worker détecté",
|
||||
}
|
||||
except Exception as e:
|
||||
checks["queue"] = {"status": "error", "error": str(e)}
|
||||
@@ -341,33 +251,19 @@ async def health_detailed(request: Request):
|
||||
checks["queue"] = {"status": "fallback"}
|
||||
checks["worker"] = {"status": "error", "detail": "ARQ non disponible"}
|
||||
|
||||
# 5. MinIO / Storage check
|
||||
from app.services.storage_backend import get_storage_backend, S3Storage
|
||||
storage = get_storage_backend()
|
||||
try:
|
||||
if isinstance(storage, S3Storage):
|
||||
# On vérifie la connexion réelle à MinIO en listant les objets (ou head bucket)
|
||||
# Pour faire simple/rapide on vérifie juste si on peut accéder au bucket
|
||||
# via l'interface du storage qui a déjà les sessions asynchrones
|
||||
exists = await storage.exists(".health-check") # path arbitraire
|
||||
checks["minio"] = {
|
||||
"status": "ok",
|
||||
"backend": "s3",
|
||||
"bucket": settings.S3_BUCKET
|
||||
}
|
||||
await storage.exists(".health-check")
|
||||
checks["minio"] = {"status": "ok", "backend": "s3", "bucket": settings.S3_BUCKET}
|
||||
else:
|
||||
checks["minio"] = {
|
||||
"status": "ok",
|
||||
"backend": "local",
|
||||
"path": settings.UPLOAD_DIR
|
||||
}
|
||||
checks["minio"] = {"status": "ok", "backend": "local", "path": settings.UPLOAD_DIR}
|
||||
except Exception as e:
|
||||
checks["minio"] = {"status": "error", "error": str(e)}
|
||||
|
||||
# 6. OCR check (Tesseract)
|
||||
checks["tesseract"] = {"status": "ok" if settings.OCR_ENABLED else "disabled"}
|
||||
|
||||
# 7. AI Provider check
|
||||
ai_configured = (
|
||||
(settings.AI_PROVIDER == "gemini" and bool(settings.GEMINI_API_KEY)) or
|
||||
(settings.AI_PROVIDER == "openrouter" and bool(settings.OPENROUTER_API_KEY))
|
||||
@@ -375,16 +271,12 @@ async def health_detailed(request: Request):
|
||||
checks["ai"] = {
|
||||
"status": "ok" if ai_configured else "warning",
|
||||
"provider": settings.AI_PROVIDER,
|
||||
"configured": ai_configured
|
||||
"configured": ai_configured,
|
||||
}
|
||||
|
||||
overall = "healthy" if all(
|
||||
c.get("status") in ("ok", "enabled", "disabled", "not_configured", "fallback")
|
||||
for k, c in checks.items() if k != "ai" # On tolère AI warning
|
||||
for k, c in checks.items() if k != "ai"
|
||||
) else "degraded"
|
||||
|
||||
return {
|
||||
"status": overall,
|
||||
"checks": checks,
|
||||
"version": settings.APP_VERSION,
|
||||
}
|
||||
return {"status": overall, "checks": checks, "version": settings.APP_VERSION}
|
||||
|
||||
Reference in New Issue
Block a user