Pipeline resilience: try/finally + startup recovery for stuck images

- process_image_pipeline: wrapped in try/finally to guarantee processing_status
  is always updated, even on unexpected crashes/timeouts
- init_db: recovery query that marks images stuck in 'processing' for >10min
  as ERROR on startup
- Clean rewrite of pipeline.py for consistent indentation

app/database.py

@@ -2,8 +2,10 @@
 Configuration SQLAlchemy — session async
 """
 import logging
+from datetime import datetime, timezone, timedelta
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
 from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy import select, text
 from app.config import settings
 
 logger = logging.getLogger(__name__)
@@ -44,9 +46,9 @@ async def get_db() -> AsyncSession:
 async def init_db():
     """Crée toutes les tables, applique les migrations, et initialise un client par défaut si nécessaire."""
     import secrets
-    from sqlalchemy import select, text
     from app.dependencies.auth import hash_api_key
     from app.models.client import APIClient, ClientPlan
+    from app.models.image import Image, ProcessingStatus
 
     async with engine.begin() as conn:
         from app.models import image  # noqa: F401
@@ -59,7 +61,7 @@ async def init_db():
                 "ALTER TABLE api_clients ADD COLUMN IF NOT EXISTS ai_provider VARCHAR(64)"
             ))
         except Exception:
-            pass  # SQLite ne supporte pas ADD COLUMN IF NOT EXISTS
+            pass
 
         try:
             await conn.execute(text(
@@ -68,8 +70,27 @@ async def init_db():
         except Exception:
             pass
 
-    # Vérifier s'il y a déjà des clients
+    # Vérifier s'il y a déjà des clients + recovery des images bloquées
     async with AsyncSessionLocal() as session:
+        # Recovery : marquer les images bloquées en "processing" depuis > 10 min comme ERROR
+        cutoff = datetime.now(timezone.utc) - timedelta(minutes=10)
+        stuck_query = select(Image).where(
+            Image.processing_status == ProcessingStatus.PROCESSING,
+            Image.processing_started_at < cutoff,
+        )
+        stuck_result = await session.execute(stuck_query)
+        stuck_images = stuck_result.scalars().all()
+        for img in stuck_images:
+            img.processing_status = ProcessingStatus.ERROR
+            img.processing_error = "[Recovery] Pipeline interrompu (timeout ou crash worker)"
+            img.processing_done_at = datetime.now(timezone.utc)
+        if stuck_images:
+            await session.commit()
+            logger.warning(
+                "init_db.recovery",
+                extra={"stuck_images_recovered": len(stuck_images)},
+            )
+
         result = await session.execute(select(APIClient).limit(1))
         if result.scalar_one_or_none() is None:
             # Table vide -> Création du client bootstrap

app/services/pipeline.py

@@ -21,7 +21,6 @@ from app.metrics import (
     hub_pipeline_duration, hub_pipeline_step_duration, hub_pipeline_errors,
     hub_ai_tokens_consumed
 )
-import asyncio
 
 logger = logging.getLogger(__name__)
 
@@ -44,20 +43,17 @@ async def _publish_event(
             payload["data"] = data
         raw = json.dumps(payload)
 
-        # Publier sur le channel dédié à l'image
         await redis.publish(f"pipeline:{image_id}", raw)
 
-        # Buffer de reconnexion (10 derniers événements, TTL 60s)
         buffer_key = f"pipeline:buffer:{image_id}"
         await redis.rpush(buffer_key, raw)
         await redis.ltrim(buffer_key, -10, -1)
         await redis.expire(buffer_key, 60)
 
-        # Publier sur le channel admin pour le monitoring global
         if event in ("pipeline.started", "pipeline.done", "pipeline.error"):
             await redis.publish("pipeline:admin", raw)
     except Exception:
-        pass  # Pub/Sub non critique — ne doit pas bloquer le pipeline
+        pass
 
 
 async def process_image_pipeline(
@@ -70,10 +66,9 @@ async def process_image_pipeline(
       3. Vision AI — description + tags (async)
       4. Sauvegarde finale en BDD
 
-    Le statut est mis à jour à chaque étape pour permettre le polling.
+    Try/finally garantit que le statut est TOUJOURS mis à jour,
-    Publie des événements Redis sur le channel pipeline:{image_id}.
+    même en cas de crash inattendu (timeout, OOM, etc.).
     """
-    # ── Chargement de l'image ─────────────────────────────────
     result = await db.execute(select(Image).where(Image.id == image_id))
     image = result.scalar_one_or_none()
 
@@ -81,7 +76,6 @@ async def process_image_pipeline(
         logger.warning("pipeline.image_not_found", extra={"image_id": image_id})
         return
 
-    # ── Démarrage ─────────────────────────────────────────────
     image.processing_status = ProcessingStatus.PROCESSING
     image.processing_started_at = datetime.now(timezone.utc)
     t_start = time.time()
@@ -93,14 +87,14 @@ async def process_image_pipeline(
     errors: list[str] = []
     file_path = image.file_path
 
-    # ════════════════════════════════════════════════════════════
+    try:
+        # ════════════════════════════════════════════════════════
         # ÉTAPE 1 — Extraction EXIF
-    # ════════════════════════════════════════════════════════════
+        # ════════════════════════════════════════════════════════
         try:
             logger.info("pipeline.step.start", extra={"image_id": image_id, "step": "exif", "step_num": "1/3"})
             t0 = time.time()
 
-        # Maintenant async et utilise le backend
             exif = await extract_exif(file_path)
 
             image.exif_raw         = exif.get("raw")
@@ -134,17 +128,15 @@ async def process_image_pipeline(
             hub_pipeline_errors.labels(step="exif").inc()
             logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "exif", "error": str(e)})
 
-    # ════════════════════════════════════════════════════════════
+        # ════════════════════════════════════════════════════════
         # ÉTAPE 2 — OCR
-    # ════════════════════════════════════════════════════════════
+        # ════════════════════════════════════════════════════════
         try:
             logger.info("pipeline.step.start", extra={"image_id": image_id, "step": "ocr", "step_num": "2/3"})
             t0 = time.time()
 
-        # Maintenant async et utilise le backend
             ocr = await extract_text(file_path)
 
-        # Fallback AI si OCR classique échoue ou ne trouve rien
             if not ocr.get("has_text", False):
                 logger.info("pipeline.ocr.fallback", extra={"image_id": image_id, "reason": "tesseract_empty"})
                 ai_ocr = await extract_text_with_ai(file_path)
@@ -175,14 +167,13 @@ async def process_image_pipeline(
             hub_pipeline_errors.labels(step="ocr").inc()
             logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "ocr", "error": str(e)})
 
-    # ════════════════════════════════════════════════════════════
+        # ════════════════════════════════════════════════════════
         # ÉTAPE 3 — Vision AI (description + tags)
-    # ════════════════════════════════════════════════════════════
+        # ════════════════════════════════════════════════════════
         try:
             logger.info("pipeline.step.start", extra={"image_id": image_id, "step": "ai", "step_num": "3/3"})
             t0 = time.time()
 
-        # Charger les préférences AI du client
             provider_override = None
             model_override = None
             if image.client_id:
@@ -214,7 +205,6 @@ async def process_image_pipeline(
             elapsed = int((time.time() - t0) * 1000)
             hub_pipeline_step_duration.labels(step="ai").observe(time.time() - t0)
 
-        # Token metrics
             if image.ai_prompt_tokens or image.ai_output_tokens:
                 total_tokens = (image.ai_prompt_tokens or 0) + (image.ai_output_tokens or 0)
                 hub_ai_tokens_consumed.labels(
@@ -233,9 +223,11 @@ async def process_image_pipeline(
             hub_pipeline_errors.labels(step="ai").inc()
             logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "ai", "error": str(e)})
 
-    # ════════════════════════════════════════════════════════════
+    finally:
-    # FINALISATION
+        # ════════════════════════════════════════════════════════
-    # ════════════════════════════════════════════════════════════
+        # FINALISATION — toujours exécuté, même après un crash
+        # ════════════════════════════════════════════════════════
+        try:
             image.processing_done_at = datetime.now(timezone.utc)
 
             if errors:
@@ -262,3 +254,9 @@ async def process_image_pipeline(
                 await _publish_event(redis, image_id, "pipeline.error", {"errors": errors})
             else:
                 await _publish_event(redis, image_id, "pipeline.done")
+        except Exception as final_error:
+            # Si même la finalisation échoue, log seulement — ne pas laisser l'image bloquée
+            logger.critical("pipeline.finalization_failed", extra={
+                "image_id": image_id,
+                "error": str(final_error),
+            })