Pipeline resilience: try/finally + startup recovery for stuck images
- process_image_pipeline: wrapped in try/finally to guarantee processing_status is always updated, even on unexpected crashes/timeouts - init_db: recovery query that marks images stuck in 'processing' for >10min as ERROR on startup - Clean rewrite of pipeline.py for consistent indentation
This commit is contained in:
+24
-3
@@ -2,8 +2,10 @@
|
||||
Configuration SQLAlchemy — session async
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy import select, text
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -44,9 +46,9 @@ async def get_db() -> AsyncSession:
|
||||
async def init_db():
|
||||
"""Crée toutes les tables, applique les migrations, et initialise un client par défaut si nécessaire."""
|
||||
import secrets
|
||||
from sqlalchemy import select, text
|
||||
from app.dependencies.auth import hash_api_key
|
||||
from app.models.client import APIClient, ClientPlan
|
||||
from app.models.image import Image, ProcessingStatus
|
||||
|
||||
async with engine.begin() as conn:
|
||||
from app.models import image # noqa: F401
|
||||
@@ -59,7 +61,7 @@ async def init_db():
|
||||
"ALTER TABLE api_clients ADD COLUMN IF NOT EXISTS ai_provider VARCHAR(64)"
|
||||
))
|
||||
except Exception:
|
||||
pass # SQLite ne supporte pas ADD COLUMN IF NOT EXISTS
|
||||
pass
|
||||
|
||||
try:
|
||||
await conn.execute(text(
|
||||
@@ -68,8 +70,27 @@ async def init_db():
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Vérifier s'il y a déjà des clients
|
||||
# Vérifier s'il y a déjà des clients + recovery des images bloquées
|
||||
async with AsyncSessionLocal() as session:
|
||||
# Recovery : marquer les images bloquées en "processing" depuis > 10 min comme ERROR
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=10)
|
||||
stuck_query = select(Image).where(
|
||||
Image.processing_status == ProcessingStatus.PROCESSING,
|
||||
Image.processing_started_at < cutoff,
|
||||
)
|
||||
stuck_result = await session.execute(stuck_query)
|
||||
stuck_images = stuck_result.scalars().all()
|
||||
for img in stuck_images:
|
||||
img.processing_status = ProcessingStatus.ERROR
|
||||
img.processing_error = "[Recovery] Pipeline interrompu (timeout ou crash worker)"
|
||||
img.processing_done_at = datetime.now(timezone.utc)
|
||||
if stuck_images:
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"init_db.recovery",
|
||||
extra={"stuck_images_recovered": len(stuck_images)},
|
||||
)
|
||||
|
||||
result = await session.execute(select(APIClient).limit(1))
|
||||
if result.scalar_one_or_none() is None:
|
||||
# Table vide -> Création du client bootstrap
|
||||
|
||||
Reference in New Issue
Block a user