Add admin panel, WebSocket support, and API versioning
Introduce an admin portal (React + Nginx), WebSocket routing, and API versioning middleware with `/api/v1/` prefix deprecation. Add master API key authentication, new Prometheus metrics for AI token consumption and active WebSockets, and extend S3 config with a public endpoint URL. Update test paths and fixtures to align with the new routing structure.
This commit is contained in:
@@ -17,6 +17,10 @@ from app.models.image import Image, ProcessingStatus
|
||||
from app.services.exif_service import extract_exif
|
||||
from app.services.ocr_service import extract_text
|
||||
from app.services.ai_vision import analyze_image, extract_text_with_ai
|
||||
from app.metrics import (
|
||||
hub_pipeline_duration, hub_pipeline_step_duration, hub_pipeline_errors,
|
||||
hub_ai_tokens_consumed
|
||||
)
|
||||
import asyncio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -25,14 +29,33 @@ logger = logging.getLogger(__name__)
|
||||
async def _publish_event(
|
||||
redis: Any, image_id: int, event: str, data: dict | None = None
|
||||
) -> None:
|
||||
"""Publie un événement sur le channel Redis pipeline:{image_id}."""
|
||||
"""
|
||||
Publie un événement sur le channel Redis pipeline:{image_id}.
|
||||
|
||||
Stocke aussi les 10 derniers événements dans un buffer Redis
|
||||
(clé pipeline:buffer:{image_id}, TTL 60s) pour la reconnexion WebSocket.
|
||||
Publie sur pipeline:admin pour le monitoring admin.
|
||||
"""
|
||||
if redis is None:
|
||||
return
|
||||
try:
|
||||
payload = {"event": event, "image_id": image_id, "timestamp": time.time()}
|
||||
if data:
|
||||
payload["data"] = data
|
||||
await redis.publish(f"pipeline:{image_id}", json.dumps(payload))
|
||||
raw = json.dumps(payload)
|
||||
|
||||
# Publier sur le channel dédié à l'image
|
||||
await redis.publish(f"pipeline:{image_id}", raw)
|
||||
|
||||
# Buffer de reconnexion (10 derniers événements, TTL 60s)
|
||||
buffer_key = f"pipeline:buffer:{image_id}"
|
||||
await redis.rpush(buffer_key, raw)
|
||||
await redis.ltrim(buffer_key, -10, -1)
|
||||
await redis.expire(buffer_key, 60)
|
||||
|
||||
# Publier sur le channel admin pour le monitoring global
|
||||
if event in ("pipeline.started", "pipeline.done", "pipeline.error"):
|
||||
await redis.publish("pipeline:admin", raw)
|
||||
except Exception:
|
||||
pass # Pub/Sub non critique — ne doit pas bloquer le pipeline
|
||||
|
||||
@@ -61,6 +84,7 @@ async def process_image_pipeline(
|
||||
# ── Démarrage ─────────────────────────────────────────────
|
||||
image.processing_status = ProcessingStatus.PROCESSING
|
||||
image.processing_started_at = datetime.now(timezone.utc)
|
||||
t_start = time.time()
|
||||
await db.commit()
|
||||
await db.refresh(image)
|
||||
|
||||
@@ -97,6 +121,8 @@ async def process_image_pipeline(
|
||||
|
||||
await db.commit()
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
hub_pipeline_step_duration.labels(step="exif").observe(time.time() - t0)
|
||||
|
||||
logger.info("pipeline.step.done", extra={"image_id": image_id, "step": "exif", "duration_ms": elapsed, "camera": image.exif_make})
|
||||
await _publish_event(redis, image_id, "step.completed", {
|
||||
"step": "exif", "duration_ms": elapsed, "camera": image.exif_make,
|
||||
@@ -105,6 +131,7 @@ async def process_image_pipeline(
|
||||
except Exception as e:
|
||||
msg = f"EXIF : {str(e)}"
|
||||
errors.append(msg)
|
||||
hub_pipeline_errors.labels(step="exif").inc()
|
||||
logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "exif", "error": str(e)})
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
@@ -135,6 +162,8 @@ async def process_image_pipeline(
|
||||
|
||||
await db.commit()
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
hub_pipeline_step_duration.labels(step="ocr").observe(time.time() - t0)
|
||||
|
||||
logger.info("pipeline.step.done", extra={"image_id": image_id, "step": "ocr", "duration_ms": elapsed, "has_text": image.ocr_has_text})
|
||||
await _publish_event(redis, image_id, "step.completed", {
|
||||
"step": "ocr", "duration_ms": elapsed, "has_text": image.ocr_has_text,
|
||||
@@ -143,6 +172,7 @@ async def process_image_pipeline(
|
||||
except Exception as e:
|
||||
msg = f"OCR : {str(e)}"
|
||||
errors.append(msg)
|
||||
hub_pipeline_errors.labels(step="ocr").inc()
|
||||
logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "ocr", "error": str(e)})
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
@@ -166,6 +196,16 @@ async def process_image_pipeline(
|
||||
|
||||
await db.commit()
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
hub_pipeline_step_duration.labels(step="ai").observe(time.time() - t0)
|
||||
|
||||
# Token metrics
|
||||
if image.ai_prompt_tokens or image.ai_output_tokens:
|
||||
total_tokens = (image.ai_prompt_tokens or 0) + (image.ai_output_tokens or 0)
|
||||
hub_ai_tokens_consumed.labels(
|
||||
client_id=image.client_id,
|
||||
model=image.ai_model_used or "unknown"
|
||||
).inc(total_tokens)
|
||||
|
||||
logger.info("pipeline.step.done", extra={"image_id": image_id, "step": "ai", "duration_ms": elapsed, "tags_count": len(image.ai_tags or [])})
|
||||
await _publish_event(redis, image_id, "step.completed", {
|
||||
"step": "ai", "duration_ms": elapsed, "tags_count": len(image.ai_tags or []),
|
||||
@@ -174,6 +214,7 @@ async def process_image_pipeline(
|
||||
except Exception as e:
|
||||
msg = f"AI Vision : {str(e)}"
|
||||
errors.append(msg)
|
||||
hub_pipeline_errors.labels(step="ai").inc()
|
||||
logger.error("pipeline.step.error", extra={"image_id": image_id, "step": "ai", "error": str(e)})
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
@@ -193,6 +234,8 @@ async def process_image_pipeline(
|
||||
image.processing_error = None
|
||||
|
||||
await db.commit()
|
||||
hub_pipeline_duration.observe(time.time() - t_start)
|
||||
|
||||
logger.info("pipeline.completed", extra={
|
||||
"image_id": image_id,
|
||||
"status": image.processing_status.value,
|
||||
|
||||
Reference in New Issue
Block a user