Add admin panel, WebSocket support, and API versioning

Introduce an admin portal (React + Nginx), WebSocket routing, and
API versioning middleware with `/api/v1/` prefix deprecation.
Add master API key authentication, new Prometheus metrics for AI
token consumption and active WebSockets, and extend S3 config
with a public endpoint URL. Update test paths and fixtures to
align with the new routing structure.

app/routers/__init__.py

@@ -2,5 +2,7 @@ from app.routers.images import router as images_router
 from app.routers.ai import router as ai_router
 from app.routers.auth import router as auth_router
 from app.routers.files import router as files_router
+from app.routers.websocket import router as ws_router
+from app.routers.admin import router as admin_router
 
-__all__ = ["images_router", "ai_router", "auth_router", "files_router"]
+__all__ = ["images_router", "ai_router", "auth_router", "files_router", "ws_router", "admin_router"]

app/routers/admin.py

@@ -0,0 +1,165 @@
+from fastapi import APIRouter, Depends, HTTPException, Request
+from sqlalchemy import select, func
+from sqlalchemy.ext.asyncio import AsyncSession
+from typing import Dict, Any, List
+from pathlib import Path
+
+from app.database import get_db
+from app.models.image import Image
+from app.models.client import APIClient
+from app.dependencies.auth import require_scope
+
+router = APIRouter(prefix="/admin/api", tags=["Admin"])
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+ALLOWED_DOCS = {
+    "README.md": PROJECT_ROOT / "README.md",
+    "USER_GUIDE.md": PROJECT_ROOT / "docs" / "USER_GUIDE.md",
+    "ARCHITECTURE.md": PROJECT_ROOT / "docs" / "ARCHITECTURE.md",
+    "API_GUIDE.md": PROJECT_ROOT / "docs" / "API_GUIDE.md",
+    "SDK.md": PROJECT_ROOT / "docs" / "SDK.md",
+    "SHAARLI-INTEGRATION.md": PROJECT_ROOT / "docs" / "SHAARLI-INTEGRATION.md",
+    "WEBSOCKET.md": PROJECT_ROOT / "docs" / "WEBSOCKET.md",
+}
+
+@router.get("/stats")
+async def get_stats(
+    _: APIClient = Depends(require_scope("admin")),
+    db: AsyncSession = Depends(get_db)
+) -> Dict[str, Any]:
+    """Get global system statistics."""
+    # Count images
+    total_images_query = await db.execute(select(func.count(Image.id)))
+    total_images = total_images_query.scalar() or 0
+    
+    # Storage used (sum of file_size)
+    storage_query = await db.execute(select(func.sum(Image.file_size)))
+    storage_bytes = storage_query.scalar() or 0
+    
+    # Total AI tokens (sum of prompt + output tokens)
+    tokens_query = await db.execute(
+        select(func.sum(Image.ai_prompt_tokens) + func.sum(Image.ai_output_tokens))
+        .where(
+            Image.ai_prompt_tokens.isnot(None) | Image.ai_output_tokens.isnot(None)
+        )
+    )
+    total_tokens = tokens_query.scalar() or 0
+
+    # Total clients
+    clients_query = await db.execute(select(func.count(APIClient.id)))
+    total_clients = clients_query.scalar() or 0
+
+    return {
+        "total_images": total_images,
+        "total_storage_bytes": storage_bytes,
+        "total_ai_tokens": total_tokens,
+        "total_clients": total_clients
+    }
+
+@router.get("/clients")
+async def get_clients(
+    _: APIClient = Depends(require_scope("admin")),
+    db: AsyncSession = Depends(get_db)
+) -> List[Dict[str, Any]]:
+    """List all clients with metrics."""
+    result = await db.execute(select(APIClient).order_by(APIClient.created_at.desc()))
+    clients = result.scalars().all()
+    
+    return [
+        {
+            "id": c.id,
+            "name": c.name,
+            "is_active": c.is_active,
+            "is_premium": c.plan == "premium",
+            "storage_used": c.storage_used_bytes,
+            "storage_quota": c.quota_storage_mb,
+            "created_at": c.created_at
+        } for c in clients
+    ]
+
+@router.get("/queue/status")
+async def get_queue_status(
+    request: Request,
+    _: APIClient = Depends(require_scope("admin"))
+) -> Dict[str, Any]:
+    """Get ARQ queue status using the redis connection."""
+    redis = request.app.state.redis
+    
+    # Note: A real implementation would parse the ARQ keys here. 
+    # For now, we return basic statistics by probing redis directly with arq known queues.
+    # Count pending jobs in arq:queue
+    pending_count = await redis.llen("arq:queue") if hasattr(redis, "llen") else 0
+    # In ARQ, active jobs are harder to count without querying the worker sets, 
+    # so we return pending count.
+    
+    return {
+        "pending_jobs": pending_count,
+        "status": "active"
+    }
+
+@router.post("/clients/{client_id}/toggle")
+async def toggle_client(
+    client_id: str,
+    _: APIClient = Depends(require_scope("admin")),
+    db: AsyncSession = Depends(get_db)
+) -> Dict[str, Any]:
+    """Activate or deactivate a client."""
+    result = await db.execute(select(APIClient).where(APIClient.id == client_id))
+    client = result.scalar_one_or_none()
+    if not client:
+        raise HTTPException(status_code=404, detail="Client not found")
+        
+    client.is_active = not client.is_active
+    await db.commit()
+    
+    return {"id": client.id, "is_active": client.is_active}
+
+@router.post("/clients/{client_id}/reset-quota")
+async def reset_client_quota(
+    client_id: str,
+    _: APIClient = Depends(require_scope("admin")),
+    db: AsyncSession = Depends(get_db)
+) -> Dict[str, Any]:
+    """Reset the quota counters for a client."""
+    result = await db.execute(select(APIClient).where(APIClient.id == client_id))
+    client = result.scalar_one_or_none()
+    if not client:
+        raise HTTPException(status_code=404, detail="Client not found")
+        
+    client.storage_used_bytes = 0
+    await db.commit()
+    
+    return {"id": client.id, "storage_used": 0}
+
+@router.get("/docs")
+async def list_docs(
+    _: APIClient = Depends(require_scope("admin"))
+) -> List[Dict[str, str]]:
+    """List available documentation files."""
+    docs = []
+    for name, path in ALLOWED_DOCS.items():
+        if path.exists():
+            docs.append({"name": name, "title": name.replace(".md", "").replace("-", " ").title()})
+        else:
+            docs.append({"name": name, "title": name.replace(".md", "").replace("-", " ").title()}) # we might still want to list them, or omit
+    return docs
+
+@router.get("/docs/{filename}")
+async def get_doc(
+    filename: str,
+    _: APIClient = Depends(require_scope("admin"))
+) -> Dict[str, str]:
+    """Get the content of a specific documentation file."""
+    if filename not in ALLOWED_DOCS:
+        raise HTTPException(status_code=404, detail="Document not found or access denied")
+    
+    path = ALLOWED_DOCS[filename]
+    if not path.exists():
+        raise HTTPException(status_code=404, detail=f"Document file {filename} not found on server")
+        
+    try:
+        content = path.read_text(encoding="utf-8")
+        return {"filename": filename, "content": content}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error reading document: {str(e)}")
+

app/routers/ai.py

@@ -14,6 +14,7 @@ from app.services.scraper import fetch_page_content
 from app.services.ai_vision import summarize_url, draft_task
 from app.config import settings
 from app.middleware import limiter
+from app.metrics import hub_ai_tokens_consumed
 
 
 router = APIRouter(prefix="/ai", tags=["Intelligence Artificielle"])
@@ -62,7 +63,7 @@ async def summarize_link(
         language=body.language,
     )
 
-    return SummarizeResponse(
+    resp = SummarizeResponse(
         url=body.url,
         title=page.get("title"),
         summary=result["summary"],
@@ -70,6 +71,13 @@ async def summarize_link(
         model=result["model"],
     )
 
+    # Metrics
+    if result.get("prompt_tokens") or result.get("output_tokens"):
+        total = (result.get("prompt_tokens") or 0) + (result.get("output_tokens") or 0)
+        hub_ai_tokens_consumed.labels(client_id=client.id, model=result["model"]).inc(total)
+
+    return resp
+
 
 @router.post(
     "/draft-task",
@@ -99,4 +107,9 @@ async def generate_task(
     if not result.get("title"):
         raise HTTPException(status_code=500, detail="Échec de la génération de la tâche")
 
+    # Metrics
+    if result.get("prompt_tokens") or result.get("output_tokens"):
+        total = (result.get("prompt_tokens") or 0) + (result.get("output_tokens") or 0)
+        hub_ai_tokens_consumed.labels(client_id=client.id, model=result.get("model", "unknown")).inc(total)
+
     return DraftTaskResponse(**result)

app/routers/images.py

@@ -25,6 +25,9 @@ from app.schemas import (
 from app.services import storage
 from app.middleware import limiter, get_upload_rate_limit
 from app.workers.image_worker import QUEUE_STANDARD, QUEUE_PREMIUM
+from app.metrics import (
+    hub_images_uploaded, hub_images_deleted, hub_storage_used_bytes, hub_arq_jobs_enqueued
+)
 
 logger = logging.getLogger(__name__)
 
@@ -102,13 +105,14 @@ async def upload_image(
     await db.commit()
     await db.refresh(image)
 
-    # Enqueue dans ARQ (persistant, avec retry)
+    # Enqueue dans ARQ
     arq_pool = request.app.state.arq_pool
-    await arq_pool.enqueue_job(
-        "process_image_task",
-        image.id,
-        str(client.id)
-    )
+    await arq_pool.enqueue_job("process_image_task", image.id, str(client.id))
+
+    # Metrics
+    hub_images_uploaded.labels(client_id=client.id).inc()
+    hub_storage_used_bytes.labels(client_id=client.id).set(client.storage_used_bytes)
+    hub_arq_jobs_enqueued.labels(queue="default").inc()
 
     return UploadResponse(
         id=image.id,
@@ -190,6 +194,8 @@ async def list_images(
                 height=img.height,
                 uploaded_at=img.uploaded_at,
                 processing_status=img.processing_status,
+                client_id=img.client_id,
+                client_name=client.name,
                 ai_tags=img.ai_tags,
                 ai_description=img.ai_description,
                 thumbnail_path=img.thumbnail_path,
@@ -444,6 +450,10 @@ async def delete_image(
     await db.delete(image)
     await db.commit()
 
+    # Metrics
+    hub_images_deleted.labels(client_id=client.id).inc()
+    hub_storage_used_bytes.labels(client_id=client.id).set(client.storage_used_bytes)
+
     return DeleteResponse(deleted_id=image_id)
 
 

app/routers/websocket.py

@@ -0,0 +1,251 @@
+"""
+Router WebSocket — suivi temps réel du pipeline de traitement d'images.
+
+Endpoints :
+- WS /ws/pipeline/{image_id}?token=<api_key>  → événements d'un pipeline
+- WS /ws/admin/monitor?token=<admin_api_key>   → monitoring admin global
+"""
+import json
+import logging
+from typing import Any
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect, status
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.database import AsyncSessionLocal, get_db
+from app.dependencies.auth import hash_api_key
+from app.metrics import hub_active_websockets
+from app.models.client import APIClient
+from app.models.image import Image, ProcessingStatus
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["WebSocket"])
+
+
+# ─────────────────────────────────────────────────────────────
+# Helpers
+# ─────────────────────────────────────────────────────────────
+
+async def _authenticate_ws(websocket: WebSocket, db: AsyncSession) -> APIClient | None:
+    """
+    Authentifie une connexion WebSocket via le query param `token`.
+    Retourne le client ou None si invalide.
+    """
+    token = websocket.query_params.get("token")
+    if not token:
+        return None
+
+    # Vérification Master Key
+    from app.config import settings
+    if settings.ADMIN_API_KEY and token == settings.ADMIN_API_KEY:
+        return APIClient(
+            id="admin-master",
+            name="Imago Master Admin",
+            scopes=["admin", "images:read", "images:write", "ai:use"],
+            plan="premium",
+        )
+
+    key_hash = hash_api_key(token)
+    result = await db.execute(
+        select(APIClient).where(APIClient.api_key_hash == key_hash)
+    )
+    client = result.scalar_one_or_none()
+    if client and client.is_active:
+        return client
+    return None
+
+
+async def _get_image(image_id: int, db: AsyncSession) -> Image | None:
+    """Charge une image depuis la BDD."""
+    result = await db.execute(select(Image).where(Image.id == image_id))
+    return result.scalar_one_or_none()
+
+
+# ─────────────────────────────────────────────────────────────
+# WS /ws/pipeline/{image_id} — suivi d'un pipeline
+# ─────────────────────────────────────────────────────────────
+
+from fastapi import Depends
+
+@router.websocket("/ws/pipeline/{image_id}")
+async def ws_pipeline(
+    websocket: WebSocket,
+    image_id: int,
+    db: AsyncSession = Depends(get_db)
+) -> None:
+    """
+    WebSocket de suivi temps réel d'un pipeline image.
+
+    - Authentification via query param `token`
+    - Vérifie que l'image appartient au client
+    - Envoie le buffer de reconnexion puis les événements live
+    - Ferme après pipeline.done ou pipeline.error
+    """
+    # ── Authentification ──────────────────────────────────────
+    client = await _authenticate_ws(websocket, db)
+    if client is None:
+        await websocket.close(code=4001, reason="Token manquant ou invalide")
+        return
+
+    # ── Vérification propriété de l'image ─────────────────────
+    image = await _get_image(image_id, db)
+
+    if image is None:
+        await websocket.accept()
+        await websocket.close(code=4004, reason="Image introuvable")
+        return
+
+    # Admin peut voir toutes les images, sinon vérifier ownership
+    if not client.has_scope("admin") and image.client_id != client.id:
+        await websocket.close(code=4003, reason="Accès interdit")
+        return
+
+    # ── Accepter la connexion ─────────────────────────────────
+    await websocket.accept()
+    hub_active_websockets.inc()
+
+    try:
+        # ── Image déjà terminée → message synthétique ─────────
+        if image.processing_status == ProcessingStatus.DONE:
+            await websocket.send_json({
+                "event": "pipeline.done",
+                "image_id": image_id,
+                "status": "done",
+                "synthetic": True,
+            })
+            return
+
+        if image.processing_status == ProcessingStatus.ERROR:
+            await websocket.send_json({
+                "event": "pipeline.error",
+                "image_id": image_id,
+                "error": image.processing_error or "Erreur inconnue",
+                "synthetic": True,
+            })
+            return
+
+        # ── Récupérer le buffer de reconnexion depuis Redis ───
+        redis = getattr(websocket.app.state, "redis", None)
+        if redis is not None:
+            try:
+                buffer_key = f"pipeline:buffer:{image_id}"
+                buffered = await redis.lrange(buffer_key, 0, -1)
+                for raw_event in buffered:
+                    try:
+                        event_data = json.loads(raw_event)
+                        await websocket.send_json(event_data)
+                    except (json.JSONDecodeError, Exception):
+                        pass
+            except Exception as e:
+                logger.warning("ws.buffer_read_error", extra={"error": str(e)})
+
+        # ── S'abonner au channel Redis et écouter les événements
+        if redis is not None:
+            pubsub = redis.pubsub()
+            try:
+                await pubsub.subscribe(f"pipeline:{image_id}")
+
+                async for message in pubsub.listen():
+                    if message["type"] != "message":
+                        continue
+
+                    try:
+                        data = json.loads(message["data"])
+                    except (json.JSONDecodeError, TypeError):
+                        continue
+
+                    await websocket.send_json(data)
+
+                    # Fermer après pipeline.done ou pipeline.error
+                    event_type = data.get("event", "")
+                    if event_type in ("pipeline.done", "pipeline.error"):
+                        break
+            finally:
+                await pubsub.unsubscribe(f"pipeline:{image_id}")
+                await pubsub.close()
+        else:
+            # Pas de Redis — envoyer un message d'info et fermer
+            await websocket.send_json({
+                "event": "error",
+                "message": "Redis indisponible — utilisez le polling GET /images/{id}/status",
+            })
+
+    except WebSocketDisconnect:
+        logger.info("ws.client_disconnected", extra={
+            "image_id": image_id,
+            "client_id": client.id,
+        })
+    except Exception as e:
+        logger.error("ws.unexpected_error", extra={
+            "image_id": image_id,
+            "error": str(e),
+        })
+    finally:
+        hub_active_websockets.dec()
+
+
+# ─────────────────────────────────────────────────────────────
+# WS /ws/admin/monitor — monitoring admin global
+# ─────────────────────────────────────────────────────────────
+
+@router.websocket("/ws/admin/monitor")
+async def ws_admin_monitor(
+    websocket: WebSocket,
+    db: AsyncSession = Depends(get_db)
+) -> None:
+    """
+    WebSocket admin pour surveiller tous les pipelines en temps réel.
+
+    Nécessite le scope `admin`. Pousse un événement à chaque démarrage
+    ou fin de pipeline sur n'importe quelle image.
+    """
+    # ── Authentification ──────────────────────────────────────
+    client = await _authenticate_ws(websocket, db)
+    if client is None:
+        await websocket.close(code=4001, reason="Token manquant ou invalide")
+        return
+
+    if not client.has_scope("admin"):
+        await websocket.close(code=4003, reason="Scope admin requis")
+        return
+
+    # ── Accepter la connexion ─────────────────────────────────
+    await websocket.accept()
+    hub_active_websockets.inc()
+
+    try:
+        redis = getattr(websocket.app.state, "redis", None)
+        if redis is None:
+            await websocket.send_json({
+                "event": "error",
+                "message": "Redis indisponible",
+            })
+            return
+
+        pubsub = redis.pubsub()
+        try:
+            await pubsub.subscribe("pipeline:admin")
+
+            async for message in pubsub.listen():
+                if message["type"] != "message":
+                    continue
+
+                try:
+                    data = json.loads(message["data"])
+                except (json.JSONDecodeError, TypeError):
+                    continue
+
+                await websocket.send_json(data)
+
+        finally:
+            await pubsub.unsubscribe("pipeline:admin")
+            await pubsub.close()
+
+    except WebSocketDisconnect:
+        logger.info("ws.admin_disconnected", extra={"client_id": client.id})
+    except Exception as e:
+        logger.error("ws.admin_error", extra={"error": str(e)})
+    finally:
+        hub_active_websockets.dec()