Upgrade configuration and Docker stack to v2.0.0
This commit is contained in:
+49
-9
@@ -14,6 +14,7 @@ router = APIRouter(prefix="/admin/api", tags=["Admin"])
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
ALLOWED_DOCS = {
|
||||
"README.md": PROJECT_ROOT / "README.md",
|
||||
"ROADMAP.md": PROJECT_ROOT / "docs" / "ROADMAP.md",
|
||||
"USER_GUIDE.md": PROJECT_ROOT / "docs" / "USER_GUIDE.md",
|
||||
"ARCHITECTURE.md": PROJECT_ROOT / "docs" / "ARCHITECTURE.md",
|
||||
"API_GUIDE.md": PROJECT_ROOT / "docs" / "API_GUIDE.md",
|
||||
@@ -82,21 +83,60 @@ async def get_queue_status(
|
||||
request: Request,
|
||||
_: APIClient = Depends(require_scope("admin"))
|
||||
) -> Dict[str, Any]:
|
||||
"""Get ARQ queue status using the redis connection."""
|
||||
"""Get ARQ queue status including Dead Letter Queue."""
|
||||
redis = request.app.state.redis
|
||||
|
||||
# Note: A real implementation would parse the ARQ keys here.
|
||||
# For now, we return basic statistics by probing redis directly with arq known queues.
|
||||
# Count pending jobs in arq:queue
|
||||
from app.workers.dead_letter import get_dead_job_count
|
||||
|
||||
pending_count = await redis.llen("arq:queue") if hasattr(redis, "llen") else 0
|
||||
# In ARQ, active jobs are harder to count without querying the worker sets,
|
||||
# so we return pending count.
|
||||
|
||||
dead_count = await get_dead_job_count(redis)
|
||||
|
||||
return {
|
||||
"pending_jobs": pending_count,
|
||||
"status": "active"
|
||||
"dead_jobs": dead_count,
|
||||
"status": "active",
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Dead Letter Queue
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/queue/dead")
|
||||
async def list_dead_jobs(
|
||||
request: Request,
|
||||
_: APIClient = Depends(require_scope("admin")),
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
"""List dead jobs from the Dead Letter Queue."""
|
||||
from app.workers.dead_letter import get_dead_jobs
|
||||
redis = request.app.state.redis
|
||||
return await get_dead_jobs(redis, limit=limit)
|
||||
|
||||
|
||||
@router.post("/queue/dead/{index}/retry")
|
||||
async def retry_dead_job(
|
||||
request: Request,
|
||||
index: int,
|
||||
_: APIClient = Depends(require_scope("admin")),
|
||||
) -> dict:
|
||||
"""Retry a dead job by its index (0 = most recent)."""
|
||||
from app.workers.dead_letter import retry_dead_job
|
||||
redis = request.app.state.redis
|
||||
arq_pool = request.app.state.arq_pool
|
||||
return await retry_dead_job(redis, arq_pool, index)
|
||||
|
||||
|
||||
@router.delete("/queue/dead")
|
||||
async def clear_dead_jobs(
|
||||
request: Request,
|
||||
_: APIClient = Depends(require_scope("admin")),
|
||||
) -> Dict[str, Any]:
|
||||
"""Clear all dead jobs from the DLQ."""
|
||||
from app.workers.dead_letter import clear_dead_jobs
|
||||
redis = request.app.state.redis
|
||||
count = await clear_dead_jobs(redis)
|
||||
return {"cleared": count}
|
||||
|
||||
@router.post("/clients/{client_id}/toggle")
|
||||
async def toggle_client(
|
||||
client_id: str,
|
||||
|
||||
+37
-17
@@ -23,8 +23,9 @@ from app.schemas import (
|
||||
TagsResponse, ReprocessResponse,
|
||||
)
|
||||
from app.services import storage
|
||||
from app.middleware import limiter, get_upload_rate_limit
|
||||
from app.middleware import limiter, dynamic_upload_limit
|
||||
from app.workers.image_worker import QUEUE_STANDARD, QUEUE_PREMIUM
|
||||
from app.workers.arq_fallback import is_fallback_pool
|
||||
from app.metrics import (
|
||||
hub_images_uploaded, hub_images_deleted, hub_storage_used_bytes, hub_arq_jobs_enqueued
|
||||
)
|
||||
@@ -57,12 +58,6 @@ async def get_image_or_404(
|
||||
return image
|
||||
|
||||
|
||||
def _dynamic_upload_limit(key: str) -> str:
|
||||
"""Retourne la limite dynamique basée sur le plan du client."""
|
||||
# On parse le plan depuis la clé ou le state — fallback free
|
||||
return get_upload_rate_limit("free")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# UPLOAD
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
@@ -75,7 +70,7 @@ def _dynamic_upload_limit(key: str) -> str:
|
||||
description="Upload une image, lance automatiquement le pipeline AI (EXIF + OCR + Vision).",
|
||||
dependencies=[Depends(require_scope("images:write"))],
|
||||
)
|
||||
@limiter.limit("500/hour")
|
||||
@limiter.limit(dynamic_upload_limit)
|
||||
async def upload_image(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
@@ -102,23 +97,49 @@ async def upload_image(
|
||||
file_size = file_data.get("file_size", 0)
|
||||
client.storage_used_bytes = (client.storage_used_bytes or 0) + file_size
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(image)
|
||||
try:
|
||||
await db.commit()
|
||||
await db.refresh(image)
|
||||
except Exception:
|
||||
# Rollback : supprimer les fichiers uploadés car la BDD n'a pas persisté
|
||||
await storage.delete_files(
|
||||
file_data["file_path"],
|
||||
file_data.get("thumbnail_path"),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Échec de l'enregistrement — veuillez réessayer",
|
||||
)
|
||||
|
||||
# Enqueue dans ARQ
|
||||
# Enqueue dans ARQ — détecter le fallback
|
||||
arq_pool = request.app.state.arq_pool
|
||||
await arq_pool.enqueue_job("process_image_task", image.id, str(client.id))
|
||||
job_enqueued = True
|
||||
if is_fallback_pool(arq_pool):
|
||||
job_enqueued = False
|
||||
logger.warning("upload.arq_fallback", extra={
|
||||
"image_id": image.id,
|
||||
"client_id": client.id,
|
||||
})
|
||||
else:
|
||||
await arq_pool.enqueue_job("process_image_task", image.id, str(client.id))
|
||||
|
||||
# Metrics
|
||||
hub_images_uploaded.labels(client_id=client.id).inc()
|
||||
hub_storage_used_bytes.labels(client_id=client.id).set(client.storage_used_bytes)
|
||||
hub_arq_jobs_enqueued.labels(queue="default").inc()
|
||||
if job_enqueued:
|
||||
hub_arq_jobs_enqueued.labels(queue="default").inc()
|
||||
|
||||
return UploadResponse(
|
||||
id=image.id,
|
||||
uuid=image.uuid,
|
||||
original_name=image.original_name,
|
||||
status=image.processing_status,
|
||||
message=(
|
||||
"Image uploadée — traitement AI en cours"
|
||||
if job_enqueued
|
||||
else "Image uploadée — le pipeline AI est indisponible (Redis/ARQ down). "
|
||||
"Le traitement sera effectué automatiquement dès le rétablissement."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -397,7 +418,7 @@ async def get_all_tags(
|
||||
description="Reprocess une image existante (utile après changement de modèle AI).",
|
||||
dependencies=[Depends(require_scope("images:write"))],
|
||||
)
|
||||
@limiter.limit("500/hour")
|
||||
@limiter.limit(dynamic_upload_limit)
|
||||
async def reprocess_image(
|
||||
request: Request,
|
||||
image_id: int,
|
||||
@@ -444,8 +465,8 @@ async def delete_image(
|
||||
file_size = image.file_size or 0
|
||||
client.storage_used_bytes = max(0, (client.storage_used_bytes or 0) - file_size)
|
||||
|
||||
# Suppression des fichiers sur disque
|
||||
storage.delete_files(image.file_path, image.thumbnail_path)
|
||||
# Suppression des fichiers — maintenant async et déterministe
|
||||
await storage.delete_files(image.file_path, image.thumbnail_path)
|
||||
|
||||
await db.delete(image)
|
||||
await db.commit()
|
||||
@@ -503,4 +524,3 @@ async def get_thumbnail_url(
|
||||
backend = get_storage_backend()
|
||||
url = await backend.get_signed_url(image.thumbnail_path, expires_in=expires_in)
|
||||
return {"url": url, "expires_in": expires_in}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user