Add comprehensive test suite for image processing and related services
- Implement tests for database generator to ensure proper session handling. - Create tests for EXIF extraction and conversion functions. - Add tests for image-related endpoints, ensuring proper data retrieval and isolation between clients. - Develop tests for OCR functionality, including language detection and text extraction. - Introduce tests for the image processing pipeline, covering success and failure scenarios. - Validate rate limiting functionality and ensure independent counters for different clients. - Implement scraper tests to verify HTML content fetching and error handling. - Add unit tests for various services, including storage and filename generation. - Establish worker entry point for ARQ to handle background image processing tasks.
This commit is contained in:
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Router — Images : upload, lecture, suppression, retraitement
|
||||
Sécurisé : authentification par API Key + isolation par client_id.
|
||||
"""
|
||||
import logging
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import (
|
||||
APIRouter, Depends, HTTPException, UploadFile, File,
|
||||
Request, status, Query,
|
||||
)
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func, or_
|
||||
|
||||
from app.database import get_db
|
||||
from app.dependencies.auth import get_current_client, require_scope
|
||||
from app.models.client import APIClient
|
||||
from app.models.image import Image, ProcessingStatus
|
||||
from app.schemas import (
|
||||
UploadResponse, ImageDetail, ImageSummary,
|
||||
StatusResponse, PaginatedImages, DeleteResponse,
|
||||
TagsResponse, ReprocessResponse,
|
||||
)
|
||||
from app.services import storage
|
||||
from app.middleware import limiter, get_upload_rate_limit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/images", tags=["Images"])
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# UTILITAIRE : récupérer une image avec isolation client
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_image_or_404(
|
||||
image_id: int, client_id: str, db: AsyncSession
|
||||
) -> Image:
|
||||
"""
|
||||
Récupère une image par ID en vérifiant qu'elle appartient au client.
|
||||
Lève HTTP 404 si introuvable ou si elle n'appartient pas au client.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(Image).where(
|
||||
Image.id == image_id,
|
||||
Image.client_id == client_id,
|
||||
)
|
||||
)
|
||||
image = result.scalar_one_or_none()
|
||||
if not image:
|
||||
raise HTTPException(status_code=404, detail="Image introuvable")
|
||||
return image
|
||||
|
||||
|
||||
def _dynamic_upload_limit(key: str) -> str:
|
||||
"""Retourne la limite dynamique basée sur le plan du client."""
|
||||
# On parse le plan depuis la clé ou le state — fallback free
|
||||
return get_upload_rate_limit("free")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# UPLOAD
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post(
|
||||
"/upload",
|
||||
response_model=UploadResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
summary="Uploader une image",
|
||||
description="Upload une image, lance automatiquement le pipeline AI (EXIF + OCR + Vision).",
|
||||
dependencies=[Depends(require_scope("images:write"))],
|
||||
)
|
||||
@limiter.limit("500/hour")
|
||||
async def upload_image(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
# Vérification quota avant upload
|
||||
quota_mb = client.quota_storage_mb or 500
|
||||
used_bytes = client.storage_used_bytes or 0
|
||||
if used_bytes >= quota_mb * 1024 * 1024:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail=f"Quota de stockage dépassé ({quota_mb} MB)",
|
||||
)
|
||||
|
||||
# Sauvegarde fichier + thumbnail (isolé par client_id)
|
||||
file_data = await storage.save_upload(file, client_id=client.id)
|
||||
|
||||
# Création de l'enregistrement BDD
|
||||
image = Image(**file_data)
|
||||
db.add(image)
|
||||
|
||||
# Mise à jour du quota
|
||||
file_size = file_data.get("file_size", 0)
|
||||
client.storage_used_bytes = (client.storage_used_bytes or 0) + file_size
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(image)
|
||||
|
||||
# Enqueue dans ARQ (persistant, avec retry)
|
||||
arq_pool = request.app.state.arq_pool
|
||||
queue_name = "premium" if client.plan and client.plan.value == "premium" else "standard"
|
||||
await arq_pool.enqueue_job(
|
||||
"process_image_task",
|
||||
image.id,
|
||||
str(client.id),
|
||||
_queue_name=queue_name,
|
||||
)
|
||||
|
||||
return UploadResponse(
|
||||
id=image.id,
|
||||
uuid=image.uuid,
|
||||
original_name=image.original_name,
|
||||
status=image.processing_status,
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# LISTE
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=PaginatedImages,
|
||||
summary="Lister les images",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def list_images(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
tag: Optional[str] = Query(None, description="Filtrer par tag AI"),
|
||||
status_filter: Optional[ProcessingStatus] = Query(None, alias="status"),
|
||||
search: Optional[str] = Query(None, description="Recherche dans description et OCR"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
# Filtre d'isolation par client
|
||||
query = select(Image).where(Image.client_id == client.id)
|
||||
|
||||
if status_filter:
|
||||
query = query.where(Image.processing_status == status_filter)
|
||||
|
||||
if tag:
|
||||
query = query.where(Image.ai_tags.contains([tag]))
|
||||
|
||||
if search:
|
||||
query = query.where(
|
||||
or_(
|
||||
Image.ai_description.ilike(f"%{search}%"),
|
||||
Image.ocr_text.ilike(f"%{search}%"),
|
||||
Image.original_name.ilike(f"%{search}%"),
|
||||
)
|
||||
)
|
||||
|
||||
# Count total
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total_result = await db.execute(count_query)
|
||||
total = total_result.scalar_one()
|
||||
|
||||
# Pagination
|
||||
offset = (page - 1) * page_size
|
||||
query = query.order_by(Image.uploaded_at.desc()).offset(offset).limit(page_size)
|
||||
result = await db.execute(query)
|
||||
images = result.scalars().all()
|
||||
|
||||
# Quota info
|
||||
used_mb = round((client.storage_used_bytes or 0) / (1024 * 1024), 2)
|
||||
quota_mb = client.quota_storage_mb or 500
|
||||
pct = round(used_mb / quota_mb * 100, 1) if quota_mb > 0 else 0.0
|
||||
|
||||
return PaginatedImages(
|
||||
total=total,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
pages=math.ceil(total / page_size) if total else 0,
|
||||
storage_used_mb=used_mb,
|
||||
storage_quota_mb=quota_mb,
|
||||
quota_pct=pct,
|
||||
items=[
|
||||
ImageSummary(
|
||||
id=img.id,
|
||||
uuid=img.uuid,
|
||||
original_name=img.original_name,
|
||||
mime_type=img.mime_type,
|
||||
file_size=img.file_size,
|
||||
width=img.width,
|
||||
height=img.height,
|
||||
uploaded_at=img.uploaded_at,
|
||||
processing_status=img.processing_status,
|
||||
ai_tags=img.ai_tags,
|
||||
ai_description=img.ai_description,
|
||||
thumbnail_path=img.thumbnail_path,
|
||||
)
|
||||
for img in images
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# DÉTAIL COMPLET
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/{image_id}",
|
||||
response_model=ImageDetail,
|
||||
summary="Détail complet d'une image",
|
||||
description="Retourne toutes les données : fichier, EXIF, OCR et résultats AI.",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_image(
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
return ImageDetail.from_orm_full(image)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# STATUT DU PIPELINE
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/{image_id}/status",
|
||||
response_model=StatusResponse,
|
||||
summary="Statut du traitement AI",
|
||||
description="Permet de poller l'avancement du pipeline (pending → processing → done/error).",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_status(
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
|
||||
return StatusResponse(
|
||||
id=image.id,
|
||||
uuid=image.uuid,
|
||||
status=image.processing_status,
|
||||
error=image.processing_error,
|
||||
started_at=image.processing_started_at,
|
||||
done_at=image.processing_done_at,
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# DONNÉES EXIF
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/{image_id}/exif",
|
||||
summary="Métadonnées EXIF de l'image",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_exif(
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
|
||||
return {
|
||||
"id": image.id,
|
||||
"camera": {
|
||||
"make": image.exif_make,
|
||||
"model": image.exif_model,
|
||||
"lens": image.exif_lens,
|
||||
"iso": image.exif_iso,
|
||||
"aperture": image.exif_aperture,
|
||||
"shutter_speed": image.exif_shutter,
|
||||
"focal_length": image.exif_focal,
|
||||
"flash": image.exif_flash,
|
||||
"orientation": image.exif_orientation,
|
||||
"software": image.exif_software,
|
||||
"taken_at": image.exif_taken_at,
|
||||
},
|
||||
"gps": {
|
||||
"latitude": image.exif_gps_lat,
|
||||
"longitude": image.exif_gps_lon,
|
||||
"altitude": image.exif_altitude,
|
||||
"has_gps": image.has_gps,
|
||||
"maps_url": (
|
||||
f"https://maps.google.com/?q={image.exif_gps_lat},{image.exif_gps_lon}"
|
||||
if image.has_gps else None
|
||||
),
|
||||
},
|
||||
"raw": image.exif_raw,
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# DONNÉES OCR
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/{image_id}/ocr",
|
||||
summary="Texte extrait de l'image (OCR)",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_ocr(
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
|
||||
return {
|
||||
"id": image.id,
|
||||
"has_text": image.ocr_has_text,
|
||||
"text": image.ocr_text,
|
||||
"language": image.ocr_language,
|
||||
"confidence": image.ocr_confidence,
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# DONNÉES AI
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/{image_id}/ai",
|
||||
summary="Résultats AI (description + tags)",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_ai(
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
|
||||
return {
|
||||
"id": image.id,
|
||||
"description": image.ai_description,
|
||||
"tags": image.ai_tags,
|
||||
"confidence": image.ai_confidence,
|
||||
"model_used": image.ai_model_used,
|
||||
"processed_at": image.ai_processed_at,
|
||||
"tokens": {
|
||||
"prompt": image.ai_prompt_tokens,
|
||||
"output": image.ai_output_tokens,
|
||||
"total": (image.ai_prompt_tokens or 0) + (image.ai_output_tokens or 0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# TAGS — Vue globale (filtrée par client)
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/tags/all",
|
||||
response_model=TagsResponse,
|
||||
summary="Tous les tags utilisés",
|
||||
description="Liste dédupliquée de tous les tags AI générés sur les images du client.",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_all_tags(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
result = await db.execute(
|
||||
select(Image.ai_tags).where(
|
||||
Image.client_id == client.id,
|
||||
Image.ai_tags.isnot(None),
|
||||
)
|
||||
)
|
||||
all_tag_lists = result.scalars().all()
|
||||
|
||||
unique_tags = sorted(set(
|
||||
tag
|
||||
for tag_list in all_tag_lists
|
||||
for tag in (tag_list or [])
|
||||
))
|
||||
|
||||
return TagsResponse(tags=unique_tags, total=len(unique_tags))
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# RETRAITEMENT AI
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post(
|
||||
"/{image_id}/reprocess",
|
||||
response_model=ReprocessResponse,
|
||||
summary="Relancer le pipeline AI",
|
||||
description="Reprocess une image existante (utile après changement de modèle AI).",
|
||||
dependencies=[Depends(require_scope("images:write"))],
|
||||
)
|
||||
@limiter.limit("500/hour")
|
||||
async def reprocess_image(
|
||||
request: Request,
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
|
||||
# Reset du statut
|
||||
image.processing_status = ProcessingStatus.PENDING
|
||||
image.processing_error = None
|
||||
image.processing_started_at = None
|
||||
image.processing_done_at = None
|
||||
await db.commit()
|
||||
|
||||
# Enqueue dans ARQ
|
||||
arq_pool = request.app.state.arq_pool
|
||||
queue_name = "premium" if client.plan and client.plan.value == "premium" else "standard"
|
||||
await arq_pool.enqueue_job(
|
||||
"process_image_task",
|
||||
image_id,
|
||||
str(client.id),
|
||||
_queue_name=queue_name,
|
||||
)
|
||||
|
||||
return ReprocessResponse(id=image_id)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# SUPPRESSION
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.delete(
|
||||
"/{image_id}",
|
||||
response_model=DeleteResponse,
|
||||
summary="Supprimer une image",
|
||||
dependencies=[Depends(require_scope("images:delete"))],
|
||||
)
|
||||
async def delete_image(
|
||||
image_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
|
||||
# Décrémentation du quota
|
||||
file_size = image.file_size or 0
|
||||
client.storage_used_bytes = max(0, (client.storage_used_bytes or 0) - file_size)
|
||||
|
||||
# Suppression des fichiers sur disque
|
||||
storage.delete_files(image.file_path, image.thumbnail_path)
|
||||
|
||||
await db.delete(image)
|
||||
await db.commit()
|
||||
|
||||
return DeleteResponse(deleted_id=image_id)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# URLs SIGNÉES
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get(
|
||||
"/{image_id}/download-url",
|
||||
summary="URL signée de téléchargement",
|
||||
description="Retourne une URL signée temporaire pour télécharger l'image originale.",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_download_url(
|
||||
image_id: int,
|
||||
expires_in: int = Query(900, ge=60, le=86400, description="Durée de validité en secondes"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
from app.services.storage_backend import get_storage_backend
|
||||
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
backend = get_storage_backend()
|
||||
|
||||
url = await backend.get_signed_url(image.file_path, expires_in=expires_in)
|
||||
return {"url": url, "expires_in": expires_in}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{image_id}/thumbnail-url",
|
||||
summary="URL signée du thumbnail",
|
||||
description="Retourne une URL signée temporaire pour le thumbnail de l'image.",
|
||||
dependencies=[Depends(require_scope("images:read"))],
|
||||
)
|
||||
async def get_thumbnail_url(
|
||||
image_id: int,
|
||||
expires_in: int = Query(900, ge=60, le=86400, description="Durée de validité en secondes"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
client: APIClient = Depends(get_current_client),
|
||||
):
|
||||
from app.services.storage_backend import get_storage_backend
|
||||
|
||||
image = await get_image_or_404(image_id, client.id, db)
|
||||
if not image.thumbnail_path:
|
||||
raise HTTPException(status_code=404, detail="Thumbnail non disponible")
|
||||
|
||||
backend = get_storage_backend()
|
||||
url = await backend.get_signed_url(image.thumbnail_path, expires_in=expires_in)
|
||||
return {"url": url, "expires_in": expires_in}
|
||||
|
||||
Reference in New Issue
Block a user