Replace manual upsert logic with SQLite native upsert in Docker CRUD repositories, enhance Ansible backup playbook with better error handling and file permissions, add favicon endpoint, and improve playbook editor UI with syntax highlighting, lint integration, quality badges, and enhanced code editing features
Tests / Backend Tests (Python) (3.10) (push) Has been cancelled
Tests / Backend Tests (Python) (3.11) (push) Has been cancelled
Tests / Backend Tests (Python) (3.12) (push) Has been cancelled
Tests / Frontend Tests (JS) (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Tests / All Tests Passed (push) Has been cancelled
Tests / Backend Tests (Python) (3.10) (push) Has been cancelled
Tests / Backend Tests (Python) (3.11) (push) Has been cancelled
Tests / Backend Tests (Python) (3.12) (push) Has been cancelled
Tests / Frontend Tests (JS) (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Tests / All Tests Passed (push) Has been cancelled
This commit is contained in:
@@ -25,6 +25,7 @@ from app.routes.builtin_playbooks import router as builtin_playbooks_router
|
||||
from app.routes.server import router as server_router
|
||||
from app.routes.alerts import router as alerts_router
|
||||
from app.routes.docker import router as docker_router
|
||||
from app.routes.lint import router as lint_router
|
||||
|
||||
# Router principal qui agrège tous les sous-routers
|
||||
api_router = APIRouter()
|
||||
@@ -48,6 +49,7 @@ api_router.include_router(builtin_playbooks_router, prefix="/builtin-playbooks",
|
||||
api_router.include_router(server_router, prefix="/server", tags=["Server"])
|
||||
api_router.include_router(alerts_router, prefix="/alerts", tags=["Alerts"])
|
||||
api_router.include_router(docker_router, prefix="/docker", tags=["Docker"])
|
||||
api_router.include_router(lint_router, prefix="/playbooks", tags=["Lint"])
|
||||
|
||||
__all__ = [
|
||||
"api_router",
|
||||
|
||||
@@ -7,6 +7,7 @@ import uuid
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
||||
import logging
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
@@ -24,6 +25,7 @@ from app.services.builtin_playbooks import (
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("homelab.metrics")
|
||||
|
||||
|
||||
async def _sync_hosts_from_inventory(db_session: AsyncSession) -> int:
|
||||
@@ -202,11 +204,40 @@ async def collect_all_metrics(
|
||||
|
||||
hosts_bg = await host_repo_bg.list(limit=1000)
|
||||
|
||||
inventory_hosts = ansible_service.get_hosts_from_inventory()
|
||||
inventory_names = {h.name for h in (inventory_hosts or []) if getattr(h, "name", None)}
|
||||
skipped = 0
|
||||
if inventory_names:
|
||||
filtered = [h for h in hosts_bg if h.name in inventory_names]
|
||||
skipped = len(hosts_bg) - len(filtered)
|
||||
hosts_bg = filtered
|
||||
|
||||
for host in hosts_bg:
|
||||
try:
|
||||
result = await service.execute_builtin("collect_system_info", host.name)
|
||||
ok = bool(result.get("success", False))
|
||||
|
||||
stderr = (result.get("stderr") or "").strip()
|
||||
stdout = (result.get("stdout") or "").strip()
|
||||
return_code = result.get("return_code")
|
||||
|
||||
error_detail = None
|
||||
if not ok:
|
||||
# Keep payload small but actionable
|
||||
if stderr:
|
||||
error_detail = stderr[:400]
|
||||
elif stdout:
|
||||
error_detail = stdout[:400]
|
||||
else:
|
||||
error_detail = result.get("error") or "Unknown error"
|
||||
|
||||
logger.warning(
|
||||
"[METRICS] Collection failed for host=%s return_code=%s error=%s",
|
||||
host.name,
|
||||
return_code,
|
||||
error_detail,
|
||||
)
|
||||
|
||||
if ok:
|
||||
for hostname, metrics_data in result.get("parsed_metrics", {}).items():
|
||||
target_host = await host_repo_bg.get_by_name(hostname)
|
||||
@@ -226,9 +257,12 @@ async def collect_all_metrics(
|
||||
results.append({
|
||||
"host": host.name,
|
||||
"success": ok,
|
||||
"return_code": return_code,
|
||||
"error": error_detail,
|
||||
})
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.exception("[METRICS] Unhandled exception during collection for host=%s", host.name)
|
||||
results.append({
|
||||
"host": host.name,
|
||||
"success": False,
|
||||
@@ -240,7 +274,9 @@ async def collect_all_metrics(
|
||||
"data": {
|
||||
"total": len(results),
|
||||
"success": sum(1 for r in results if r.get("success")),
|
||||
"failed": sum(1 for r in results if not r.get("success"))
|
||||
"failed": sum(1 for r in results if not r.get("success")),
|
||||
"skipped": skipped,
|
||||
"results": results,
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
"""
|
||||
Routes API pour l'intégration ansible-lint.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from app.core.dependencies import verify_api_key
|
||||
from app.schemas.lint import (
|
||||
LintRequest,
|
||||
LintResponse,
|
||||
LintIssue,
|
||||
LintSummary,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("homelab.lint")
|
||||
|
||||
# Mapping des tags ansible-lint vers nos sévérités
|
||||
SEVERITY_MAP = {
|
||||
"blocker": "error",
|
||||
"critical": "error",
|
||||
"major": "error",
|
||||
"minor": "warning",
|
||||
"info": "info",
|
||||
# Fallback par type de règle
|
||||
"error": "error",
|
||||
"warning": "warning",
|
||||
}
|
||||
|
||||
# Suggestions de fix pour certaines règles courantes
|
||||
FIX_SUGGESTIONS = {
|
||||
"risky-file-permissions": "Ajouter 'mode: \"0644\"' ou le mode approprié",
|
||||
"no-changed-when": "Ajouter 'changed_when: false' ou une condition appropriée",
|
||||
"name[missing]": "Ajouter un 'name:' descriptif à cette tâche",
|
||||
"yaml[truthy]": "Remplacer 'yes/no' par 'true/false'",
|
||||
"fqcn[action-core]": "Utiliser le nom complet du module (ex: ansible.builtin.copy)",
|
||||
"fqcn[action]": "Utiliser le nom complet du module avec sa collection",
|
||||
"no-free-form": "Utiliser la syntaxe structurée au lieu de la forme libre",
|
||||
"command-instead-of-shell": "Utiliser ansible.builtin.command au lieu de shell",
|
||||
"package-latest": "Spécifier une version au lieu de 'state: latest'",
|
||||
"no-handler": "Ajouter un handler avec notify pour cette tâche",
|
||||
"literal-compare": "Utiliser 'when: var' au lieu de 'when: var == true'",
|
||||
"empty-string-compare": "Utiliser 'when: var | length > 0' au lieu de comparaison vide",
|
||||
"risky-shell-pipe": "Ajouter 'set -o pipefail' ou gérer les erreurs de pipe",
|
||||
"no-jinja-when": "Retirer les accolades Jinja dans la clause when",
|
||||
}
|
||||
|
||||
# URLs de documentation pour les règles
|
||||
HELP_URLS = {
|
||||
"risky-file-permissions": "https://ansible.readthedocs.io/projects/lint/rules/risky-file-permissions/",
|
||||
"no-changed-when": "https://ansible.readthedocs.io/projects/lint/rules/no-changed-when/",
|
||||
"name": "https://ansible.readthedocs.io/projects/lint/rules/name/",
|
||||
"yaml": "https://ansible.readthedocs.io/projects/lint/rules/yaml/",
|
||||
"fqcn": "https://ansible.readthedocs.io/projects/lint/rules/fqcn/",
|
||||
}
|
||||
|
||||
|
||||
def calculate_quality_score(issues: list[LintIssue], line_count: int) -> int:
|
||||
"""
|
||||
Calcule un score de qualité 0-100 basé sur les problèmes détectés.
|
||||
|
||||
Formule:
|
||||
- Base: 100 points
|
||||
- Erreur: -15 points
|
||||
- Warning: -5 points
|
||||
- Info: -1 point
|
||||
- Bonus: +5 si aucune erreur, +3 si aucun warning
|
||||
- Minimum: 0, Maximum: 100
|
||||
"""
|
||||
score = 100
|
||||
|
||||
errors = sum(1 for i in issues if i.severity == "error")
|
||||
warnings = sum(1 for i in issues if i.severity == "warning")
|
||||
infos = sum(1 for i in issues if i.severity == "info")
|
||||
|
||||
score -= errors * 15
|
||||
score -= warnings * 5
|
||||
score -= infos * 1
|
||||
|
||||
# Bonus pour code propre
|
||||
if errors == 0:
|
||||
score = min(100, score + 5)
|
||||
if warnings == 0:
|
||||
score = min(100, score + 3)
|
||||
|
||||
# Normalisation relative à la taille du fichier (playbooks plus longs tolèrent plus d'issues)
|
||||
if line_count > 50:
|
||||
tolerance_factor = min(1.2, 1 + (line_count - 50) / 200)
|
||||
score = int(score * tolerance_factor)
|
||||
|
||||
return max(0, min(100, score))
|
||||
|
||||
|
||||
def get_severity_from_tag(tag: str) -> str:
|
||||
"""Détermine la sévérité à partir du tag ansible-lint."""
|
||||
tag_lower = tag.lower()
|
||||
return SEVERITY_MAP.get(tag_lower, "warning")
|
||||
|
||||
|
||||
def get_help_url(rule_id: str) -> Optional[str]:
|
||||
"""Retourne l'URL de documentation pour une règle."""
|
||||
# Essayer le rule_id complet d'abord
|
||||
if rule_id in HELP_URLS:
|
||||
return HELP_URLS[rule_id]
|
||||
|
||||
# Essayer la partie principale (avant les crochets)
|
||||
base_rule = rule_id.split("[")[0]
|
||||
if base_rule in HELP_URLS:
|
||||
return HELP_URLS[base_rule]
|
||||
|
||||
# URL générique ansible-lint
|
||||
return f"https://ansible.readthedocs.io/projects/lint/rules/{base_rule}/"
|
||||
|
||||
|
||||
def get_fix_suggestion(rule_id: str) -> Optional[str]:
|
||||
"""Retourne une suggestion de fix pour une règle."""
|
||||
if rule_id in FIX_SUGGESTIONS:
|
||||
return FIX_SUGGESTIONS[rule_id]
|
||||
|
||||
# Essayer la partie principale
|
||||
base_rule = rule_id.split("[")[0]
|
||||
return FIX_SUGGESTIONS.get(base_rule)
|
||||
|
||||
|
||||
def parse_ansible_lint_json(stdout: str, original_filename: str) -> list[LintIssue]:
|
||||
"""
|
||||
Parse la sortie JSON de ansible-lint.
|
||||
|
||||
Format attendu (ansible-lint >= 6.0):
|
||||
[
|
||||
{
|
||||
"type": "issue",
|
||||
"check_name": "name[missing]",
|
||||
"categories": ["idiom"],
|
||||
"severity": "minor",
|
||||
"description": "All tasks should be named.",
|
||||
"fingerprint": "...",
|
||||
"location": {
|
||||
"path": "playbook.yml",
|
||||
"lines": {"begin": 10}
|
||||
}
|
||||
}
|
||||
]
|
||||
"""
|
||||
issues = []
|
||||
|
||||
if not stdout.strip():
|
||||
return issues
|
||||
|
||||
try:
|
||||
# ansible-lint peut retourner une liste ou un objet
|
||||
data = json.loads(stdout)
|
||||
|
||||
if isinstance(data, dict):
|
||||
# Nouveau format avec clé "results" ou autre
|
||||
items = data.get("results", data.get("issues", []))
|
||||
elif isinstance(data, list):
|
||||
items = data
|
||||
else:
|
||||
logger.warning("Format ansible-lint inattendu: %s", type(data))
|
||||
return issues
|
||||
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
# Extraire les informations
|
||||
rule_id = item.get("check_name") or item.get("rule", {}).get("id", "unknown")
|
||||
|
||||
# Déterminer la sévérité
|
||||
severity_raw = item.get("severity", "minor")
|
||||
severity = get_severity_from_tag(severity_raw)
|
||||
|
||||
# Message
|
||||
message = (
|
||||
item.get("description") or
|
||||
item.get("message") or
|
||||
item.get("rule", {}).get("description", "Issue detected")
|
||||
)
|
||||
|
||||
# Location
|
||||
location = item.get("location", {})
|
||||
lines = location.get("lines", {})
|
||||
line = lines.get("begin", 1) if isinstance(lines, dict) else 1
|
||||
|
||||
# Fallback pour anciens formats
|
||||
if line == 1 and "linenumber" in item:
|
||||
line = item["linenumber"]
|
||||
|
||||
column = location.get("positions", {}).get("begin", {}).get("column", 1)
|
||||
if column == 1 and "column" in item:
|
||||
column = item["column"]
|
||||
|
||||
# Contexte
|
||||
context = item.get("content", {}).get("body") if isinstance(item.get("content"), dict) else None
|
||||
|
||||
issue = LintIssue(
|
||||
rule_id=rule_id,
|
||||
severity=severity,
|
||||
message=message,
|
||||
line=line,
|
||||
column=column,
|
||||
context=context,
|
||||
help_url=get_help_url(rule_id),
|
||||
fix_suggestion=get_fix_suggestion(rule_id),
|
||||
)
|
||||
issues.append(issue)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("Impossible de parser JSON ansible-lint: %s", e)
|
||||
# Fallback: essayer de parser le format texte
|
||||
issues = parse_ansible_lint_text(stdout, original_filename)
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def parse_ansible_lint_text(stdout: str, original_filename: str) -> list[LintIssue]:
|
||||
"""
|
||||
Parse la sortie texte de ansible-lint (fallback).
|
||||
|
||||
Format typique:
|
||||
playbook.yml:10: name[missing]: All tasks should be named.
|
||||
"""
|
||||
issues = []
|
||||
|
||||
# Pattern: filename:line: rule: message
|
||||
# ou: filename:line:col: rule: message
|
||||
pattern = re.compile(
|
||||
r'^(?P<file>[^:]+):(?P<line>\d+)(?::(?P<col>\d+))?:\s*(?P<rule>[^:]+):\s*(?P<message>.+)$'
|
||||
)
|
||||
|
||||
for line in stdout.strip().split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
match = pattern.match(line)
|
||||
if match:
|
||||
rule_id = match.group('rule').strip()
|
||||
|
||||
# Déterminer sévérité basée sur le nom de la règle
|
||||
severity = "warning"
|
||||
if any(x in rule_id.lower() for x in ['error', 'fatal', 'risky']):
|
||||
severity = "error"
|
||||
elif any(x in rule_id.lower() for x in ['info', 'hint']):
|
||||
severity = "info"
|
||||
|
||||
issue = LintIssue(
|
||||
rule_id=rule_id,
|
||||
severity=severity,
|
||||
message=match.group('message').strip(),
|
||||
line=int(match.group('line')),
|
||||
column=int(match.group('col') or 1),
|
||||
help_url=get_help_url(rule_id),
|
||||
fix_suggestion=get_fix_suggestion(rule_id),
|
||||
)
|
||||
issues.append(issue)
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
@router.post("/{filename}/lint", response_model=LintResponse)
|
||||
async def lint_playbook(
|
||||
filename: str,
|
||||
request: LintRequest,
|
||||
api_key_valid: bool = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Exécute ansible-lint sur le contenu d'un playbook.
|
||||
|
||||
Retourne les problèmes détectés avec leur sévérité, position,
|
||||
et suggestions de correction.
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
# Valider le filename
|
||||
if not filename.endswith(('.yml', '.yaml')):
|
||||
raise HTTPException(status_code=400, detail="Extension invalide")
|
||||
|
||||
# Créer fichier temporaire pour ansible-lint
|
||||
temp_file = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode='w',
|
||||
suffix='.yml',
|
||||
delete=False,
|
||||
encoding='utf-8'
|
||||
) as f:
|
||||
f.write(request.content)
|
||||
temp_file = Path(f.name)
|
||||
|
||||
# Construire la commande ansible-lint
|
||||
cmd = [
|
||||
'ansible-lint',
|
||||
'--format', 'json',
|
||||
'--nocolor',
|
||||
'--offline', # Ne pas télécharger de rôles Galaxy
|
||||
'-q', # Quiet mode
|
||||
]
|
||||
|
||||
# Ajouter les règles à ignorer
|
||||
if request.skip_rules:
|
||||
skip_list = ','.join(request.skip_rules)
|
||||
cmd.extend(['--skip-list', skip_list])
|
||||
|
||||
# Fichier à analyser
|
||||
cmd.append(str(temp_file))
|
||||
|
||||
logger.debug("Executing ansible-lint: %s", ' '.join(cmd))
|
||||
|
||||
# Exécuter ansible-lint
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=temp_file.parent
|
||||
)
|
||||
|
||||
# Timeout de 30 secondes
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=30.0
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
raise HTTPException(
|
||||
status_code=504,
|
||||
detail="ansible-lint timeout (>30s)"
|
||||
)
|
||||
|
||||
stdout_str = stdout.decode('utf-8', errors='replace')
|
||||
stderr_str = stderr.decode('utf-8', errors='replace')
|
||||
|
||||
logger.debug("ansible-lint return code: %d", process.returncode)
|
||||
logger.debug("ansible-lint stdout: %s", stdout_str[:500] if stdout_str else "(empty)")
|
||||
|
||||
# Parser les résultats
|
||||
# Note: ansible-lint retourne un code non-zero s'il trouve des problèmes
|
||||
# ce qui est normal et ne doit pas être traité comme une erreur
|
||||
issues = parse_ansible_lint_json(stdout_str, filename)
|
||||
|
||||
# Si pas de JSON, essayer stderr (certaines versions)
|
||||
if not issues and stderr_str:
|
||||
issues = parse_ansible_lint_text(stderr_str, filename)
|
||||
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="ansible-lint non disponible sur ce serveur"
|
||||
)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions (like timeout 504)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception("Erreur exécution ansible-lint")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Erreur ansible-lint: {str(e)}"
|
||||
)
|
||||
|
||||
# Calculer le score de qualité
|
||||
line_count = len(request.content.split('\n'))
|
||||
quality_score = calculate_quality_score(issues, line_count)
|
||||
|
||||
# Calculer le temps d'exécution
|
||||
execution_time_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
# Construire la réponse
|
||||
summary = LintSummary(
|
||||
total=len(issues),
|
||||
errors=sum(1 for i in issues if i.severity == "error"),
|
||||
warnings=sum(1 for i in issues if i.severity == "warning"),
|
||||
info=sum(1 for i in issues if i.severity == "info"),
|
||||
)
|
||||
|
||||
return LintResponse(
|
||||
success=True,
|
||||
execution_time_ms=execution_time_ms,
|
||||
summary=summary,
|
||||
quality_score=quality_score,
|
||||
issues=issues,
|
||||
)
|
||||
|
||||
finally:
|
||||
# Nettoyer le fichier temporaire
|
||||
if temp_file and temp_file.exists():
|
||||
try:
|
||||
temp_file.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@router.get("/rules")
|
||||
async def list_lint_rules(api_key_valid: bool = Depends(verify_api_key)):
|
||||
"""
|
||||
Liste les règles ansible-lint disponibles.
|
||||
|
||||
Utile pour permettre à l'utilisateur de configurer les règles à ignorer.
|
||||
"""
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
'ansible-lint',
|
||||
'--list-rules',
|
||||
'--format', 'json',
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
stdout, _ = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=10.0
|
||||
)
|
||||
|
||||
stdout_str = stdout.decode('utf-8', errors='replace')
|
||||
|
||||
try:
|
||||
rules = json.loads(stdout_str)
|
||||
return {"rules": rules}
|
||||
except json.JSONDecodeError:
|
||||
# Retourner les règles courantes connues
|
||||
return {
|
||||
"rules": [
|
||||
{"id": "yaml", "description": "YAML syntax and formatting"},
|
||||
{"id": "name", "description": "Task and play naming"},
|
||||
{"id": "fqcn", "description": "Fully Qualified Collection Names"},
|
||||
{"id": "risky-file-permissions", "description": "File permissions"},
|
||||
{"id": "no-changed-when", "description": "Changed when conditions"},
|
||||
{"id": "command-instead-of-shell", "description": "Command vs shell"},
|
||||
{"id": "no-free-form", "description": "Module syntax"},
|
||||
]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Impossible de lister les règles: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Impossible de récupérer la liste des règles"
|
||||
)
|
||||
Reference in New Issue
Block a user