Add audit logging, rate limiting, secret redactor, and backlinks
Implement several security and feature improvements across the backend and frontend: - New IP-based rate limiter for authentication endpoints - New audit logging system for sensitive operations - New secret redactor to mask sensitive patterns in rendered content - Configurable token TTL and IGNORED_DIRS via environment variables - Add backlink index and API endpoint - Add preview tab support with single/double-click behavior in tree - Add file backup before write/delete operations
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
Audit logging: traces sensitive operations to data/audit.log.
|
||||
|
||||
Logs all write, delete, and configuration change operations
|
||||
with timestamp, username, IP address, action, and target.
|
||||
|
||||
Format: JSON lines (one JSON object per line) for easy parsing.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("obsigate.audit")
|
||||
|
||||
AUDIT_LOG_FILE = Path("data/audit.log")
|
||||
|
||||
# Max file size before rotation (10 MB)
|
||||
MAX_LOG_SIZE = int(os.environ.get("OBSIGATE_AUDIT_MAX_SIZE", str(10 * 1024 * 1024)))
|
||||
|
||||
|
||||
def _rotate_if_needed():
|
||||
"""Rotate audit log if it exceeds the max size."""
|
||||
if not AUDIT_LOG_FILE.exists():
|
||||
return
|
||||
if AUDIT_LOG_FILE.stat().st_size > MAX_LOG_SIZE:
|
||||
backup = AUDIT_LOG_FILE.with_suffix(".log.1")
|
||||
if backup.exists():
|
||||
backup.unlink()
|
||||
AUDIT_LOG_FILE.rename(backup)
|
||||
logger.info("Rotated audit log")
|
||||
|
||||
|
||||
def _write_entry(entry: dict):
|
||||
"""Append a JSON entry to the audit log."""
|
||||
try:
|
||||
AUDIT_LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
_rotate_if_needed()
|
||||
line = json.dumps(entry, default=str, ensure_ascii=False) + "\n"
|
||||
with open(AUDIT_LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write audit log: {e}")
|
||||
|
||||
|
||||
def log_file_save(
|
||||
username: str,
|
||||
vault_name: str,
|
||||
file_path: str,
|
||||
size: int,
|
||||
ip: Optional[str] = None,
|
||||
):
|
||||
"""Log a file save (PUT) operation."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"action": "file_save",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
"vault": vault_name,
|
||||
"path": file_path,
|
||||
"size": size,
|
||||
})
|
||||
|
||||
|
||||
def log_file_delete(
|
||||
username: str,
|
||||
vault_name: str,
|
||||
file_path: str,
|
||||
ip: Optional[str] = None,
|
||||
):
|
||||
"""Log a file delete operation."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"action": "file_delete",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
"vault": vault_name,
|
||||
"path": file_path,
|
||||
})
|
||||
|
||||
|
||||
def log_config_change(
|
||||
username: str,
|
||||
changes: dict,
|
||||
ip: Optional[str] = None,
|
||||
):
|
||||
"""Log a configuration change."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"action": "config_change",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
"changes": changes,
|
||||
})
|
||||
|
||||
|
||||
def log_vault_add(username: str, vault_name: str, vault_path: str, ip: Optional[str] = None):
|
||||
"""Log a vault addition."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"action": "vault_add",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
"vault": vault_name,
|
||||
"vault_path": vault_path,
|
||||
})
|
||||
|
||||
|
||||
def log_vault_remove(username: str, vault_name: str, ip: Optional[str] = None):
|
||||
"""Log a vault removal."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"action": "vault_remove",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
"vault": vault_name,
|
||||
})
|
||||
|
||||
|
||||
def get_recent_entries(limit: int = 100, action: Optional[str] = None) -> list:
|
||||
"""Read the most recent audit log entries.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of entries to return.
|
||||
action: Optional filter by action type.
|
||||
|
||||
Returns:
|
||||
List of audit entries (most recent first).
|
||||
"""
|
||||
if not AUDIT_LOG_FILE.exists():
|
||||
return []
|
||||
entries = []
|
||||
try:
|
||||
with open(AUDIT_LOG_FILE, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
if action and entry.get("action") != action:
|
||||
continue
|
||||
entries.append(entry)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
# Return most recent first
|
||||
entries.reverse()
|
||||
return entries[:limit]
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read audit log: {e}")
|
||||
return []
|
||||
@@ -19,8 +19,8 @@ SECRET_KEY_FILE = Path("data/secret.key")
|
||||
REVOKED_TOKENS_FILE = Path("data/revoked_tokens.json")
|
||||
|
||||
ALGORITHM = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_SECONDS = 3600 # 1 hour
|
||||
REFRESH_TOKEN_EXPIRE_SECONDS = 604800 # 7 days
|
||||
ACCESS_TOKEN_EXPIRE_SECONDS = int(os.environ.get("OBSIGATE_ACCESS_TOKEN_TTL", "3600")) # default 1 hour
|
||||
REFRESH_TOKEN_EXPIRE_SECONDS = int(os.environ.get("OBSIGATE_REFRESH_TOKEN_TTL", "604800")) # default 7 days
|
||||
|
||||
# In-memory revoked token set (loaded from disk on startup)
|
||||
_revoked_jtis: set = set()
|
||||
|
||||
@@ -19,6 +19,7 @@ from .jwt_handler import (
|
||||
)
|
||||
from .middleware import require_auth, require_admin, is_auth_enabled
|
||||
from .password import verify_password, hash_password
|
||||
from backend.ratelimit import is_rate_limited, record_failure as rl_record_failure, record_success as rl_record_success
|
||||
|
||||
logger = logging.getLogger("obsigate.auth.router")
|
||||
|
||||
@@ -105,19 +106,26 @@ async def login(request: LoginRequest, response: Response):
|
||||
if not user.get("active"):
|
||||
raise HTTPException(403, "Compte désactivé")
|
||||
|
||||
# IP-based rate limiting (10 failures / 15 min per IP)
|
||||
client_ip = request.client.host if request.client else "unknown"
|
||||
if is_rate_limited(client_ip):
|
||||
raise HTTPException(429, "Trop de tentatives depuis cette adresse IP (15min)")
|
||||
|
||||
if is_locked(request.username):
|
||||
raise HTTPException(429, "Compte temporairement verrouillé (15min)")
|
||||
|
||||
if not verify_password(request.password, user["password_hash"]):
|
||||
attempts = record_login_failure(request.username)
|
||||
rl_attempts, rl_remaining = rl_record_failure(client_ip)
|
||||
remaining = max(0, 5 - attempts)
|
||||
detail = "Identifiants invalides"
|
||||
if 0 < remaining <= 2:
|
||||
detail += f" ({remaining} tentative(s) restante(s))"
|
||||
raise HTTPException(401, detail)
|
||||
|
||||
# Success — generate tokens
|
||||
# Success — clear rate limits and generate tokens
|
||||
record_login_success(request.username)
|
||||
rl_record_success(client_ip)
|
||||
|
||||
access_token = create_access_token(user)
|
||||
refresh_token, refresh_jti = create_refresh_token(request.username)
|
||||
|
||||
@@ -30,6 +30,9 @@ _index_generation: int = 0
|
||||
# O(1) lookup table for wikilink resolution: {filename_lower: [{vault, path}, ...]}
|
||||
_file_lookup: Dict[str, List[Dict[str, str]]] = {}
|
||||
|
||||
# Backlink index: {vault_name: {relative_path: [{vault, path, title}, ...]}}
|
||||
_backlink_index: Dict[str, Dict[str, List[Dict[str, str]]]] = {}
|
||||
|
||||
# O(1) path index for tree filtering: {vault_name: [{path, name, type}, ...]}
|
||||
path_index: Dict[str, List[Dict[str, str]]] = {}
|
||||
|
||||
@@ -49,6 +52,12 @@ SUPPORTED_EXTENSIONS = {
|
||||
}
|
||||
|
||||
|
||||
# Ignored directories (configurable via OBSIGATE_IGNORED_DIRS env var)
|
||||
_DEFAULT_IGNORED = {'.obsidian', '.trash', '.git', '__pycache__', 'node_modules', '.obsigate-backup'}
|
||||
_env_ignored = os.environ.get("OBSIGATE_IGNORED_DIRS", "")
|
||||
IGNORED_DIRS = set(d.strip() for d in _env_ignored.split(",") if d.strip()) if _env_ignored else _DEFAULT_IGNORED.copy()
|
||||
|
||||
|
||||
def load_vault_config() -> Dict[str, Dict[str, Any]]:
|
||||
"""Read VAULT_N_* and DIR_N_* env vars and return vault configuration.
|
||||
|
||||
@@ -223,6 +232,10 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
|
||||
return {"files": [], "tags": {}, "path": vault_path, "paths": []}
|
||||
|
||||
for fpath in vault_root.rglob("*"):
|
||||
# Skip ignored directories
|
||||
if any(part in IGNORED_DIRS for part in fpath.relative_to(vault_root).parts):
|
||||
continue
|
||||
|
||||
rel_path_str = str(fpath.relative_to(vault_root)).replace("\\", "/")
|
||||
|
||||
# Add all paths (files and directories) to path index
|
||||
@@ -270,6 +283,12 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
|
||||
title = _extract_title(post, fpath)
|
||||
content_preview = post.content[:200].strip()
|
||||
|
||||
# Extract wikilinks for backlink index
|
||||
_extract_wikilinks_for_backlinks(
|
||||
vault_name, str(relative).replace("\\", "/"),
|
||||
title, post.content
|
||||
)
|
||||
|
||||
files.append({
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
"title": title,
|
||||
@@ -849,3 +868,71 @@ def find_file_in_index(link_target: str, current_vault: str) -> Optional[Dict[st
|
||||
if c["vault"] == current_vault:
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backlink index: tracks which files link to which targets
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_wikilinks_for_backlinks(
|
||||
vault_name: str, source_path: str, source_title: str, content: str
|
||||
):
|
||||
"""Extract wikilinks from markdown content and populate the backlink index.
|
||||
|
||||
For each `[[target]]` or `[[target|display]]` found in the content,
|
||||
adds the source file to the backlink index of the target.
|
||||
|
||||
Args:
|
||||
vault_name: The vault containing the source file.
|
||||
source_path: Relative path of the source file in the vault.
|
||||
source_title: Title of the source file.
|
||||
content: The markdown content to scan for wikilinks.
|
||||
"""
|
||||
global _backlink_index
|
||||
wikilink_pattern = re.compile(r'\[\[([^\]|#]+)(?:[|#][^\]]+)?\]\]')
|
||||
targets = set()
|
||||
for match in wikilink_pattern.finditer(content):
|
||||
target = match.group(1).strip()
|
||||
target_lower = target.lower()
|
||||
if not target_lower.endswith(".md"):
|
||||
target_lower += ".md"
|
||||
targets.add(target_lower)
|
||||
|
||||
if vault_name not in _backlink_index:
|
||||
_backlink_index[vault_name] = {}
|
||||
|
||||
for target in targets:
|
||||
if target not in _backlink_index[vault_name]:
|
||||
_backlink_index[vault_name][target] = []
|
||||
# Avoid duplicates for same source-target pair
|
||||
existing = [e for e in _backlink_index[vault_name][target] if e["path"] == source_path]
|
||||
if not existing:
|
||||
_backlink_index[vault_name][target].append({
|
||||
"vault": vault_name,
|
||||
"path": source_path,
|
||||
"title": source_title,
|
||||
})
|
||||
|
||||
|
||||
def get_backlinks(vault_name: str, file_path: str) -> List[Dict[str, str]]:
|
||||
"""Get all files that link to the given file via wikilinks.
|
||||
|
||||
Searches across all vaults for backlinks pointing to the target file.
|
||||
|
||||
Args:
|
||||
vault_name: The vault containing the target file.
|
||||
file_path: Relative path of the target file in the vault.
|
||||
|
||||
Returns:
|
||||
List of ``{vault, path, title}`` dicts for files linking to the target.
|
||||
"""
|
||||
global _backlink_index
|
||||
target_key = file_path.lower()
|
||||
if not target_key.endswith(".md"):
|
||||
target_key += ".md"
|
||||
|
||||
results = []
|
||||
for vname, vindex in _backlink_index.items():
|
||||
bl = vindex.get(target_key, [])
|
||||
results.extend(bl)
|
||||
return results
|
||||
|
||||
+109
-14
@@ -33,6 +33,7 @@ from backend.indexer import (
|
||||
get_vault_data,
|
||||
get_vault_names,
|
||||
find_file_in_index,
|
||||
get_backlinks,
|
||||
parse_markdown_file,
|
||||
_extract_tags,
|
||||
SUPPORTED_EXTENSIONS,
|
||||
@@ -100,6 +101,8 @@ class FileContentResponse(BaseModel):
|
||||
raw_length: int
|
||||
extension: str
|
||||
is_markdown: bool
|
||||
unsupported: Optional[bool] = False
|
||||
size_bytes: Optional[int] = None
|
||||
|
||||
|
||||
class FileRawResponse(BaseModel):
|
||||
@@ -550,6 +553,8 @@ app.add_middleware(SecurityHeadersMiddleware)
|
||||
# Auth router
|
||||
from backend.auth.router import router as auth_router
|
||||
from backend.auth.middleware import require_auth, require_admin, check_vault_access
|
||||
from backend.secret_redactor import redact_file_content
|
||||
from backend.audit import log_file_save, log_file_delete
|
||||
|
||||
app.include_router(auth_router)
|
||||
|
||||
@@ -603,6 +608,27 @@ def _resolve_safe_path(vault_root: Path, relative_path: str) -> Path:
|
||||
return resolved
|
||||
|
||||
|
||||
def _backup_file(file_path: Path, vault_name: str, relative_path: str):
|
||||
"""Create a timestamped backup of a file before modification.
|
||||
|
||||
Backups are stored in .obsigate-backup/{vault}/{relative_path}.{timestamp}.bak
|
||||
Silently skips if the file doesn't exist or can't be read.
|
||||
"""
|
||||
try:
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
return
|
||||
backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup"))
|
||||
backup_dir = backup_root / vault_name / Path(relative_path).parent
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = int(time.time())
|
||||
backup_name = f"{file_path.name}.{timestamp}.bak"
|
||||
backup_path = backup_dir / backup_name
|
||||
shutil.copy2(file_path, backup_path)
|
||||
logger.debug(f"Backed up {relative_path} to {backup_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to backup {relative_path}: {e}")
|
||||
|
||||
|
||||
def _check_vault_writable(vault_root: Path) -> bool:
|
||||
"""Check if a vault is writable (not mounted read-only).
|
||||
|
||||
@@ -625,7 +651,8 @@ import unicodedata
|
||||
def _heading_slugify(text: str) -> str:
|
||||
"""Generate a URL-safe slug from heading text.
|
||||
|
||||
Matches the JavaScript slugify algorithm exactly:
|
||||
Matches the JavaScript slugify algorithm exactly using
|
||||
Unicode-aware character classification:
|
||||
1. Lowercase
|
||||
2. NFD normalize + strip combining marks
|
||||
3. Keep only Unicode letters, numbers, spaces, hyphens
|
||||
@@ -640,15 +667,17 @@ def _heading_slugify(text: str) -> str:
|
||||
text = text.lower()
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(ch for ch in text if not unicodedata.combining(ch))
|
||||
# Keep only Unicode letters, numbers, spaces, and hyphens
|
||||
# Unicode-aware: keep letters (L*), numbers (N*), spaces, and hyphens
|
||||
cleaned = []
|
||||
for ch in text:
|
||||
if ch.isalpha() or ch.isdigit() or ch in (" ", "-"):
|
||||
cat = unicodedata.category(ch)
|
||||
if cat.startswith('L') or cat.startswith('N') or ch in (' ', '-'):
|
||||
cleaned.append(ch)
|
||||
text = "".join(cleaned)
|
||||
text = re.sub(r"\s+", "-", text)
|
||||
text = re.sub(r"-+", "-", text)
|
||||
return text.strip("-") or "heading"
|
||||
result = text.strip("-")
|
||||
return result if result else "heading"
|
||||
|
||||
|
||||
def _add_heading_ids(html: str) -> str:
|
||||
@@ -737,7 +766,10 @@ def _render_markdown(raw_md: str, vault_name: str, current_file_path: Optional[P
|
||||
vault_data = get_vault_data(vault_name)
|
||||
vault_root = Path(vault_data["path"]) if vault_data else None
|
||||
attachments_path = vault_data.get("config", {}).get("attachmentsPath") if vault_data else None
|
||||
|
||||
|
||||
# Redact secrets before rendering (P0 security)
|
||||
raw_md = redact_file_content(raw_md, str(current_file_path) if current_file_path else "")
|
||||
|
||||
# Preprocess images first
|
||||
if vault_root:
|
||||
raw_md = preprocess_images(raw_md, vault_name, vault_root, current_file_path, attachments_path)
|
||||
@@ -1155,8 +1187,16 @@ async def api_file_save(
|
||||
content = body.get('content', '')
|
||||
|
||||
try:
|
||||
# Backup original content before overwriting
|
||||
_backup_file(file_path, vault_name, path)
|
||||
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
logger.info(f"File saved: {vault_name}/{path}")
|
||||
|
||||
# Audit log
|
||||
client_ip = current_user.get("_request_ip", "unknown")
|
||||
log_file_save(current_user["username"], vault_name, path, len(content), client_ip)
|
||||
|
||||
return {"status": "ok", "vault": vault_name, "path": path, "size": len(content)}
|
||||
except PermissionError:
|
||||
raise HTTPException(status_code=403, detail="Permission denied: vault may be read-only")
|
||||
@@ -1195,9 +1235,16 @@ async def api_file_delete(vault_name: str, path: str = Query(..., description="R
|
||||
raise HTTPException(status_code=403, detail="Vault is read-only")
|
||||
|
||||
try:
|
||||
# Backup original content before deletion
|
||||
_backup_file(file_path, vault_name, path)
|
||||
|
||||
file_path.unlink()
|
||||
logger.info(f"File deleted: {vault_name}/{path}")
|
||||
|
||||
|
||||
# Audit log
|
||||
client_ip = current_user.get("_request_ip", "unknown")
|
||||
log_file_delete(current_user["username"], vault_name, path, client_ip)
|
||||
|
||||
# Update index
|
||||
await remove_single_file(vault_name, path)
|
||||
|
||||
@@ -1571,6 +1618,45 @@ async def api_file_rename(
|
||||
raise HTTPException(status_code=500, detail=f"Error renaming file: {str(e)}")
|
||||
|
||||
|
||||
@app.get("/api/file/{vault_name}/backlinks")
|
||||
async def api_file_backlinks(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Get backlinks (files linking to this file via wikilinks).
|
||||
|
||||
Returns a list of files that contain `[[wikilinks]]` pointing
|
||||
to the requested file, across all accessible vaults.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault containing the target file.
|
||||
path: Relative path of the target file within the vault.
|
||||
|
||||
Returns:
|
||||
``{"vault": str, "path": str, "backlinks": [...]}``
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", [])
|
||||
backlinks = get_backlinks(vault_name, path)
|
||||
|
||||
# Filter by user-accessible vaults
|
||||
if "*" not in user_vaults:
|
||||
backlinks = [b for b in backlinks if b["vault"] in user_vaults]
|
||||
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"backlinks": backlinks,
|
||||
"total": len(backlinks),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/file/{vault_name}", response_model=FileContentResponse)
|
||||
async def api_file(vault_name: str, path: str = Query(..., description="Relative path to file"), current_user=Depends(require_auth)):
|
||||
"""Return rendered HTML and metadata for a file.
|
||||
@@ -1601,24 +1687,33 @@ async def api_file(vault_name: str, path: str = Query(..., description="Relative
|
||||
# Record history
|
||||
record_open(current_user.get("username"), vault_name, path, title=file_path.name)
|
||||
|
||||
ext = file_path.suffix.lower()
|
||||
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except PermissionError as e:
|
||||
logger.error(f"Permission denied reading file {path}: {e}")
|
||||
raise HTTPException(status_code=403, detail=f"Permission denied: cannot read file {path}")
|
||||
except UnicodeDecodeError:
|
||||
# Binary file - try to read as binary and decode with errors='replace'
|
||||
try:
|
||||
raw = file_path.read_bytes().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading binary file {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Cannot read file: {str(e)}")
|
||||
# Binary / unsupported file — return structured info with download option
|
||||
size = file_path.stat().st_size
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": "",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"unsupported": True,
|
||||
"size_bytes": size,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error reading file {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading file: {str(e)}")
|
||||
|
||||
ext = file_path.suffix.lower()
|
||||
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
IP-based rate limiter for authentication endpoints.
|
||||
|
||||
Tracks failed login attempts per IP address with automatic
|
||||
cleanup of expired entries. Complements the per-account lockout
|
||||
in user_store.py.
|
||||
|
||||
Configuration via environment variables:
|
||||
OBSIGATE_LOGIN_MAX_ATTEMPTS Max failures per IP (default: 10)
|
||||
OBSIGATE_LOGIN_WINDOW_SECONDS Lockout window in seconds (default: 900 = 15min)
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from typing import Dict, Tuple
|
||||
|
||||
logger = logging.getLogger("obsigate.ratelimit")
|
||||
|
||||
# --- Configuration ---
|
||||
MAX_ATTEMPTS = int(os.environ.get("OBSIGATE_LOGIN_MAX_ATTEMPTS", "10"))
|
||||
WINDOW_SECONDS = int(os.environ.get("OBSIGATE_LOGIN_WINDOW_SECONDS", "900")) # 15 min
|
||||
|
||||
# --- In-memory store: {ip: [(timestamp, success_bool), ...]} ---
|
||||
_ip_attempts: Dict[str, list] = defaultdict(list)
|
||||
_last_cleanup = time.time()
|
||||
CLEANUP_INTERVAL = 60 # seconds
|
||||
|
||||
|
||||
def _cleanup_expired():
|
||||
"""Remove entries older than the window."""
|
||||
global _last_cleanup
|
||||
now = time.time()
|
||||
if now - _last_cleanup < CLEANUP_INTERVAL:
|
||||
return
|
||||
_last_cleanup = now
|
||||
cutoff = now - WINDOW_SECONDS
|
||||
expired_ips = []
|
||||
for ip, attempts in _ip_attempts.items():
|
||||
_ip_attempts[ip] = [a for a in attempts if a[0] > cutoff]
|
||||
if not _ip_attempts[ip]:
|
||||
expired_ips.append(ip)
|
||||
for ip in expired_ips:
|
||||
del _ip_attempts[ip]
|
||||
|
||||
|
||||
def record_failure(ip: str) -> Tuple[int, int]:
|
||||
"""Record a failed login attempt from an IP.
|
||||
|
||||
Returns:
|
||||
(current_failure_count, remaining_attempts)
|
||||
"""
|
||||
_cleanup_expired()
|
||||
_ip_attempts[ip].append((time.time(), False))
|
||||
failures = sum(1 for _, success in _ip_attempts[ip] if not success)
|
||||
remaining = max(0, MAX_ATTEMPTS - failures)
|
||||
if failures >= MAX_ATTEMPTS:
|
||||
logger.warning(f"IP {ip} rate-limited after {failures} failed logins")
|
||||
return failures, remaining
|
||||
|
||||
|
||||
def record_success(ip: str):
|
||||
"""Clear rate limit state for an IP after successful login."""
|
||||
_cleanup_expired()
|
||||
_ip_attempts[ip] = [(time.time(), True)]
|
||||
|
||||
|
||||
def is_rate_limited(ip: str) -> bool:
|
||||
"""Check if an IP has exceeded the rate limit."""
|
||||
_cleanup_expired()
|
||||
failures = sum(1 for _, success in _ip_attempts.get(ip, []) if not success)
|
||||
return failures >= MAX_ATTEMPTS
|
||||
|
||||
|
||||
def get_status(ip: str = None) -> dict:
|
||||
"""Get rate limit status for an IP (for diagnostics)."""
|
||||
_cleanup_expired()
|
||||
if ip:
|
||||
attempts = _ip_attempts.get(ip, [])
|
||||
failures = sum(1 for _, s in attempts if not s)
|
||||
return {
|
||||
"ip": ip,
|
||||
"failures": failures,
|
||||
"max": MAX_ATTEMPTS,
|
||||
"limited": failures >= MAX_ATTEMPTS,
|
||||
"window_seconds": WINDOW_SECONDS,
|
||||
}
|
||||
return {
|
||||
"tracked_ips": len(_ip_attempts),
|
||||
"max_attempts": MAX_ATTEMPTS,
|
||||
"window_seconds": WINDOW_SECONDS,
|
||||
"limited_ips": sum(
|
||||
1 for ip_addr in _ip_attempts
|
||||
if sum(1 for _, s in _ip_attempts[ip_addr] if not s) >= MAX_ATTEMPTS
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
Secret redactor: masks sensitive patterns in rendered text.
|
||||
|
||||
Scans for common secret patterns and replaces them with [MASQUÉ]
|
||||
before content is served to the frontend. Prevents accidental
|
||||
exposure of API keys, tokens, and passwords in previews.
|
||||
|
||||
Patterns detected:
|
||||
- Generic API keys (long alphanumeric strings with key/secret/token prefix)
|
||||
- JWT tokens (eyJ... base64url)
|
||||
- AWS-style keys (AKIA..., sk-..., etc.)
|
||||
- Private key blocks (-----BEGIN ... PRIVATE KEY-----)
|
||||
- Connection strings with passwords
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("obsigate.redactor")
|
||||
|
||||
# --- Patterns ---
|
||||
# Order matters: more specific patterns first
|
||||
_PATTERNS = [
|
||||
# Private key blocks
|
||||
(re.compile(r'-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----.*?-----END (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----', re.DOTALL), '[CLÉ PRIVÉE MASQUÉE]'),
|
||||
|
||||
# JWT tokens (base64url encoded, starts with eyJ)
|
||||
(re.compile(r'eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}'), '[JWT MASQUÉ]'),
|
||||
|
||||
# Connection strings with passwords
|
||||
(re.compile(r'(?:mongodb|mysql|postgres(?:ql)?|redis|sqlite)://[^:]+:[^@\s]+@'), '[CONNECTION_STRING MASQUÉE]'),
|
||||
|
||||
# Generic API key patterns: key=... or token=... or secret=...
|
||||
(re.compile(r'(?:api[_-]?key|apikey|secret|token|password|passwd|auth[_-]?token)\s*[:=]\s*[\'"]?([^\s\'"]{20,})[\'"]?', re.IGNORECASE),
|
||||
lambda m: f'{m.group(0).split("=")[0].split(":")[0]}=[MASQUÉ]' if "=" in m.group(0) or ":" in m.group(0) else '[MASQUÉ]'),
|
||||
|
||||
# Generic long hex/base64 strings that look like secrets (40+ chars)
|
||||
(re.compile(r'(?:sk|pk|rk)-[a-zA-Z0-9]{20,}'), '[CLÉ API MASQUÉE]'),
|
||||
|
||||
# AWS access keys
|
||||
(re.compile(r'AKIA[0-9A-Z]{16}'), '[AWS_KEY MASQUÉ]'),
|
||||
|
||||
# GitHub tokens (ghp_, gho_, ghu_, ghs_, ghr_)
|
||||
(re.compile(r'gh[pousr]_[a-zA-Z0-9]{36,}'), '[GITHUB_TOKEN MASQUÉ]'),
|
||||
|
||||
# Generic long random-looking strings (40+ hex chars)
|
||||
(re.compile(r'\b[a-fA-F0-9]{40,64}\b'), '[HEX_KEY MASQUÉ]'),
|
||||
]
|
||||
|
||||
|
||||
def redact(text: str) -> tuple:
|
||||
"""Redact sensitive patterns from text.
|
||||
|
||||
Args:
|
||||
text: The raw text content to scan.
|
||||
|
||||
Returns:
|
||||
(redacted_text, redaction_count) tuple.
|
||||
"""
|
||||
count = 0
|
||||
result = text
|
||||
for pattern, replacement in _PATTERNS:
|
||||
if callable(replacement):
|
||||
new_result, n = pattern.subn(replacement, result)
|
||||
else:
|
||||
new_result, n = pattern.subn(replacement, result)
|
||||
count += n
|
||||
result = new_result
|
||||
if count > 0:
|
||||
logger.info(f"Redacted {count} secret(s) from content")
|
||||
return result, count
|
||||
|
||||
|
||||
def redact_file_content(content: str, file_path: str = "") -> str:
|
||||
"""Redact a file's content for preview rendering.
|
||||
|
||||
Args:
|
||||
content: Raw file content.
|
||||
file_path: Optional file path for logging context.
|
||||
|
||||
Returns:
|
||||
Redacted content string.
|
||||
"""
|
||||
redacted, count = redact(content)
|
||||
if count > 0:
|
||||
logger.warning(f"Redacted {count} potential secret(s) from {file_path or '<unknown>'}")
|
||||
return redacted
|
||||
+14
-1
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional
|
||||
@@ -12,7 +13,19 @@ from backend.indexer import SUPPORTED_EXTENSIONS
|
||||
logger = logging.getLogger("obsigate.watcher")
|
||||
|
||||
# Extensions de fichiers surveillées
|
||||
IGNORED_DIRS = {'.obsidian', '.trash', '.git', '__pycache__', 'node_modules'}
|
||||
# Default ignored directories (can be overridden via OBSIGATE_IGNORED_DIRS env var)
|
||||
_DEFAULT_IGNORED_DIRS = {'.obsidian', '.trash', '.git', '__pycache__', 'node_modules', '.obsigate-backup'}
|
||||
|
||||
def _load_ignored_dirs() -> set:
|
||||
"""Load ignored directories from environment or use defaults."""
|
||||
env_val = os.environ.get("OBSIGATE_IGNORED_DIRS", "")
|
||||
if env_val:
|
||||
custom = set(d.strip() for d in env_val.split(",") if d.strip())
|
||||
logger.info(f"Using custom IGNORED_DIRS: {custom}")
|
||||
return custom
|
||||
return _DEFAULT_IGNORED_DIRS.copy()
|
||||
|
||||
IGNORED_DIRS = _load_ignored_dirs()
|
||||
|
||||
|
||||
class VaultEventHandler(FileSystemEventHandler):
|
||||
|
||||
Reference in New Issue
Block a user