fix(security): consolidation & securite phase 1 (#84, BUG-021 a BUG-034)
- sanitizer XSS serveur (markdown + page de partage) [BUG-021/022] - rate-limit/lockout MFA [BUG-023] - isolation vaults par segments [BUG-024] - caps regex ReDoS [BUG-025] - SSRF webhooks + secrets externalises [BUG-026] - rotation/revocation des jetons [BUG-027] - politique de mot de passe + invalidation sessions [BUG-028] - verrous users.json [BUG-029] - IP reelle dans les audits [BUG-030] - rate-limit par compte [BUG-031] - symlinks hors vault ignores [BUG-032] - recherche simple via inverted index [BUG-033] - token en memoire + cookie HttpOnly, CSP durcie [BUG-034] Tests: pytest 961 passed / 6 skipped, ruff 0, mypy 0, frontend vert.
This commit is contained in:
+95
-76
@@ -423,94 +423,113 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
|
||||
logger.warning(f"Vault path does not exist: {vault_path}")
|
||||
return {"files": [], "tags": {}, "path": vault_path, "paths": []}
|
||||
|
||||
for fpath in vault_root.rglob("*"):
|
||||
# Skip ignored directories
|
||||
if any(part in IGNORED_DIRS for part in fpath.relative_to(vault_root).parts):
|
||||
continue
|
||||
root_resolved = vault_root.resolve(strict=False)
|
||||
|
||||
rel_path_str = str(fpath.relative_to(vault_root)).replace("\\", "/")
|
||||
|
||||
# Add all paths (files and directories) to path index
|
||||
if fpath.is_dir():
|
||||
# BUG-032: walk without following symlinks and refuse any symlink that
|
||||
# escapes the vault root, so external data can never be indexed/exposed.
|
||||
for dirpath, dirnames, filenames in os.walk(vault_root, followlinks=False):
|
||||
current_dir = Path(dirpath)
|
||||
|
||||
# Prune ignored and symlinked directories in place (no recursion).
|
||||
dirnames[:] = [
|
||||
d for d in dirnames
|
||||
if d not in IGNORED_DIRS and not (current_dir / d).is_symlink()
|
||||
]
|
||||
|
||||
for d in dirnames:
|
||||
dpath = current_dir / d
|
||||
rel_path_str = str(dpath.relative_to(vault_root)).replace("\\", "/")
|
||||
paths.append({
|
||||
"path": rel_path_str,
|
||||
"name": fpath.name,
|
||||
"name": d,
|
||||
"type": "directory"
|
||||
})
|
||||
continue
|
||||
|
||||
# Files only from here
|
||||
if not fpath.is_file():
|
||||
continue
|
||||
ext = fpath.suffix.lower()
|
||||
# Also match extensionless files named like Dockerfile, Makefile
|
||||
basename_lower = fpath.name.lower()
|
||||
if ext not in SUPPORTED_EXTENSIONS and basename_lower not in ("dockerfile", "makefile", "cmakelists.txt"):
|
||||
continue
|
||||
|
||||
# Add file to path index
|
||||
paths.append({
|
||||
"path": rel_path_str,
|
||||
"name": fpath.name,
|
||||
"type": "file"
|
||||
})
|
||||
|
||||
try:
|
||||
relative = fpath.relative_to(vault_root)
|
||||
stat = fpath.stat()
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
|
||||
|
||||
# PDF handling — special path (binary, uses pdf_reader)
|
||||
tags: list[str] = []
|
||||
if ext == ".pdf":
|
||||
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text
|
||||
raw = extract_pdf_text(fpath, max_chars=100000)
|
||||
pdf_meta = extract_pdf_metadata(fpath)
|
||||
title = pdf_meta.get("title") or fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
elif ext == ".excalidraw" or fpath.name.lower().endswith(".excalidraw.md"):
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
raw = extract_excalidraw_indexable(raw)
|
||||
title = fpath.stem.replace(".excalidraw", "").replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
else:
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
title = fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
for fname in filenames:
|
||||
fpath = current_dir / fname
|
||||
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
tags = _extract_tags(post)
|
||||
inline_tags = _extract_inline_tags(post.content)
|
||||
tags = list(set(tags) | set(inline_tags))
|
||||
title = _extract_title(post, fpath)
|
||||
content_preview = post.content[:200].strip()
|
||||
if fpath.is_symlink():
|
||||
try:
|
||||
target = fpath.resolve(strict=True)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
target.relative_to(root_resolved)
|
||||
except ValueError:
|
||||
logger.warning(f"Skipping symlink outside vault: {fpath}")
|
||||
continue
|
||||
|
||||
_extract_wikilinks_for_backlinks(
|
||||
vault_name, str(relative).replace("\\", "/"),
|
||||
title, post.content
|
||||
)
|
||||
rel_path_str = str(fpath.relative_to(vault_root)).replace("\\", "/")
|
||||
|
||||
files.append({
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"content_preview": content_preview,
|
||||
"content": raw[:SEARCH_CONTENT_LIMIT],
|
||||
"size": stat.st_size,
|
||||
"modified": modified,
|
||||
"extension": ext,
|
||||
ext = fpath.suffix.lower()
|
||||
# Also match extensionless files named like Dockerfile, Makefile
|
||||
basename_lower = fpath.name.lower()
|
||||
if ext not in SUPPORTED_EXTENSIONS and basename_lower not in ("dockerfile", "makefile", "cmakelists.txt"):
|
||||
continue
|
||||
|
||||
# Add file to path index
|
||||
paths.append({
|
||||
"path": rel_path_str,
|
||||
"name": fname,
|
||||
"type": "file"
|
||||
})
|
||||
|
||||
for tag in tags:
|
||||
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
||||
try:
|
||||
relative = fpath.relative_to(vault_root)
|
||||
stat = fpath.stat()
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
|
||||
|
||||
except PermissionError:
|
||||
logger.debug(f"Permission denied, skipping {fpath}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing {fpath}: {e}")
|
||||
continue
|
||||
# PDF handling — special path (binary, uses pdf_reader)
|
||||
tags: list[str] = []
|
||||
if ext == ".pdf":
|
||||
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text
|
||||
raw = extract_pdf_text(fpath, max_chars=100000)
|
||||
pdf_meta = extract_pdf_metadata(fpath)
|
||||
title = pdf_meta.get("title") or fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
elif ext == ".excalidraw" or fpath.name.lower().endswith(".excalidraw.md"):
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
raw = extract_excalidraw_indexable(raw)
|
||||
title = fpath.stem.replace(".excalidraw", "").replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
else:
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
title = fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
tags = _extract_tags(post)
|
||||
inline_tags = _extract_inline_tags(post.content)
|
||||
tags = list(set(tags) | set(inline_tags))
|
||||
title = _extract_title(post, fpath)
|
||||
content_preview = post.content[:200].strip()
|
||||
|
||||
_extract_wikilinks_for_backlinks(
|
||||
vault_name, str(relative).replace("\\", "/"),
|
||||
title, post.content
|
||||
)
|
||||
|
||||
files.append({
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"content_preview": content_preview,
|
||||
"content": raw[:SEARCH_CONTENT_LIMIT],
|
||||
"size": stat.st_size,
|
||||
"modified": modified,
|
||||
"extension": ext,
|
||||
})
|
||||
|
||||
for tag in tags:
|
||||
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
||||
|
||||
except PermissionError:
|
||||
logger.debug(f"Permission denied, skipping {fpath}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing {fpath}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Vault '{vault_name}': indexed {len(files)} files, {len(paths)} paths, {len(tag_counts)} unique tags")
|
||||
return {"files": files, "tags": tag_counts, "path": vault_path, "paths": paths, "config": {}}
|
||||
|
||||
Reference in New Issue
Block a user