fix(security): consolidation & securite phase 1 (#84, BUG-021 a BUG-034)
- sanitizer XSS serveur (markdown + page de partage) [BUG-021/022] - rate-limit/lockout MFA [BUG-023] - isolation vaults par segments [BUG-024] - caps regex ReDoS [BUG-025] - SSRF webhooks + secrets externalises [BUG-026] - rotation/revocation des jetons [BUG-027] - politique de mot de passe + invalidation sessions [BUG-028] - verrous users.json [BUG-029] - IP reelle dans les audits [BUG-030] - rate-limit par compte [BUG-031] - symlinks hors vault ignores [BUG-032] - recherche simple via inverted index [BUG-033] - token en memoire + cookie HttpOnly, CSP durcie [BUG-034] Tests: pytest 961 passed / 6 skipped, ruff 0, mypy 0, frontend vert.
This commit is contained in:
+103
-50
@@ -13,6 +13,11 @@ from sortedcontainers import SortedList
|
||||
from backend import indexer as _indexer
|
||||
from backend import semantic_search as _semantic
|
||||
from backend.indexer import index
|
||||
from backend.services.regex_safety import (
|
||||
MAX_REGEX_MATCHES,
|
||||
truncate_for_regex,
|
||||
validate_regex,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.search")
|
||||
|
||||
@@ -226,12 +231,16 @@ def _extract_regex_snippet(
|
||||
if not content or not pattern_text:
|
||||
return content[:200].strip() if content else ""
|
||||
|
||||
# BUG-025: bound the text scanned and the number of matches collected.
|
||||
content = truncate_for_regex(content)
|
||||
|
||||
try:
|
||||
validate_regex(pattern_text)
|
||||
pattern = re.compile(pattern_text, re.IGNORECASE)
|
||||
except re.error:
|
||||
except (re.error, ValueError):
|
||||
return _escape_html(content[:200].strip())
|
||||
|
||||
matches = list(pattern.finditer(content))
|
||||
matches = list(pattern.finditer(content))[:MAX_REGEX_MATCHES]
|
||||
if not matches:
|
||||
return _escape_html(content[:200].strip())
|
||||
|
||||
@@ -729,60 +738,97 @@ def search(
|
||||
query_lower = query.lower()
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for vault_name, vault_data in index.items():
|
||||
if vault_filter != "all" and vault_name != vault_filter:
|
||||
inv = get_inverted_index()
|
||||
use_index = (not inv.is_stale()) and inv.doc_count > 0
|
||||
|
||||
if use_index:
|
||||
# BUG-033: retrieve candidates from the inverted index instead of
|
||||
# scanning every document. Multi-term queries require all terms
|
||||
# (a superset of exact-phrase matches), single terms use prefix
|
||||
# expansion. Falls back to a full scan while the index is building.
|
||||
if has_query:
|
||||
terms = [t for t in tokenize(query) if t]
|
||||
if not terms:
|
||||
return []
|
||||
doc_sets: list[set] = []
|
||||
for term in terms:
|
||||
term_docs: set = set(inv.word_index.get(term, {}).keys())
|
||||
if len(term) >= MIN_PREFIX_LENGTH:
|
||||
for expanded in inv.get_prefix_tokens(term):
|
||||
term_docs.update(inv.word_index.get(expanded, {}).keys())
|
||||
doc_sets.append(term_docs)
|
||||
doc_keys = set.intersection(*doc_sets) if doc_sets else set()
|
||||
else:
|
||||
doc_keys = set(inv.doc_info.keys())
|
||||
|
||||
if vault_filter != "all":
|
||||
doc_keys &= inv.vault_docs.get(vault_filter, set())
|
||||
for tag in selected_tags:
|
||||
doc_keys &= inv.tag_docs.get(tag.lower(), set())
|
||||
|
||||
candidates = [
|
||||
(inv.doc_vault[dk], inv.doc_info[dk])
|
||||
for dk in doc_keys
|
||||
if dk in inv.doc_info
|
||||
]
|
||||
else:
|
||||
candidates = [
|
||||
(vault_name, file_info)
|
||||
for vault_name, vault_data in index.items()
|
||||
if vault_filter == "all" or vault_name == vault_filter
|
||||
for file_info in vault_data["files"]
|
||||
]
|
||||
|
||||
for vault_name, file_info in candidates:
|
||||
# Tag filter: all selected tags must be present
|
||||
if selected_tags and not all(tag in file_info["tags"] for tag in selected_tags):
|
||||
continue
|
||||
|
||||
for file_info in vault_data["files"]:
|
||||
# Tag filter: all selected tags must be present
|
||||
if selected_tags and not all(tag in file_info["tags"] for tag in selected_tags):
|
||||
continue
|
||||
score = 0
|
||||
snippet = file_info.get("content_preview", "")
|
||||
|
||||
score = 0
|
||||
snippet = file_info.get("content_preview", "")
|
||||
if has_query:
|
||||
title_lower = file_info["title"].lower()
|
||||
|
||||
if has_query:
|
||||
title_lower = file_info["title"].lower()
|
||||
# Exact title match (highest weight)
|
||||
if query_lower == title_lower:
|
||||
score += 20
|
||||
# Partial title match
|
||||
elif query_lower in title_lower:
|
||||
score += 10
|
||||
|
||||
# Exact title match (highest weight)
|
||||
if query_lower == title_lower:
|
||||
score += 20
|
||||
# Partial title match
|
||||
elif query_lower in title_lower:
|
||||
score += 10
|
||||
# Path match (folder/filename relevance)
|
||||
if query_lower in file_info["path"].lower():
|
||||
score += 5
|
||||
|
||||
# Path match (folder/filename relevance)
|
||||
if query_lower in file_info["path"].lower():
|
||||
score += 5
|
||||
# Tag name match
|
||||
for tag in file_info.get("tags", []):
|
||||
if query_lower in tag.lower():
|
||||
score += 3
|
||||
break # count once per file
|
||||
|
||||
# Tag name match
|
||||
for tag in file_info.get("tags", []):
|
||||
if query_lower in tag.lower():
|
||||
score += 3
|
||||
break # count once per file
|
||||
# Content match — use cached content (no disk I/O)
|
||||
content = file_info.get("content", "")
|
||||
content_lower = content.lower()
|
||||
if query_lower in content_lower:
|
||||
# Frequency-based scoring, capped to avoid over-weighting
|
||||
occurrences = content_lower.count(query_lower)
|
||||
score += min(occurrences, 10)
|
||||
snippet = _extract_snippet(content, query)
|
||||
else:
|
||||
# Tag-only filter: all matching files get score 1
|
||||
score = 1
|
||||
|
||||
# Content match — use cached content (no disk I/O)
|
||||
content = file_info.get("content", "")
|
||||
content_lower = content.lower()
|
||||
if query_lower in content_lower:
|
||||
# Frequency-based scoring, capped to avoid over-weighting
|
||||
occurrences = content_lower.count(query_lower)
|
||||
score += min(occurrences, 10)
|
||||
snippet = _extract_snippet(content, query)
|
||||
else:
|
||||
# Tag-only filter: all matching files get score 1
|
||||
score = 1
|
||||
|
||||
if score > 0:
|
||||
results.append({
|
||||
"vault": vault_name,
|
||||
"path": file_info["path"],
|
||||
"title": file_info["title"],
|
||||
"tags": file_info["tags"],
|
||||
"score": score,
|
||||
"snippet": snippet,
|
||||
"modified": file_info["modified"],
|
||||
})
|
||||
if score > 0:
|
||||
results.append({
|
||||
"vault": vault_name,
|
||||
"path": file_info["path"],
|
||||
"title": file_info["title"],
|
||||
"tags": file_info["tags"],
|
||||
"score": score,
|
||||
"snippet": snippet,
|
||||
"modified": file_info["modified"],
|
||||
})
|
||||
|
||||
results.sort(key=lambda x: -x["score"])
|
||||
return results[:limit]
|
||||
@@ -908,12 +954,14 @@ def _passes_search_filters(
|
||||
title = file_info.get("title", "")
|
||||
content = file_info.get("content", "")
|
||||
path = file_info.get("path", "")
|
||||
search_text = f"{title} {content}"
|
||||
# BUG-025: cap the text scanned by a user-supplied regex.
|
||||
search_text = truncate_for_regex(f"{title} {content}")
|
||||
search_text_norm = normalize_text(search_text)
|
||||
|
||||
# --- Regex mode ---
|
||||
if regex and raw_query:
|
||||
try:
|
||||
validate_regex(raw_query)
|
||||
flags = 0 if case_sensitive else re.IGNORECASE
|
||||
if whole_word:
|
||||
pattern = re.compile(rf"\b{raw_query}\b", flags)
|
||||
@@ -921,7 +969,7 @@ def _passes_search_filters(
|
||||
pattern = re.compile(raw_query, flags)
|
||||
if not pattern.search(search_text):
|
||||
return False
|
||||
except re.error:
|
||||
except (re.error, ValueError):
|
||||
return False
|
||||
return _passes_path_filters(path, include_paths, exclude_paths)
|
||||
|
||||
@@ -1138,6 +1186,11 @@ def advanced_search(
|
||||
"""
|
||||
t0 = time.monotonic()
|
||||
query = query.strip() if query else ""
|
||||
|
||||
# BUG-025: reject oversized / catastrophic regex patterns up front.
|
||||
if regex and query:
|
||||
validate_regex(query)
|
||||
|
||||
parsed = _parse_advanced_query(query)
|
||||
|
||||
# Merge explicit tag_filter with parsed tag: operators
|
||||
|
||||
Reference in New Issue
Block a user