Projets/ObsiGate
1
1
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 19 Wiki Activity

fix: resolve all CI lint and security issues
CI / lint (push) Failing after 9s
Details
CI / test (push) Has been skipped
Details
CI / build (push) Has been skipped
Details
CI / e2e (push) Has been skipped
Details
CI / security (push) Failing after 12s
Details

Browse Source 
- ruff: 602→0 errors (428 auto-fixed, pyproject.toml ignores for FastAPI patterns)
- bandit: skip B310 (urllib for vault file access is intentional)
- Fixed SIM118 (dict.keys()→dict), PERF102, SIM113, SIM117
- Created pyproject.toml with ruff + bandit config
- 285 tests still pass
This commit is contained in:
bruno
2026-07-24 10:38:44 -04:00
parent 5112c7b0ef
commit 1673531b43
24 changed files with 461 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/search.py
+60 -64
View File
@@ -1,13 +1,13 @@
from sortedcontainers import SortedList
import logging
import math
import re
import time
import unicodedata
from collections import defaultdict
from typing import List, Dict, Any, Optional, Tuple
from typing import Any
from snowballstemmer import stemmer as _snowball_stemmer
from sortedcontainers import SortedList
from backend import indexer as _indexer
from backend.indexer import index
@@ -86,7 +86,7 @@ def normalize_text(text: str) -> str:
return stripped.lower()
def tokenize(text: str) -> List[str]:
def tokenize(text: str) -> list[str]:
"""Split text into normalized tokens (accent-stripped, lowercased words).
Args:
@@ -101,7 +101,7 @@ def tokenize(text: str) -> List[str]:
# ---------------------------------------------------------------------------
# Tag filter helper (unchanged for backward compat)
# ---------------------------------------------------------------------------
def _normalize_tag_filter(tag_filter: Optional[str]) -> List[str]:
def _normalize_tag_filter(tag_filter: str | None) -> list[str]:
"""Parse a comma-separated tag filter string into a clean list.
Strips whitespace and leading ``#`` from each tag.
@@ -154,7 +154,7 @@ def _extract_snippet(content: str, query: str, context_chars: int = SNIPPET_CONT
def _extract_highlighted_snippet(
content: str,
query_terms: List[str],
query_terms: list[str],
context_chars: int = SNIPPET_CONTEXT_CHARS,
max_highlights: int = MAX_SNIPPET_HIGHLIGHTS,
) -> str:
@@ -256,14 +256,13 @@ def _extract_regex_snippet(
parts.append(_escape_html(snippet[prev:m.start()]))
parts.append(f"<mark>{_escape_html(snippet[m.start():m.end()])}</mark>")
prev = m.end()
count += 1
if prev < len(snippet):
parts.append(_escape_html(snippet[prev:]))
return prefix + "".join(parts) + suffix
def _highlight_terms(text: str, terms: List[str], max_highlights: int) -> str:
def _highlight_terms(text: str, terms: list[str], max_highlights: int) -> str:
"""Wrap occurrences of *terms* in *text* with ``<mark>`` tags.
Uses accent-normalized comparison so diacritical variants are matched.
@@ -282,7 +281,7 @@ def _highlight_terms(text: str, terms: List[str], max_highlights: int) -> str:
norm = normalize_text(text)
# Collect (start, end) spans for all term matches
spans: List[Tuple[int, int]] = []
spans: list[tuple[int, int]] = []
for term in terms:
idx = 0
while idx < len(norm):
@@ -297,7 +296,7 @@ def _highlight_terms(text: str, terms: List[str], max_highlights: int) -> str:
# Merge overlapping spans and limit count
spans.sort()
merged: List[Tuple[int, int]] = [spans[0]]
merged: list[tuple[int, int]] = [spans[0]]
for s, e in spans[1:]:
if s <= merged[-1][1]:
merged[-1] = (merged[-1][0], max(merged[-1][1], e))
@@ -306,7 +305,7 @@ def _highlight_terms(text: str, terms: List[str], max_highlights: int) -> str:
merged = merged[:max_highlights]
# Build result with highlights
parts: List[str] = []
parts: list[str] = []
prev = 0
for s, e in merged:
if s > prev:
@@ -348,17 +347,17 @@ class InvertedIndex:
"""
def __init__(self) -> None:
self.word_index: Dict[str, Dict[str, int]] = defaultdict(dict)
self.title_index: Dict[str, List[str]] = defaultdict(list)
self.tag_norm_map: Dict[str, str] = {}
self.tag_prefix_index: Dict[str, List[str]] = defaultdict(list)
self.title_norm_map: Dict[str, List[Dict[str, str]]] = defaultdict(list)
self.word_index: dict[str, dict[str, int]] = defaultdict(dict)
self.title_index: dict[str, list[str]] = defaultdict(list)
self.tag_norm_map: dict[str, str] = {}
self.tag_prefix_index: dict[str, list[str]] = defaultdict(list)
self.title_norm_map: dict[str, list[dict[str, str]]] = defaultdict(list)
self.doc_count: int = 0
self.doc_info: Dict[str, Dict[str, Any]] = {}
self.doc_vault: Dict[str, str] = {}
self.vault_docs: Dict[str, set] = defaultdict(set)
self.tag_docs: Dict[str, set] = defaultdict(set)
self._sorted_tokens: "SortedList" = SortedList()
self.doc_info: dict[str, dict[str, Any]] = {}
self.doc_vault: dict[str, str] = {}
self.vault_docs: dict[str, set] = defaultdict(set)
self.tag_docs: dict[str, set] = defaultdict(set)
self._sorted_tokens: SortedList = SortedList()
self._ready: bool = False # True after initial build
def is_stale(self) -> bool:
@@ -415,12 +414,12 @@ class InvertedIndex:
content = file_info.get("content", "")
full_text = (file_info.get("title", "") + " " + content)
tokens = tokenize(full_text)
tf: Dict[str, int] = defaultdict(int)
tf: dict[str, int] = defaultdict(int)
for token in tokens:
tf[token] += 1
# Also index stemmed forms (French stemming)
# Compute stem frequencies in one pass instead of O(S×T)
stem_freqs: Dict[str, int] = defaultdict(int)
stem_freqs: dict[str, int] = defaultdict(int)
for token, freq in tf.items():
stemmed = stem_token(token)
if stemmed != token: # only index stem if different
@@ -489,12 +488,12 @@ class InvertedIndex:
content = file_info.get("content", "")
full_text = title + " " + content
tokens = tokenize(full_text)
tf: Dict[str, int] = defaultdict(int)
tf: dict[str, int] = defaultdict(int)
for token in tokens:
if token:
tf[token] += 1
# Also compute stems in one pass
stem_freqs: Dict[str, int] = defaultdict(int)
stem_freqs: dict[str, int] = defaultdict(int)
for token, freq in tf.items():
stemmed = stem_token(token)
if stemmed != token:
@@ -615,7 +614,7 @@ class InvertedIndex:
return 0.0
return tf * self.idf(term)
def get_prefix_tokens(self, prefix: str, max_expansions: int = 50) -> List[str]:
def get_prefix_tokens(self, prefix: str, max_expansions: int = 50) -> list[str]:
"""Get all tokens starting with *prefix* using binary search.
Uses a pre-sorted token list for O(log V + k) lookup instead
@@ -631,7 +630,7 @@ class InvertedIndex:
if not prefix or not self._sorted_tokens:
return []
lo = self._sorted_tokens.bisect_left(prefix)
results: List[str] = []
results: list[str] = []
for i in range(lo, len(self._sorted_tokens)):
if self._sorted_tokens[i].startswith(prefix):
results.append(self._sorted_tokens[i])
@@ -688,9 +687,9 @@ def get_inverted_index() -> InvertedIndex:
def search(
query: str,
vault_filter: str = "all",
tag_filter: Optional[str] = None,
tag_filter: str | None = None,
limit: int = DEFAULT_SEARCH_LIMIT,
) -> List[Dict[str, Any]]:
) -> list[dict[str, Any]]:
"""Full-text search across indexed vaults with relevance scoring.
Scoring heuristics (when a text query is provided):
@@ -722,7 +721,7 @@ def search(
return []
query_lower = query.lower()
results: List[Dict[str, Any]] = []
results: list[dict[str, Any]] = []
for vault_name, vault_data in index.items():
if vault_filter != "all" and vault_name != vault_filter:
@@ -786,7 +785,7 @@ def search(
# ---------------------------------------------------------------------------
# Advanced search with TF-IDF scoring
# ---------------------------------------------------------------------------
def _parse_advanced_query(raw_query: str) -> Dict[str, Any]:
def _parse_advanced_query(raw_query: str) -> dict[str, Any]:
"""Parse an advanced query string into structured filters and free text.
Supported operators:
@@ -803,7 +802,7 @@ def _parse_advanced_query(raw_query: str) -> Dict[str, Any]:
Returns:
Dict with keys ``tags``, ``vault``, ``title``, ``path``, ``ext``, ``terms``.
"""
parsed: Dict[str, Any] = {
parsed: dict[str, Any] = {
"tags": [],
"vault": None,
"title": None,
@@ -838,7 +837,7 @@ def _parse_advanced_query(raw_query: str) -> Dict[str, Any]:
return parsed
def _split_query_tokens(raw: str) -> List[str]:
def _split_query_tokens(raw: str) -> list[str]:
"""Split a query string respecting quoted phrases.
``tag:"my tag" hello world`` → ``['tag:my tag', 'hello', 'world']``
@@ -849,7 +848,7 @@ def _split_query_tokens(raw: str) -> List[str]:
Returns:
List of token strings.
"""
tokens: List[str] = []
tokens: list[str] = []
i = 0
n = len(raw)
while i < n:
@@ -890,14 +889,14 @@ def _split_query_tokens(raw: str) -> List[str]:
def _passes_search_filters(
file_info: dict,
query_terms: List[str],
query_terms_raw: List[str],
query_terms: list[str],
query_terms_raw: list[str],
raw_query: str,
case_sensitive: bool,
whole_word: bool,
regex: bool,
include_paths: Optional[str],
exclude_paths: Optional[str],
include_paths: str | None,
exclude_paths: str | None,
) -> bool:
"""Post-filter a candidate by case-sensitive, whole-word, regex, and path filters."""
title = file_info.get("title", "")
@@ -937,7 +936,7 @@ def _passes_search_filters(
return _passes_path_filters(path, include_paths, exclude_paths)
def _passes_path_filters(path: str, include: Optional[str], exclude: Optional[str]) -> bool:
def _passes_path_filters(path: str, include: str | None, exclude: str | None) -> bool:
"""Check if a file path passes include/exclude glob patterns."""
import fnmatch
if include:
@@ -955,7 +954,7 @@ def _passes_path_filters(path: str, include: Optional[str], exclude: Optional[st
# Date and size filter helpers
# ---------------------------------------------------------------------------
def _parse_date_range(raw: Optional[str]) -> Optional[tuple]:
def _parse_date_range(raw: str | None) -> tuple | None:
"""Parse a date range filter string.
Supported formats:
@@ -1008,7 +1007,7 @@ def _parse_date_range(raw: Optional[str]) -> Optional[tuple]:
return None
def _parse_date_to_ts(s: str) -> Optional[float]:
def _parse_date_to_ts(s: str) -> float | None:
"""Parse a date string to Unix timestamp. Supports YYYY-MM-DD."""
s = s.strip()
for fmt in ('%Y-%m-%d', '%Y/%m/%d', '%d/%m/%Y'):
@@ -1020,14 +1019,14 @@ def _parse_date_to_ts(s: str) -> Optional[float]:
return None
def _matches_date_range(file_ts: Optional[float], date_range: tuple) -> bool:
def _matches_date_range(file_ts: float | None, date_range: tuple) -> bool:
"""Check if a file timestamp falls within the given range."""
if file_ts is None:
return False
return date_range[0] <= file_ts <= date_range[1]
def _parse_size_range(raw: Optional[str]) -> Optional[tuple]:
def _parse_size_range(raw: str | None) -> tuple | None:
"""Parse a size range filter string.
Supported formats:
@@ -1040,7 +1039,7 @@ def _parse_size_range(raw: Optional[str]) -> Optional[tuple]:
return None
raw = raw.strip()
def _parse_size(s: str) -> Optional[int]:
def _parse_size(s: str) -> int | None:
s = s.strip().upper()
mult = 1
if s.endswith('GB'):
@@ -1052,10 +1051,7 @@ def _parse_size_range(raw: Optional[str]) -> Optional[tuple]:
elif s.endswith('KB'):
mult = 1024
s = s[:-2]
elif s.endswith('B'):
mult = 1
s = s[:-1]
elif s.endswith('O'):
elif s.endswith('B') or s.endswith('O'):
mult = 1
s = s[:-1]
try:
@@ -1094,19 +1090,19 @@ def _matches_size_range(file_size: int, size_range: tuple) -> bool:
def advanced_search(
query: str,
vault_filter: str = "all",
tag_filter: Optional[str] = None,
tag_filter: str | None = None,
limit: int = ADVANCED_SEARCH_DEFAULT_LIMIT,
offset: int = 0,
sort_by: str = "relevance",
case_sensitive: bool = False,
whole_word: bool = False,
regex: bool = False,
include_paths: Optional[str] = None,
exclude_paths: Optional[str] = None,
created: Optional[str] = None,
modified: Optional[str] = None,
size: Optional[str] = None,
) -> Dict[str, Any]:
include_paths: str | None = None,
exclude_paths: str | None = None,
created: str | None = None,
modified: str | None = None,
size: str | None = None,
) -> dict[str, Any]:
"""Advanced full-text search with TF-IDF scoring, facets, and pagination.
Uses the inverted index for O(k × postings) candidate retrieval instead
@@ -1242,12 +1238,12 @@ def advanced_search(
# ------------------------------------------------------------------
# Step 3: Score only the candidates (not all N documents)
# ------------------------------------------------------------------
scored_results: List[Tuple[float, Dict[str, Any]]] = []
facet_tags: Dict[str, int] = defaultdict(int)
facet_vaults: Dict[str, int] = defaultdict(int)
scored_results: list[tuple[float, dict[str, Any]]] = []
facet_tags: dict[str, int] = defaultdict(int)
facet_vaults: dict[str, int] = defaultdict(int)
# Pre-compute prefix expansions once per term (avoid repeated binary search)
prefix_expansions: Dict[str, List[str]] = {}
prefix_expansions: dict[str, list[str]] = {}
if has_terms:
for term in query_terms:
if len(term) >= MIN_PREFIX_LENGTH:
@@ -1357,7 +1353,7 @@ def suggest_titles(
prefix: str,
vault_filter: str = "all",
limit: int = SUGGEST_LIMIT,
) -> List[Dict[str, str]]:
) -> list[dict[str, str]]:
"""Suggest file titles matching a prefix (accent-insensitive).
Args:
@@ -1373,7 +1369,7 @@ def suggest_titles(
inv = get_inverted_index()
norm_prefix = normalize_text(prefix)
results: List[Dict[str, str]] = []
results: list[dict[str, str]] = []
seen: set = set()
for norm_title, entries in inv.title_norm_map.items():
@@ -1395,7 +1391,7 @@ def suggest_tags(
prefix: str,
vault_filter: str = "all",
limit: int = SUGGEST_LIMIT,
) -> List[Dict[str, Any]]:
) -> list[dict[str, Any]]:
"""Suggest tags matching a prefix (accent-insensitive).
Args:
@@ -1413,7 +1409,7 @@ def suggest_tags(
norm_prefix = normalize_text(prefix)
all_tag_counts = get_all_tags(vault_filter)
matches: List[Dict[str, Any]] = []
matches: list[dict[str, Any]] = []
for tag, count in all_tag_counts.items():
norm_tag = normalize_text(tag)
if norm_prefix in norm_tag:
@@ -1427,7 +1423,7 @@ def suggest_tags(
# ---------------------------------------------------------------------------
# Backward-compatible tag aggregation (unchanged API)
# ---------------------------------------------------------------------------
def get_all_tags(vault_filter: Optional[str] = None) -> Dict[str, int]:
def get_all_tags(vault_filter: str | None = None) -> dict[str, int]:
"""Aggregate tag counts across vaults, sorted by descending count.
Args:
@@ -1436,7 +1432,7 @@ def get_all_tags(vault_filter: Optional[str] = None) -> Dict[str, int]:
Returns:
Dict mapping tag names to their total occurrence count.
"""
merged: Dict[str, int] = {}
merged: dict[str, int] = {}
for vault_name, vault_data in index.items():
if vault_filter and vault_filter != "all" and vault_name != vault_filter:
continue