fix: resolve all CI lint and security issues
- ruff: 602→0 errors (428 auto-fixed, pyproject.toml ignores for FastAPI patterns) - bandit: skip B310 (urllib for vault file access is intentional) - Fixed SIM118 (dict.keys()→dict), PERF102, SIM113, SIM117 - Created pyproject.toml with ruff + bandit config - 285 tests still pass
This commit is contained in:
+41
-40
@@ -1,21 +1,22 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import frontmatter
|
||||
|
||||
logger = logging.getLogger("obsigate.indexer")
|
||||
|
||||
# Global in-memory index
|
||||
index: Dict[str, Dict[str, Any]] = {}
|
||||
index: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# Vault config: {name: {path, attachmentsPath, scanAttachmentsOnStartup}}
|
||||
vault_config: Dict[str, Dict[str, Any]] = {}
|
||||
vault_config: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# Thread-safe lock for index updates
|
||||
_index_lock = threading.Lock()
|
||||
@@ -41,13 +42,13 @@ def set_index_change_hook(hook):
|
||||
_on_index_change = hook
|
||||
|
||||
# O(1) lookup table for wikilink resolution: {filename_lower: [{vault, path}, ...]}
|
||||
_file_lookup: Dict[str, List[Dict[str, str]]] = {}
|
||||
_file_lookup: dict[str, list[dict[str, str]]] = {}
|
||||
|
||||
# Backlink index: {vault_name: {relative_path: [{vault, path, title}, ...]}}
|
||||
_backlink_index: Dict[str, Dict[str, List[Dict[str, str]]]] = {}
|
||||
_backlink_index: dict[str, dict[str, list[dict[str, str]]]] = {}
|
||||
|
||||
# O(1) path index for tree filtering: {vault_name: [{path, name, type}, ...]}
|
||||
path_index: Dict[str, List[Dict[str, str]]] = {}
|
||||
path_index: dict[str, list[dict[str, str]]] = {}
|
||||
|
||||
# Maximum content size stored per file for in-memory search (bytes)
|
||||
SEARCH_CONTENT_LIMIT = 100_000
|
||||
@@ -68,10 +69,10 @@ SUPPORTED_EXTENSIONS = {
|
||||
# Ignored directories (configurable via OBSIGATE_IGNORED_DIRS env var)
|
||||
_DEFAULT_IGNORED = {'.obsidian', '.trash', '.git', '__pycache__', 'node_modules', '.obsigate-backup'}
|
||||
_env_ignored = os.environ.get("OBSIGATE_IGNORED_DIRS", "")
|
||||
IGNORED_DIRS = set(d.strip() for d in _env_ignored.split(",") if d.strip()) if _env_ignored else _DEFAULT_IGNORED.copy()
|
||||
IGNORED_DIRS = {d.strip() for d in _env_ignored.split(",") if d.strip()} if _env_ignored else _DEFAULT_IGNORED.copy()
|
||||
|
||||
|
||||
def load_vault_config() -> Dict[str, Dict[str, Any]]:
|
||||
def load_vault_config() -> dict[str, dict[str, Any]]:
|
||||
"""Read VAULT_N_* and DIR_N_* env vars and return vault configuration.
|
||||
|
||||
Scans environment variables ``VAULT_1_NAME``/``VAULT_1_PATH``,
|
||||
@@ -89,7 +90,7 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
|
||||
- scanAttachmentsOnStartup: boolean (default True)
|
||||
- type: "VAULT" or "DIR"
|
||||
"""
|
||||
vaults: Dict[str, Dict[str, Any]] = {}
|
||||
vaults: dict[str, dict[str, Any]] = {}
|
||||
n = 1
|
||||
while True:
|
||||
name = os.environ.get(f"VAULT_{n}_NAME")
|
||||
@@ -135,7 +136,7 @@ _CODE_BLOCK_RE = re.compile(r'```[\s\S]*?```', re.MULTILINE)
|
||||
_INLINE_CODE_RE = re.compile(r'`[^`]+`')
|
||||
|
||||
|
||||
def _extract_tags(post: frontmatter.Post) -> List[str]:
|
||||
def _extract_tags(post: frontmatter.Post) -> list[str]:
|
||||
"""Extract tags from frontmatter metadata.
|
||||
|
||||
|
||||
@@ -158,7 +159,7 @@ def _extract_tags(post: frontmatter.Post) -> List[str]:
|
||||
return tags
|
||||
|
||||
|
||||
def _extract_inline_tags(content: str) -> List[str]:
|
||||
def _extract_inline_tags(content: str) -> list[str]:
|
||||
"""Extract inline #tag patterns from markdown content.
|
||||
|
||||
Strips fenced and inline code blocks before scanning to avoid
|
||||
@@ -215,10 +216,10 @@ def parse_markdown_file(raw: str) -> frontmatter.Post:
|
||||
match = re.match(r"^---\s*\r?\n.*?\r?\n---\s*\r?\n?", raw, flags=re.DOTALL)
|
||||
if match:
|
||||
content = raw[match.end():]
|
||||
return frontmatter.Post(content, **{})
|
||||
return frontmatter.Post(content)
|
||||
|
||||
|
||||
def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Synchronously scan a single vault directory and build file index.
|
||||
|
||||
Walks the vault tree, reads supported files, extracts metadata
|
||||
@@ -236,9 +237,9 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
|
||||
Dict with keys ``files`` (list), ``tags`` (counter dict), ``path`` (str), ``paths`` (list).
|
||||
"""
|
||||
vault_root = Path(vault_path)
|
||||
files: List[Dict[str, Any]] = []
|
||||
tag_counts: Dict[str, int] = {}
|
||||
paths: List[Dict[str, str]] = []
|
||||
files: list[dict[str, Any]] = []
|
||||
tag_counts: dict[str, int] = {}
|
||||
paths: list[dict[str, str]] = []
|
||||
|
||||
if not vault_root.exists():
|
||||
logger.warning(f"Vault path does not exist: {vault_path}")
|
||||
@@ -283,15 +284,15 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
|
||||
|
||||
# PDF handling — special path (binary, uses pdf_reader)
|
||||
if ext == ".pdf":
|
||||
from backend.pdf_reader import extract_pdf_text, extract_pdf_metadata
|
||||
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text
|
||||
raw = extract_pdf_text(fpath, max_chars=100000)
|
||||
pdf_meta = extract_pdf_metadata(fpath)
|
||||
title = pdf_meta.get("title") or fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
tags: List[str] = []
|
||||
tags: list[str] = []
|
||||
else:
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
tags: List[str] = []
|
||||
tags: list[str] = []
|
||||
title = fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
|
||||
@@ -365,13 +366,13 @@ async def build_index(progress_callback=None) -> None:
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
async def _process_vault(name: str, config: Dict[str, Any]):
|
||||
async def _process_vault(name: str, config: dict[str, Any]):
|
||||
vault_path = config["path"]
|
||||
vault_data = await loop.run_in_executor(None, _scan_vault, name, vault_path, config)
|
||||
vault_data["config"] = config
|
||||
|
||||
# Build lookup entries for the new vault
|
||||
new_lookup_entries: Dict[str, List[Dict[str, str]]] = {}
|
||||
new_lookup_entries: dict[str, list[dict[str, str]]] = {}
|
||||
for f in vault_data["files"]:
|
||||
entry = {"vault": name, "path": f["path"]}
|
||||
fname = f["path"].rsplit("/", 1)[-1].lower()
|
||||
@@ -419,7 +420,7 @@ async def build_index(progress_callback=None) -> None:
|
||||
await progress_callback("complete", {"total_vaults": len(vault_config), "total_files": total_files})
|
||||
|
||||
|
||||
async def reload_index() -> Dict[str, Any]:
|
||||
async def reload_index() -> dict[str, Any]:
|
||||
"""Force a full re-index of all vaults and return per-vault statistics.
|
||||
|
||||
Returns:
|
||||
@@ -432,7 +433,7 @@ async def reload_index() -> Dict[str, Any]:
|
||||
return stats
|
||||
|
||||
|
||||
async def reload_single_vault(vault_name: str) -> Dict[str, Any]:
|
||||
async def reload_single_vault(vault_name: str) -> dict[str, Any]:
|
||||
"""Force a re-index of a single vault and return its statistics.
|
||||
|
||||
Args:
|
||||
@@ -464,7 +465,7 @@ async def reload_single_vault(vault_name: str) -> Dict[str, Any]:
|
||||
vault_data["config"] = config
|
||||
|
||||
# Build lookup entries for the vault
|
||||
new_lookup_entries: Dict[str, List[Dict[str, str]]] = {}
|
||||
new_lookup_entries: dict[str, list[dict[str, str]]] = {}
|
||||
for f in vault_data["files"]:
|
||||
entry = {"vault": vault_name, "path": f["path"]}
|
||||
fname = f["path"].rsplit("/", 1)[-1].lower()
|
||||
@@ -495,12 +496,12 @@ async def reload_single_vault(vault_name: str) -> Dict[str, Any]:
|
||||
return stats
|
||||
|
||||
|
||||
def get_vault_names() -> List[str]:
|
||||
def get_vault_names() -> list[str]:
|
||||
"""Return the list of all indexed vault names."""
|
||||
return list(index.keys())
|
||||
|
||||
|
||||
def get_vault_data(vault_name: str) -> Optional[Dict[str, Any]]:
|
||||
def get_vault_data(vault_name: str) -> dict[str, Any] | None:
|
||||
"""Return the full index data for a vault, or ``None`` if not found."""
|
||||
return index.get(vault_name)
|
||||
|
||||
@@ -513,7 +514,7 @@ def _get_async_lock() -> asyncio.Lock:
|
||||
return _async_index_lock
|
||||
|
||||
|
||||
def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, vault_cfg: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, vault_cfg: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
||||
"""Synchronously read and parse a single file for indexing.
|
||||
|
||||
All files are indexed, including hidden files (starting with '.').
|
||||
@@ -545,7 +546,7 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
tags: List[str] = []
|
||||
tags: list[str] = []
|
||||
title = fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
|
||||
@@ -575,7 +576,7 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
|
||||
return None
|
||||
|
||||
|
||||
def _remove_file_from_structures(vault_name: str, rel_path: str) -> Optional[Dict[str, Any]]:
|
||||
def _remove_file_from_structures(vault_name: str, rel_path: str) -> dict[str, Any] | None:
|
||||
"""Remove a file from all index structures. Returns removed file info or None.
|
||||
|
||||
Must be called under _index_lock or _async_index_lock.
|
||||
@@ -650,7 +651,7 @@ def _ensure_parent_dirs_in_path_index(vault_name: str, rel_path: str, existing:
|
||||
})
|
||||
|
||||
|
||||
def _add_file_to_structures(vault_name: str, file_info: Dict[str, Any]):
|
||||
def _add_file_to_structures(vault_name: str, file_info: dict[str, Any]):
|
||||
"""Add a file entry to all index structures.
|
||||
|
||||
Must be called under _index_lock or _async_index_lock.
|
||||
@@ -695,7 +696,7 @@ def _add_file_to_structures(vault_name: str, file_info: Dict[str, Any]):
|
||||
_on_index_change('add', vault_name, file_info["path"], file_info) # type: ignore[misc]
|
||||
|
||||
|
||||
async def update_single_file(vault_name: str, abs_file_path: str) -> Optional[Dict[str, Any]]:
|
||||
async def update_single_file(vault_name: str, abs_file_path: str) -> dict[str, Any] | None:
|
||||
"""Re-index a single file without full rebuild.
|
||||
|
||||
Reads the file, removes the old entry if present, inserts the new one.
|
||||
@@ -742,7 +743,7 @@ async def update_single_file(vault_name: str, abs_file_path: str) -> Optional[Di
|
||||
return file_info
|
||||
|
||||
|
||||
async def remove_single_file(vault_name: str, abs_file_path: str) -> Optional[Dict[str, Any]]:
|
||||
async def remove_single_file(vault_name: str, abs_file_path: str) -> dict[str, Any] | None:
|
||||
"""Remove a single file from the index.
|
||||
|
||||
Args:
|
||||
@@ -774,7 +775,7 @@ async def remove_single_file(vault_name: str, abs_file_path: str) -> Optional[Di
|
||||
return removed
|
||||
|
||||
|
||||
async def handle_file_move(vault_name: str, src_abs: str, dest_abs: str) -> Optional[Dict[str, Any]]:
|
||||
async def handle_file_move(vault_name: str, src_abs: str, dest_abs: str) -> dict[str, Any] | None:
|
||||
"""Handle a file move/rename by removing old entry and indexing new location.
|
||||
|
||||
Args:
|
||||
@@ -823,7 +824,7 @@ async def remove_vault_from_index(vault_name: str):
|
||||
logger.info(f"Removed vault '{vault_name}' from index")
|
||||
|
||||
|
||||
async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]:
|
||||
async def add_vault_to_index(vault_name: str, vault_path: str) -> dict[str, Any]:
|
||||
"""Add a new vault to the index dynamically.
|
||||
|
||||
Args:
|
||||
@@ -846,7 +847,7 @@ async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]
|
||||
vault_data["config"] = vault_config[vault_name]
|
||||
|
||||
# Build lookup entries for the new vault
|
||||
new_lookup_entries: Dict[str, List[Dict[str, str]]] = {}
|
||||
new_lookup_entries: dict[str, list[dict[str, str]]] = {}
|
||||
for f in vault_data["files"]:
|
||||
entry = {"vault": vault_name, "path": f["path"]}
|
||||
fname = f["path"].rsplit("/", 1)[-1].lower()
|
||||
@@ -871,7 +872,7 @@ async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]
|
||||
return stats
|
||||
|
||||
|
||||
def find_file_in_index(link_target: str, current_vault: str) -> Optional[Dict[str, str]]:
|
||||
def find_file_in_index(link_target: str, current_vault: str) -> dict[str, str] | None:
|
||||
"""Find a file matching a wikilink target using O(1) lookup table.
|
||||
|
||||
Searches by filename first, then by full relative path.
|
||||
@@ -943,7 +944,7 @@ def _extract_wikilinks_for_backlinks(
|
||||
})
|
||||
|
||||
|
||||
def get_backlinks(vault_name: str, file_path: str) -> List[Dict[str, str]]:
|
||||
def get_backlinks(vault_name: str, file_path: str) -> list[dict[str, str]]:
|
||||
"""Get all files that link to the given file via wikilinks.
|
||||
|
||||
Searches across all vaults for backlinks pointing to the target file.
|
||||
@@ -961,7 +962,7 @@ def get_backlinks(vault_name: str, file_path: str) -> List[Dict[str, str]]:
|
||||
target_key += ".md"
|
||||
|
||||
results = []
|
||||
for vname, vindex in _backlink_index.items():
|
||||
for vindex in _backlink_index.values():
|
||||
bl = vindex.get(target_key, [])
|
||||
results.extend(bl)
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user