Projets/ObsiGate
1
1
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 16 Wiki Activity

refactor: remove hidden files indexing configuration and convert to UI-only display preference, eliminating includeHidden and hiddenWhitelist from vault config and indexing logic while adding hideHiddenFiles client-side filtering

Browse Source
This commit is contained in:
bruno
2026-03-26 19:53:40 -04:00
parent 34f4e41419
commit 2b69c49ed1
7 changed files with 98 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/attachment_indexer.py
+2 -13
View File
@@ -4,8 +4,6 @@ from pathlib import Path
from typing import Dict, List, Optional, Set
import threading
from backend.utils import should_include_path
logger = logging.getLogger("obsigate.attachment_indexer")
# Image file extensions to index
@@ -40,12 +38,12 @@ def _scan_vault_attachments(vault_name: str, vault_path: str, vault_cfg: dict =
"""Synchronously scan a vault directory for image attachments.
Walks the vault tree and builds a filename -> absolute path mapping
for all image files.
for all image files. All files are scanned, including hidden files (starting with '.').
Args:
vault_name: Display name of the vault.
vault_path: Absolute filesystem path to the vault root.
vault_cfg: Optional vault configuration dict with hidden files settings.
vault_cfg: Optional vault configuration dict (unused for indexing, kept for compatibility).
Returns:
Dict mapping lowercase filenames to lists of absolute paths.
@@ -53,10 +51,6 @@ def _scan_vault_attachments(vault_name: str, vault_path: str, vault_cfg: dict =
vault_root = Path(vault_path)
index: Dict[str, List[Path]] = {}
# Default config if not provided
if vault_cfg is None:
vault_cfg = {"includeHidden": False, "hiddenWhitelist": []}
if not vault_root.exists():
logger.warning(f"Vault path does not exist for attachment scan: {vault_path}")
return index
@@ -65,11 +59,6 @@ def _scan_vault_attachments(vault_name: str, vault_path: str, vault_cfg: dict =
try:
for fpath in vault_root.rglob("*"):
# Check if path should be included based on hidden files configuration
rel_parts = fpath.relative_to(vault_root).parts
if not should_include_path(rel_parts, vault_cfg):
continue
# Only process files
if not fpath.is_file():
continue
backend/indexer.py
+8 -59
View File
@@ -9,8 +9,6 @@ from typing import Dict, List, Optional, Any
import frontmatter
from backend.utils import should_include_path
logger = logging.getLogger("obsigate.indexer")
# Global in-memory index
@@ -61,16 +59,12 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
Also reads optional configuration:
- VAULT_N_ATTACHMENTS_PATH: relative path to attachments folder
- VAULT_N_SCAN_ATTACHMENTS: "true"/"false" to enable/disable scanning
- VAULT_N_INCLUDE_HIDDEN: "true"/"false" to include all hidden files/folders
- VAULT_N_HIDDEN_WHITELIST: comma-separated list of hidden paths to include (e.g., ".obsidian,.github")
Returns:
Dict mapping vault names to configuration dicts with keys:
- path: filesystem path (required)
- attachmentsPath: relative attachments folder (optional)
- scanAttachmentsOnStartup: boolean (default True)
- includeHidden: boolean (default False) - include all hidden files/folders
- hiddenWhitelist: list of hidden paths to include even if includeHidden is False
- type: "VAULT" or "DIR"
"""
vaults: Dict[str, Dict[str, Any]] = {}
@@ -84,16 +78,11 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
# Optional configuration
attachments_path = os.environ.get(f"VAULT_{n}_ATTACHMENTS_PATH")
scan_attachments = os.environ.get(f"VAULT_{n}_SCAN_ATTACHMENTS", "true").lower() == "true"
include_hidden = os.environ.get(f"VAULT_{n}_INCLUDE_HIDDEN", "false").lower() == "true"
hidden_whitelist_str = os.environ.get(f"VAULT_{n}_HIDDEN_WHITELIST", "")
hidden_whitelist = [item.strip() for item in hidden_whitelist_str.split(",") if item.strip()]
vaults[name] = {
"path": path,
"attachmentsPath": attachments_path,
"scanAttachmentsOnStartup": scan_attachments,
"includeHidden": include_hidden,
"hiddenWhitelist": hidden_whitelist,
"type": "VAULT"
}
n += 1
@@ -105,16 +94,10 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
if not name or not path:
break
include_hidden = os.environ.get(f"DIR_{n}_INCLUDE_HIDDEN", "false").lower() == "true"
hidden_whitelist_str = os.environ.get(f"DIR_{n}_HIDDEN_WHITELIST", "")
hidden_whitelist = [item.strip() for item in hidden_whitelist_str.split(",") if item.strip()]
vaults[name] = {
"path": path,
"attachmentsPath": None,
"scanAttachmentsOnStartup": False,
"includeHidden": include_hidden,
"hiddenWhitelist": hidden_whitelist,
"type": "DIR"
}
n += 1
@@ -219,11 +202,13 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
Walks the vault tree, reads supported files, extracts metadata
(tags, title, content preview) and stores a capped content snapshot
for in-memory full-text search.
All files and directories are indexed, including hidden files (starting with '.').
Args:
vault_name: Display name of the vault.
vault_path: Absolute filesystem path to the vault root.
vault_cfg: Optional vault configuration dict with hidden files settings.
vault_cfg: Optional vault configuration dict (unused for indexing, kept for compatibility).
Returns:
Dict with keys ``files`` (list), ``tags`` (counter dict), ``path`` (str), ``paths`` (list).
@@ -232,21 +217,12 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
files: List[Dict[str, Any]] = []
tag_counts: Dict[str, int] = {}
paths: List[Dict[str, str]] = []
# Default config if not provided
if vault_cfg is None:
vault_cfg = {"includeHidden": False, "hiddenWhitelist": []}
if not vault_root.exists():
logger.warning(f"Vault path does not exist: {vault_path}")
return {"files": [], "tags": {}, "path": vault_path, "paths": []}
for fpath in vault_root.rglob("*"):
# Check if path should be included based on hidden files configuration
rel_parts = fpath.relative_to(vault_root).parts
if not should_include_path(rel_parts, vault_cfg):
continue
rel_path_str = str(fpath.relative_to(vault_root)).replace("\\", "/")
# Add all paths (files and directories) to path index
@@ -329,17 +305,8 @@ async def build_index(progress_callback=None) -> None:
vault_config.clear()
vault_config.update(load_vault_config())
# Merge vault_settings (from UI) with vault_config (from env vars)
from backend.vault_settings import get_all_vault_settings
saved_settings = get_all_vault_settings()
for vault_name, config in vault_config.items():
if vault_name in saved_settings:
settings = saved_settings[vault_name]
# Override with saved settings if present
if "includeHidden" in settings:
config["includeHidden"] = settings["includeHidden"]
if "hiddenWhitelist" in settings:
config["hiddenWhitelist"] = settings["hiddenWhitelist"]
# Note: vault_settings are now only used for UI display preferences (hideHiddenFiles)
# Indexing always includes all files regardless of settings
global _index_generation
with _index_lock:
@@ -443,22 +410,11 @@ async def reload_single_vault(vault_name: str) -> Dict[str, Any]:
# Reload vault config from env vars
vault_config.update(load_vault_config())
# Merge with saved settings
from backend.vault_settings import get_vault_setting
saved_settings = get_vault_setting(vault_name)
if vault_name not in vault_config:
raise ValueError(f"Vault '{vault_name}' not found in configuration")
config = vault_config[vault_name]
# Override with saved settings if present
if saved_settings:
if "includeHidden" in saved_settings:
config["includeHidden"] = saved_settings["includeHidden"]
if "hiddenWhitelist" in saved_settings:
config["hiddenWhitelist"] = saved_settings["hiddenWhitelist"]
# Remove old vault data from index structures
await remove_vault_from_index(vault_name)
@@ -520,12 +476,14 @@ def _get_async_lock() -> asyncio.Lock:
def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, vault_cfg: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
"""Synchronously read and parse a single file for indexing.
All files are indexed, including hidden files (starting with '.').
Args:
vault_name: Name of the vault.
vault_path: Absolute path to vault root.
file_path: Absolute path to the file.
vault_cfg: Optional vault configuration dict with hidden files settings.
vault_cfg: Optional vault configuration dict (unused for indexing, kept for compatibility).
Returns:
File info dict or None if the file cannot be read.
@@ -538,13 +496,6 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
return None
relative = fpath.relative_to(vault_root)
rel_parts = relative.parts
# Check if path should be included based on hidden files configuration
if vault_cfg is None:
vault_cfg = {"includeHidden": False, "hiddenWhitelist": []}
if not should_include_path(rel_parts, vault_cfg):
return None
ext = fpath.suffix.lower()
basename_lower = fpath.name.lower()
@@ -815,8 +766,6 @@ async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]
"path": vault_path,
"attachmentsPath": None,
"scanAttachmentsOnStartup": True,
"includeHidden": False,
"hiddenWhitelist": [],
}
loop = asyncio.get_event_loop()
backend/main.py
+13 -31
View File
@@ -1414,32 +1414,28 @@ async def api_attachment_stats(vault: Optional[str] = Query(None, description="V
# ---------------------------------------------------------------------------
# Vault Settings API — Hidden files configuration
# Vault Settings API — Display preferences
# ---------------------------------------------------------------------------
@app.get("/api/vaults/{vault_name}/settings")
async def api_get_vault_settings(vault_name: str, current_user=Depends(require_auth)):
"""Get settings for a specific vault including hidden files configuration.
"""Get UI display settings for a specific vault.
Args:
vault_name: Name of the vault.
Returns:
Dict with vault settings including includeHidden and hiddenWhitelist.
Dict with vault settings including hideHiddenFiles.
"""
if vault_name not in index:
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
# Get current vault config from environment
vault_cfg = vault_config.get(vault_name, {})
# Get persisted settings
persisted = get_vault_setting(vault_name) or {}
# Merge with defaults
# Default settings
settings = {
"includeHidden": vault_cfg.get("includeHidden", False),
"hiddenWhitelist": vault_cfg.get("hiddenWhitelist", []),
"hideHiddenFiles": False,
}
settings.update(persisted)
@@ -1448,11 +1444,11 @@ async def api_get_vault_settings(vault_name: str, current_user=Depends(require_a
@app.post("/api/vaults/{vault_name}/settings")
async def api_update_vault_settings(vault_name: str, body: dict = Body(...), current_user=Depends(require_admin)):
"""Update settings for a specific vault.
"""Update UI display settings for a specific vault.
Args:
vault_name: Name of the vault.
body: Dict with settings to update (includeHidden, hiddenWhitelist).
body: Dict with settings to update (hideHiddenFiles).
Returns:
Updated settings dict.
@@ -1463,18 +1459,10 @@ async def api_update_vault_settings(vault_name: str, body: dict = Body(...), cur
# Validate settings
settings_to_update = {}
if "includeHidden" in body:
if not isinstance(body["includeHidden"], bool):
raise HTTPException(status_code=400, detail="includeHidden must be a boolean")
settings_to_update["includeHidden"] = body["includeHidden"]
if "hiddenWhitelist" in body:
if not isinstance(body["hiddenWhitelist"], list):
raise HTTPException(status_code=400, detail="hiddenWhitelist must be a list")
# Validate each item is a string
if not all(isinstance(item, str) for item in body["hiddenWhitelist"]):
raise HTTPException(status_code=400, detail="All hiddenWhitelist items must be strings")
settings_to_update["hiddenWhitelist"] = body["hiddenWhitelist"]
if "hideHiddenFiles" in body:
if not isinstance(body["hideHiddenFiles"], bool):
raise HTTPException(status_code=400, detail="hideHiddenFiles must be a boolean")
settings_to_update["hideHiddenFiles"] = body["hideHiddenFiles"]
# Update persisted settings
try:
@@ -1492,10 +1480,6 @@ async def api_update_vault_settings(vault_name: str, body: dict = Body(...), cur
detail=f"Failed to save settings: {str(e)}"
)
# Update in-memory vault config
if vault_name in vault_config:
vault_config[vault_name].update(settings_to_update)
logger.info(f"Updated settings for vault '{vault_name}': {settings_to_update}")
return updated
@@ -1503,7 +1487,7 @@ async def api_update_vault_settings(vault_name: str, body: dict = Body(...), cur
@app.get("/api/vaults/settings/all")
async def api_get_all_vault_settings(current_user=Depends(require_auth)):
"""Get settings for all vaults.
"""Get UI display settings for all vaults.
Returns:
Dict mapping vault names to their settings.
@@ -1511,12 +1495,10 @@ async def api_get_all_vault_settings(current_user=Depends(require_auth)):
all_settings = {}
for vault_name in index.keys():
vault_cfg = vault_config.get(vault_name, {})
persisted = get_vault_setting(vault_name) or {}
settings = {
"includeHidden": vault_cfg.get("includeHidden", False),
"hiddenWhitelist": vault_cfg.get("hiddenWhitelist", []),
"hideHiddenFiles": False,
}
settings.update(persisted)
all_settings[vault_name] = settings
backend/utils.py
-40
View File
@@ -1,42 +1,2 @@
"""Utility functions shared across backend modules."""
from typing import Dict, Any, Tuple
def should_include_path(rel_parts: Tuple[str, ...], vault_config: Dict[str, Any]) -> bool:
"""Check if a path should be included based on hidden files configuration.
Simplified logic:
- If includeHidden=True: include ALL hidden files/folders recursively
- If includeHidden=False: check whitelist
- If a whitelisted folder is in the path, include ALL sub-hidden files/folders
- Otherwise, exclude the path
Args:
rel_parts: Tuple of path parts relative to vault root
vault_config: Vault configuration dict with includeHidden and hiddenWhitelist
Returns:
True if the path should be included, False otherwise
"""
include_hidden = vault_config.get("includeHidden", False)
hidden_whitelist = vault_config.get("hiddenWhitelist", [])
# Check if any part of the path starts with a dot (hidden)
hidden_parts = [part for part in rel_parts if part.startswith(".")]
if not hidden_parts:
# No hidden parts, always include
return True
if include_hidden:
# Include all hidden files/folders recursively
return True
# Check if the FIRST hidden part is in the whitelist
# If yes, include all sub-hidden files/folders automatically
first_hidden = hidden_parts[0]
if first_hidden in hidden_whitelist:
return True
# First hidden part not in whitelist, exclude
return False
backend/vault_settings.py
+5 -1
View File
@@ -1,7 +1,11 @@
"""Vault-specific settings management.
Provides persistent storage for per-vault configuration like hidden files settings.
Provides persistent storage for per-vault UI display preferences.
Settings are stored in /app/data/vault_settings.json
Current settings:
- hideHiddenFiles (bool): Whether to hide files/folders starting with '.' in the UI
Note: All files are always indexed regardless of this setting. This only affects display.
"""
import json