refactor: remove hidden files indexing configuration and convert to UI-only display preference, eliminating includeHidden and hiddenWhitelist from vault config and indexing logic while adding hideHiddenFiles client-side filtering
This commit is contained in:
+8
-59
@@ -9,8 +9,6 @@ from typing import Dict, List, Optional, Any
|
||||
|
||||
import frontmatter
|
||||
|
||||
from backend.utils import should_include_path
|
||||
|
||||
logger = logging.getLogger("obsigate.indexer")
|
||||
|
||||
# Global in-memory index
|
||||
@@ -61,16 +59,12 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
|
||||
Also reads optional configuration:
|
||||
- VAULT_N_ATTACHMENTS_PATH: relative path to attachments folder
|
||||
- VAULT_N_SCAN_ATTACHMENTS: "true"/"false" to enable/disable scanning
|
||||
- VAULT_N_INCLUDE_HIDDEN: "true"/"false" to include all hidden files/folders
|
||||
- VAULT_N_HIDDEN_WHITELIST: comma-separated list of hidden paths to include (e.g., ".obsidian,.github")
|
||||
|
||||
Returns:
|
||||
Dict mapping vault names to configuration dicts with keys:
|
||||
- path: filesystem path (required)
|
||||
- attachmentsPath: relative attachments folder (optional)
|
||||
- scanAttachmentsOnStartup: boolean (default True)
|
||||
- includeHidden: boolean (default False) - include all hidden files/folders
|
||||
- hiddenWhitelist: list of hidden paths to include even if includeHidden is False
|
||||
- type: "VAULT" or "DIR"
|
||||
"""
|
||||
vaults: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -84,16 +78,11 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
|
||||
# Optional configuration
|
||||
attachments_path = os.environ.get(f"VAULT_{n}_ATTACHMENTS_PATH")
|
||||
scan_attachments = os.environ.get(f"VAULT_{n}_SCAN_ATTACHMENTS", "true").lower() == "true"
|
||||
include_hidden = os.environ.get(f"VAULT_{n}_INCLUDE_HIDDEN", "false").lower() == "true"
|
||||
hidden_whitelist_str = os.environ.get(f"VAULT_{n}_HIDDEN_WHITELIST", "")
|
||||
hidden_whitelist = [item.strip() for item in hidden_whitelist_str.split(",") if item.strip()]
|
||||
|
||||
vaults[name] = {
|
||||
"path": path,
|
||||
"attachmentsPath": attachments_path,
|
||||
"scanAttachmentsOnStartup": scan_attachments,
|
||||
"includeHidden": include_hidden,
|
||||
"hiddenWhitelist": hidden_whitelist,
|
||||
"type": "VAULT"
|
||||
}
|
||||
n += 1
|
||||
@@ -105,16 +94,10 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
|
||||
if not name or not path:
|
||||
break
|
||||
|
||||
include_hidden = os.environ.get(f"DIR_{n}_INCLUDE_HIDDEN", "false").lower() == "true"
|
||||
hidden_whitelist_str = os.environ.get(f"DIR_{n}_HIDDEN_WHITELIST", "")
|
||||
hidden_whitelist = [item.strip() for item in hidden_whitelist_str.split(",") if item.strip()]
|
||||
|
||||
vaults[name] = {
|
||||
"path": path,
|
||||
"attachmentsPath": None,
|
||||
"scanAttachmentsOnStartup": False,
|
||||
"includeHidden": include_hidden,
|
||||
"hiddenWhitelist": hidden_whitelist,
|
||||
"type": "DIR"
|
||||
}
|
||||
n += 1
|
||||
@@ -219,11 +202,13 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
|
||||
Walks the vault tree, reads supported files, extracts metadata
|
||||
(tags, title, content preview) and stores a capped content snapshot
|
||||
for in-memory full-text search.
|
||||
|
||||
All files and directories are indexed, including hidden files (starting with '.').
|
||||
|
||||
Args:
|
||||
vault_name: Display name of the vault.
|
||||
vault_path: Absolute filesystem path to the vault root.
|
||||
vault_cfg: Optional vault configuration dict with hidden files settings.
|
||||
vault_cfg: Optional vault configuration dict (unused for indexing, kept for compatibility).
|
||||
|
||||
Returns:
|
||||
Dict with keys ``files`` (list), ``tags`` (counter dict), ``path`` (str), ``paths`` (list).
|
||||
@@ -232,21 +217,12 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str,
|
||||
files: List[Dict[str, Any]] = []
|
||||
tag_counts: Dict[str, int] = {}
|
||||
paths: List[Dict[str, str]] = []
|
||||
|
||||
# Default config if not provided
|
||||
if vault_cfg is None:
|
||||
vault_cfg = {"includeHidden": False, "hiddenWhitelist": []}
|
||||
|
||||
if not vault_root.exists():
|
||||
logger.warning(f"Vault path does not exist: {vault_path}")
|
||||
return {"files": [], "tags": {}, "path": vault_path, "paths": []}
|
||||
|
||||
for fpath in vault_root.rglob("*"):
|
||||
# Check if path should be included based on hidden files configuration
|
||||
rel_parts = fpath.relative_to(vault_root).parts
|
||||
if not should_include_path(rel_parts, vault_cfg):
|
||||
continue
|
||||
|
||||
rel_path_str = str(fpath.relative_to(vault_root)).replace("\\", "/")
|
||||
|
||||
# Add all paths (files and directories) to path index
|
||||
@@ -329,17 +305,8 @@ async def build_index(progress_callback=None) -> None:
|
||||
vault_config.clear()
|
||||
vault_config.update(load_vault_config())
|
||||
|
||||
# Merge vault_settings (from UI) with vault_config (from env vars)
|
||||
from backend.vault_settings import get_all_vault_settings
|
||||
saved_settings = get_all_vault_settings()
|
||||
for vault_name, config in vault_config.items():
|
||||
if vault_name in saved_settings:
|
||||
settings = saved_settings[vault_name]
|
||||
# Override with saved settings if present
|
||||
if "includeHidden" in settings:
|
||||
config["includeHidden"] = settings["includeHidden"]
|
||||
if "hiddenWhitelist" in settings:
|
||||
config["hiddenWhitelist"] = settings["hiddenWhitelist"]
|
||||
# Note: vault_settings are now only used for UI display preferences (hideHiddenFiles)
|
||||
# Indexing always includes all files regardless of settings
|
||||
|
||||
global _index_generation
|
||||
with _index_lock:
|
||||
@@ -443,22 +410,11 @@ async def reload_single_vault(vault_name: str) -> Dict[str, Any]:
|
||||
# Reload vault config from env vars
|
||||
vault_config.update(load_vault_config())
|
||||
|
||||
# Merge with saved settings
|
||||
from backend.vault_settings import get_vault_setting
|
||||
saved_settings = get_vault_setting(vault_name)
|
||||
|
||||
if vault_name not in vault_config:
|
||||
raise ValueError(f"Vault '{vault_name}' not found in configuration")
|
||||
|
||||
config = vault_config[vault_name]
|
||||
|
||||
# Override with saved settings if present
|
||||
if saved_settings:
|
||||
if "includeHidden" in saved_settings:
|
||||
config["includeHidden"] = saved_settings["includeHidden"]
|
||||
if "hiddenWhitelist" in saved_settings:
|
||||
config["hiddenWhitelist"] = saved_settings["hiddenWhitelist"]
|
||||
|
||||
# Remove old vault data from index structures
|
||||
await remove_vault_from_index(vault_name)
|
||||
|
||||
@@ -520,12 +476,14 @@ def _get_async_lock() -> asyncio.Lock:
|
||||
|
||||
def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, vault_cfg: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Synchronously read and parse a single file for indexing.
|
||||
|
||||
All files are indexed, including hidden files (starting with '.').
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
vault_path: Absolute path to vault root.
|
||||
file_path: Absolute path to the file.
|
||||
vault_cfg: Optional vault configuration dict with hidden files settings.
|
||||
vault_cfg: Optional vault configuration dict (unused for indexing, kept for compatibility).
|
||||
|
||||
Returns:
|
||||
File info dict or None if the file cannot be read.
|
||||
@@ -538,13 +496,6 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
|
||||
return None
|
||||
|
||||
relative = fpath.relative_to(vault_root)
|
||||
rel_parts = relative.parts
|
||||
|
||||
# Check if path should be included based on hidden files configuration
|
||||
if vault_cfg is None:
|
||||
vault_cfg = {"includeHidden": False, "hiddenWhitelist": []}
|
||||
if not should_include_path(rel_parts, vault_cfg):
|
||||
return None
|
||||
|
||||
ext = fpath.suffix.lower()
|
||||
basename_lower = fpath.name.lower()
|
||||
@@ -815,8 +766,6 @@ async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]
|
||||
"path": vault_path,
|
||||
"attachmentsPath": None,
|
||||
"scanAttachmentsOnStartup": True,
|
||||
"includeHidden": False,
|
||||
"hiddenWhitelist": [],
|
||||
}
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
Reference in New Issue
Block a user