feat: add hidden files configuration with per-vault settings for includeHidden and hiddenWhitelist, supporting environment variables and UI controls for selective indexing of dot-prefixed files and folders

backend/indexer.py

@@ -61,12 +61,16 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
     Also reads optional configuration:
     - VAULT_N_ATTACHMENTS_PATH: relative path to attachments folder
     - VAULT_N_SCAN_ATTACHMENTS: "true"/"false" to enable/disable scanning
+    - VAULT_N_INCLUDE_HIDDEN: "true"/"false" to include all hidden files/folders
+    - VAULT_N_HIDDEN_WHITELIST: comma-separated list of hidden paths to include (e.g., ".obsidian,.github")
 
     Returns:
         Dict mapping vault names to configuration dicts with keys:
         - path: filesystem path (required)
         - attachmentsPath: relative attachments folder (optional)
         - scanAttachmentsOnStartup: boolean (default True)
+        - includeHidden: boolean (default False) - include all hidden files/folders
+        - hiddenWhitelist: list of hidden paths to include even if includeHidden is False
         - type: "VAULT" or "DIR"
     """
     vaults: Dict[str, Dict[str, Any]] = {}
@@ -80,11 +84,16 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
         # Optional configuration
         attachments_path = os.environ.get(f"VAULT_{n}_ATTACHMENTS_PATH")
         scan_attachments = os.environ.get(f"VAULT_{n}_SCAN_ATTACHMENTS", "true").lower() == "true"
+        include_hidden = os.environ.get(f"VAULT_{n}_INCLUDE_HIDDEN", "false").lower() == "true"
+        hidden_whitelist_str = os.environ.get(f"VAULT_{n}_HIDDEN_WHITELIST", "")
+        hidden_whitelist = [item.strip() for item in hidden_whitelist_str.split(",") if item.strip()]
         
         vaults[name] = {
             "path": path,
             "attachmentsPath": attachments_path,
             "scanAttachmentsOnStartup": scan_attachments,
+            "includeHidden": include_hidden,
+            "hiddenWhitelist": hidden_whitelist,
             "type": "VAULT"
         }
         n += 1
@@ -96,10 +105,16 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
         if not name or not path:
             break
         
+        include_hidden = os.environ.get(f"DIR_{n}_INCLUDE_HIDDEN", "false").lower() == "true"
+        hidden_whitelist_str = os.environ.get(f"DIR_{n}_HIDDEN_WHITELIST", "")
+        hidden_whitelist = [item.strip() for item in hidden_whitelist_str.split(",") if item.strip()]
+        
         vaults[name] = {
             "path": path,
             "attachmentsPath": None,
             "scanAttachmentsOnStartup": False,
+            "includeHidden": include_hidden,
+            "hiddenWhitelist": hidden_whitelist,
             "type": "DIR"
         }
         n += 1
@@ -107,15 +122,49 @@ def load_vault_config() -> Dict[str, Dict[str, Any]]:
     return vaults
 
 
+def _should_include_path(rel_parts: tuple, vault_config: Dict[str, Any]) -> bool:
+    """Check if a path should be included based on hidden files configuration.
+    
+    Args:
+        rel_parts: Tuple of path parts relative to vault root
+        vault_config: Vault configuration dict with includeHidden and hiddenWhitelist
+        
+    Returns:
+        True if the path should be included, False otherwise
+    """
+    include_hidden = vault_config.get("includeHidden", False)
+    hidden_whitelist = vault_config.get("hiddenWhitelist", [])
+    
+    # Check if any part of the path starts with a dot (hidden)
+    hidden_parts = [part for part in rel_parts if part.startswith(".")]
+    
+    if not hidden_parts:
+        # No hidden parts, always include
+        return True
+    
+    if include_hidden:
+        # Include all hidden files/folders
+        return True
+    
+    # Check if any hidden part is in the whitelist
+    for hidden_part in hidden_parts:
+        if hidden_part in hidden_whitelist:
+            return True
+    
+    # Not in whitelist and includeHidden is False
+    return False
+
+
 # Regex for extracting inline #tags from markdown body (excludes code blocks)
 _INLINE_TAG_RE = re.compile(r'(?:^|\s)#([a-zA-Z][a-zA-Z0-9_/-]{1,50})', re.MULTILINE)
 # Regex patterns for stripping code blocks before inline tag extraction
-_CODE_BLOCK_RE = re.compile(r'```.*?```', re.DOTALL)
+_CODE_BLOCK_RE = re.compile(r'```[\s\S]*?```', re.MULTILINE)
 _INLINE_CODE_RE = re.compile(r'`[^`]+`')
 
 
 def _extract_tags(post: frontmatter.Post) -> List[str]:
     """Extract tags from frontmatter metadata.
+    
 
     Handles tags as comma-separated string, list, or other types.
     Strips leading ``#`` from each tag.
@@ -196,7 +245,7 @@ def parse_markdown_file(raw: str) -> frontmatter.Post:
         return frontmatter.Post(content, **{})
 
 
-def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
+def _scan_vault(vault_name: str, vault_path: str, vault_cfg: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
     """Synchronously scan a single vault directory and build file index.
 
     Walks the vault tree, reads supported files, extracts metadata
@@ -206,6 +255,7 @@ def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
     Args:
         vault_name: Display name of the vault.
         vault_path: Absolute filesystem path to the vault root.
+        vault_cfg: Optional vault configuration dict with hidden files settings.
 
     Returns:
         Dict with keys ``files`` (list), ``tags`` (counter dict), ``path`` (str), ``paths`` (list).
@@ -214,15 +264,19 @@ def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
     files: List[Dict[str, Any]] = []
     tag_counts: Dict[str, int] = {}
     paths: List[Dict[str, str]] = []
+    
+    # Default config if not provided
+    if vault_cfg is None:
+        vault_cfg = {"includeHidden": False, "hiddenWhitelist": []}
 
     if not vault_root.exists():
         logger.warning(f"Vault path does not exist: {vault_path}")
         return {"files": [], "tags": {}, "path": vault_path, "paths": []}
 
     for fpath in vault_root.rglob("*"):
-        # Skip hidden files and directories
+        # Check if path should be included based on hidden files configuration
         rel_parts = fpath.relative_to(vault_root).parts
-        if any(part.startswith(".") for part in rel_parts):
+        if not _should_include_path(rel_parts, vault_cfg):
             continue
         
         rel_path_str = str(fpath.relative_to(vault_root)).replace("\\", "/")
@@ -327,7 +381,7 @@ async def build_index(progress_callback=None) -> None:
 
     async def _process_vault(name: str, config: Dict[str, Any]):
         vault_path = config["path"]
-        vault_data = await loop.run_in_executor(None, _scan_vault, name, vault_path)
+        vault_data = await loop.run_in_executor(None, _scan_vault, name, vault_path, config)
         vault_data["config"] = config
 
         # Build lookup entries for the new vault
@@ -698,10 +752,12 @@ async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]
         "path": vault_path,
         "attachmentsPath": None,
         "scanAttachmentsOnStartup": True,
+        "includeHidden": False,
+        "hiddenWhitelist": [],
     }
 
     loop = asyncio.get_event_loop()
-    vault_data = await loop.run_in_executor(None, _scan_vault, vault_name, vault_path)
+    vault_data = await loop.run_in_executor(None, _scan_vault, vault_name, vault_path, vault_config[vault_name])
     vault_data["config"] = vault_config[vault_name]
 
     # Build lookup entries for the new vault