feat: Implement backend file indexing for vaults and initial frontend application.

backend/indexer.py

@@ -279,67 +279,84 @@ def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
     return {"files": files, "tags": tag_counts, "path": vault_path, "paths": paths, "config": {}}
 
 
-async def build_index() -> None:
+async def build_index(progress_callback=None) -> None:
     """Build the full in-memory index for all configured vaults.
 
-    Runs vault scans concurrently in a thread pool, then performs
-    an atomic swap of the global index and lookup table under a lock
-    to ensure thread-safe reads during reload.
+    Runs vault scans concurrently, inserting them incrementally into the global index.
+    Notifies progress via the provided callback.
     """
     global index, vault_config
     vault_config = load_vault_config()
-
-    if not vault_config:
-        logger.warning("No vaults configured. Set VAULT_N_NAME / VAULT_N_PATH env vars.")
-        return
-
-    loop = asyncio.get_event_loop()
-    new_index: Dict[str, Dict[str, Any]] = {}
-
-    tasks = []
-    for name, config in vault_config.items():
-        vault_path = config["path"]
-        tasks.append((name, loop.run_in_executor(None, _scan_vault, name, vault_path)))
-
-    for name, task in tasks:
-        vault_data = await task
-        # Store vault config in the index
-        vault_data["config"] = vault_config[name]
-        new_index[name] = vault_data
-
-    # Build O(1) lookup table for wikilink resolution
-    new_lookup: Dict[str, List[Dict[str, str]]] = {}
-    for vname, vdata in new_index.items():
-        for f in vdata["files"]:
-            entry = {"vault": vname, "path": f["path"]}
-            fname = f["path"].rsplit("/", 1)[-1].lower()
-            fpath_lower = f["path"].lower()
-            for key in (fname, fpath_lower):
-                if key not in new_lookup:
-                    new_lookup[key] = []
-                new_lookup[key].append(entry)
-
-    # Build path index for tree filtering
-    new_path_index: Dict[str, List[Dict[str, str]]] = {}
-    for vname, vdata in new_index.items():
-        new_path_index[vname] = vdata.get("paths", [])
     
-    # Atomic swap under lock for thread safety during concurrent reads
     global _index_generation
     with _index_lock:
         index.clear()
-        index.update(new_index)
         _file_lookup.clear()
-        _file_lookup.update(new_lookup)
         path_index.clear()
-        path_index.update(new_path_index)
         _index_generation += 1
 
+    if not vault_config:
+        logger.warning("No vaults configured. Set VAULT_N_NAME / VAULT_N_PATH env vars.")
+        if progress_callback:
+            await progress_callback("complete", {"total": 0})
+        return
+
+    if progress_callback:
+        await progress_callback("start", {"total_vaults": len(vault_config)})
+
+    loop = asyncio.get_event_loop()
+
+    async def _process_vault(name: str, config: Dict[str, Any]):
+        vault_path = config["path"]
+        vault_data = await loop.run_in_executor(None, _scan_vault, name, vault_path)
+        vault_data["config"] = config
+
+        # Build lookup entries for the new vault
+        new_lookup_entries: Dict[str, List[Dict[str, str]]] = {}
+        for f in vault_data["files"]:
+            entry = {"vault": name, "path": f["path"]}
+            fname = f["path"].rsplit("/", 1)[-1].lower()
+            fpath_lower = f["path"].lower()
+            for key in (fname, fpath_lower):
+                if key not in new_lookup_entries:
+                    new_lookup_entries[key] = []
+                new_lookup_entries[key].append(entry)
+
+        async_lock = _get_async_lock()
+        async with async_lock:
+            with _index_lock:
+                index[name] = vault_data
+                for key, entries in new_lookup_entries.items():
+                    if key not in _file_lookup:
+                        _file_lookup[key] = []
+                    _file_lookup[key].extend(entries)
+                path_index[name] = vault_data.get("paths", [])
+                global _index_generation
+                _index_generation += 1
+
+        if progress_callback:
+            await progress_callback("progress", {
+                "vault": name,
+                "files": len(vault_data["files"]),
+                "tags": len(vault_data["tags"])
+            })
+
+    # Run vault scans concurrently
+    tasks = []
+    for name, config in vault_config.items():
+        tasks.append(_process_vault(name, config))
+    
+    if tasks:
+        await asyncio.gather(*tasks)
+
+    # Build attachment index
+    await build_attachment_index(vault_config)
+
     total_files = sum(len(v["files"]) for v in index.values())
     logger.info(f"Index built: {len(index)} vaults, {total_files} total files")
     
-    # Build attachment index
-    await build_attachment_index(vault_config)
+    if progress_callback:
+        await progress_callback("complete", {"total_vaults": len(vault_config), "total_files": total_files})
 
 
 async def reload_index() -> Dict[str, Any]: