Add real-time file synchronization with watchdog, SSE notifications, and dynamic vault management API
This commit is contained in:
@@ -22,6 +22,9 @@ vault_config: Dict[str, Dict[str, Any]] = {}
|
||||
# Thread-safe lock for index updates
|
||||
_index_lock = threading.Lock()
|
||||
|
||||
# Async lock for partial index updates (coexists with threading lock)
|
||||
_async_index_lock: asyncio.Lock = None # initialized lazily
|
||||
|
||||
# Generation counter — incremented on each index rebuild so consumers
|
||||
# (e.g. the inverted index in search.py) can detect staleness.
|
||||
_index_generation: int = 0
|
||||
@@ -362,6 +365,335 @@ def get_vault_data(vault_name: str) -> Optional[Dict[str, Any]]:
|
||||
return index.get(vault_name)
|
||||
|
||||
|
||||
def _get_async_lock() -> asyncio.Lock:
|
||||
"""Get or create the async lock (must be called from an event loop)."""
|
||||
global _async_index_lock
|
||||
if _async_index_lock is None:
|
||||
_async_index_lock = asyncio.Lock()
|
||||
return _async_index_lock
|
||||
|
||||
|
||||
def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Synchronously read and parse a single file for indexing.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
vault_path: Absolute path to vault root.
|
||||
file_path: Absolute path to the file.
|
||||
|
||||
Returns:
|
||||
File info dict or None if the file cannot be read.
|
||||
"""
|
||||
try:
|
||||
fpath = Path(file_path)
|
||||
vault_root = Path(vault_path)
|
||||
|
||||
if not fpath.exists() or not fpath.is_file():
|
||||
return None
|
||||
|
||||
relative = fpath.relative_to(vault_root)
|
||||
rel_parts = relative.parts
|
||||
if any(part.startswith(".") for part in rel_parts):
|
||||
return None
|
||||
|
||||
ext = fpath.suffix.lower()
|
||||
basename_lower = fpath.name.lower()
|
||||
if ext not in SUPPORTED_EXTENSIONS and basename_lower not in ("dockerfile", "makefile", "cmakelists.txt"):
|
||||
return None
|
||||
|
||||
stat = fpath.stat()
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
tags: List[str] = []
|
||||
title = fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
tags = _extract_tags(post)
|
||||
inline_tags = _extract_inline_tags(post.content)
|
||||
tags = list(set(tags) | set(inline_tags))
|
||||
title = _extract_title(post, fpath)
|
||||
content_preview = post.content[:200].strip()
|
||||
|
||||
return {
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"content_preview": content_preview,
|
||||
"content": raw[:SEARCH_CONTENT_LIMIT],
|
||||
"size": stat.st_size,
|
||||
"modified": modified,
|
||||
"extension": ext,
|
||||
}
|
||||
except PermissionError:
|
||||
logger.debug(f"Permission denied: {file_path}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing file {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _remove_file_from_structures(vault_name: str, rel_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Remove a file from all index structures. Returns removed file info or None.
|
||||
|
||||
Must be called under _index_lock or _async_index_lock.
|
||||
"""
|
||||
global _index_generation
|
||||
vault_data = index.get(vault_name)
|
||||
if not vault_data:
|
||||
return None
|
||||
|
||||
# Remove from files list
|
||||
removed = None
|
||||
files = vault_data["files"]
|
||||
for i, f in enumerate(files):
|
||||
if f["path"] == rel_path:
|
||||
removed = files.pop(i)
|
||||
break
|
||||
|
||||
if not removed:
|
||||
return None
|
||||
|
||||
# Update tag counts
|
||||
for tag in removed.get("tags", []):
|
||||
tc = vault_data["tags"]
|
||||
if tag in tc:
|
||||
tc[tag] -= 1
|
||||
if tc[tag] <= 0:
|
||||
del tc[tag]
|
||||
|
||||
# Remove from _file_lookup
|
||||
fname_lower = rel_path.rsplit("/", 1)[-1].lower()
|
||||
fpath_lower = rel_path.lower()
|
||||
for key in (fname_lower, fpath_lower):
|
||||
entries = _file_lookup.get(key, [])
|
||||
_file_lookup[key] = [e for e in entries if not (e["vault"] == vault_name and e["path"] == rel_path)]
|
||||
if not _file_lookup[key]:
|
||||
del _file_lookup[key]
|
||||
|
||||
# Remove from path_index
|
||||
if vault_name in path_index:
|
||||
path_index[vault_name] = [p for p in path_index[vault_name] if p["path"] != rel_path]
|
||||
|
||||
_index_generation += 1
|
||||
return removed
|
||||
|
||||
|
||||
def _add_file_to_structures(vault_name: str, file_info: Dict[str, Any]):
|
||||
"""Add a file entry to all index structures.
|
||||
|
||||
Must be called under _index_lock or _async_index_lock.
|
||||
"""
|
||||
global _index_generation
|
||||
vault_data = index.get(vault_name)
|
||||
if not vault_data:
|
||||
return
|
||||
|
||||
vault_data["files"].append(file_info)
|
||||
|
||||
# Update tag counts
|
||||
for tag in file_info.get("tags", []):
|
||||
vault_data["tags"][tag] = vault_data["tags"].get(tag, 0) + 1
|
||||
|
||||
# Add to _file_lookup
|
||||
rel_path = file_info["path"]
|
||||
fname_lower = rel_path.rsplit("/", 1)[-1].lower()
|
||||
fpath_lower = rel_path.lower()
|
||||
entry = {"vault": vault_name, "path": rel_path}
|
||||
for key in (fname_lower, fpath_lower):
|
||||
if key not in _file_lookup:
|
||||
_file_lookup[key] = []
|
||||
_file_lookup[key].append(entry)
|
||||
|
||||
# Add to path_index
|
||||
if vault_name in path_index:
|
||||
# Check if already present (avoid duplicates)
|
||||
existing = {p["path"] for p in path_index[vault_name]}
|
||||
if rel_path not in existing:
|
||||
path_index[vault_name].append({
|
||||
"path": rel_path,
|
||||
"name": rel_path.rsplit("/", 1)[-1],
|
||||
"type": "file",
|
||||
})
|
||||
|
||||
_index_generation += 1
|
||||
|
||||
|
||||
async def update_single_file(vault_name: str, abs_file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Re-index a single file without full rebuild.
|
||||
|
||||
Reads the file, removes the old entry if present, inserts the new one.
|
||||
Thread-safe via async lock.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault containing the file.
|
||||
abs_file_path: Absolute filesystem path to the file.
|
||||
|
||||
Returns:
|
||||
The new file info dict, or None if file could not be indexed.
|
||||
"""
|
||||
vault_data = index.get(vault_name)
|
||||
if not vault_data:
|
||||
logger.warning(f"update_single_file: vault '{vault_name}' not in index")
|
||||
return None
|
||||
|
||||
vault_path = vault_data.get("path") or vault_config.get(vault_name, {}).get("path", "")
|
||||
if not vault_path:
|
||||
return None
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
file_info = await loop.run_in_executor(None, _index_single_file_sync, vault_name, vault_path, abs_file_path)
|
||||
|
||||
lock = _get_async_lock()
|
||||
async with lock:
|
||||
# Remove old entry if exists
|
||||
try:
|
||||
rel_path = str(Path(abs_file_path).relative_to(vault_path)).replace("\\", "/")
|
||||
except ValueError:
|
||||
logger.warning(f"File {abs_file_path} not under vault {vault_path}")
|
||||
return None
|
||||
|
||||
_remove_file_from_structures(vault_name, rel_path)
|
||||
|
||||
if file_info:
|
||||
_add_file_to_structures(vault_name, file_info)
|
||||
|
||||
if file_info:
|
||||
logger.debug(f"Updated: {vault_name}/{file_info['path']}")
|
||||
return file_info
|
||||
|
||||
|
||||
async def remove_single_file(vault_name: str, abs_file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Remove a single file from the index.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
abs_file_path: Absolute path to the deleted file.
|
||||
|
||||
Returns:
|
||||
The removed file info dict, or None if not found.
|
||||
"""
|
||||
vault_data = index.get(vault_name)
|
||||
if not vault_data:
|
||||
return None
|
||||
|
||||
vault_path = vault_data.get("path") or vault_config.get(vault_name, {}).get("path", "")
|
||||
if not vault_path:
|
||||
return None
|
||||
|
||||
try:
|
||||
rel_path = str(Path(abs_file_path).relative_to(vault_path)).replace("\\", "/")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
lock = _get_async_lock()
|
||||
async with lock:
|
||||
removed = _remove_file_from_structures(vault_name, rel_path)
|
||||
|
||||
if removed:
|
||||
logger.debug(f"Removed: {vault_name}/{rel_path}")
|
||||
return removed
|
||||
|
||||
|
||||
async def handle_file_move(vault_name: str, src_abs: str, dest_abs: str) -> Optional[Dict[str, Any]]:
|
||||
"""Handle a file move/rename by removing old entry and indexing new location.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
src_abs: Absolute path of the source (old location).
|
||||
dest_abs: Absolute path of the destination (new location).
|
||||
|
||||
Returns:
|
||||
The new file info dict, or None.
|
||||
"""
|
||||
await remove_single_file(vault_name, src_abs)
|
||||
return await update_single_file(vault_name, dest_abs)
|
||||
|
||||
|
||||
async def remove_vault_from_index(vault_name: str):
|
||||
"""Remove an entire vault from the index.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault to remove.
|
||||
"""
|
||||
global _index_generation
|
||||
lock = _get_async_lock()
|
||||
async with lock:
|
||||
vault_data = index.pop(vault_name, None)
|
||||
if not vault_data:
|
||||
return
|
||||
|
||||
# Clean _file_lookup
|
||||
for f in vault_data.get("files", []):
|
||||
rel_path = f["path"]
|
||||
fname_lower = rel_path.rsplit("/", 1)[-1].lower()
|
||||
fpath_lower = rel_path.lower()
|
||||
for key in (fname_lower, fpath_lower):
|
||||
entries = _file_lookup.get(key, [])
|
||||
_file_lookup[key] = [e for e in entries if e["vault"] != vault_name]
|
||||
if not _file_lookup[key]:
|
||||
_file_lookup.pop(key, None)
|
||||
|
||||
# Clean path_index
|
||||
path_index.pop(vault_name, None)
|
||||
|
||||
# Clean vault_config
|
||||
vault_config.pop(vault_name, None)
|
||||
|
||||
_index_generation += 1
|
||||
logger.info(f"Removed vault '{vault_name}' from index")
|
||||
|
||||
|
||||
async def add_vault_to_index(vault_name: str, vault_path: str) -> Dict[str, Any]:
|
||||
"""Add a new vault to the index dynamically.
|
||||
|
||||
Args:
|
||||
vault_name: Display name for the vault.
|
||||
vault_path: Absolute filesystem path to the vault.
|
||||
|
||||
Returns:
|
||||
Dict with vault stats (file_count, tag_count).
|
||||
"""
|
||||
global _index_generation
|
||||
|
||||
vault_config[vault_name] = {
|
||||
"path": vault_path,
|
||||
"attachmentsPath": None,
|
||||
"scanAttachmentsOnStartup": True,
|
||||
}
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
vault_data = await loop.run_in_executor(None, _scan_vault, vault_name, vault_path)
|
||||
vault_data["config"] = vault_config[vault_name]
|
||||
|
||||
# Build lookup entries for the new vault
|
||||
new_lookup_entries: Dict[str, List[Dict[str, str]]] = {}
|
||||
for f in vault_data["files"]:
|
||||
entry = {"vault": vault_name, "path": f["path"]}
|
||||
fname = f["path"].rsplit("/", 1)[-1].lower()
|
||||
fpath_lower = f["path"].lower()
|
||||
for key in (fname, fpath_lower):
|
||||
if key not in new_lookup_entries:
|
||||
new_lookup_entries[key] = []
|
||||
new_lookup_entries[key].append(entry)
|
||||
|
||||
lock = _get_async_lock()
|
||||
async with lock:
|
||||
index[vault_name] = vault_data
|
||||
for key, entries in new_lookup_entries.items():
|
||||
if key not in _file_lookup:
|
||||
_file_lookup[key] = []
|
||||
_file_lookup[key].extend(entries)
|
||||
path_index[vault_name] = vault_data.get("paths", [])
|
||||
_index_generation += 1
|
||||
|
||||
stats = {"file_count": len(vault_data["files"]), "tag_count": len(vault_data["tags"])}
|
||||
logger.info(f"Added vault '{vault_name}': {stats['file_count']} files, {stats['tag_count']} tags")
|
||||
return stats
|
||||
|
||||
|
||||
def find_file_in_index(link_target: str, current_vault: str) -> Optional[Dict[str, str]]:
|
||||
"""Find a file matching a wikilink target using O(1) lookup table.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user