first commit
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
import frontmatter
|
||||
|
||||
logger = logging.getLogger("obsigate.indexer")
|
||||
|
||||
# Global in-memory index
|
||||
index: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
# Vault config: {name: path}
|
||||
vault_config: Dict[str, str] = {}
|
||||
|
||||
|
||||
def load_vault_config() -> Dict[str, str]:
|
||||
"""Read VAULT_N_NAME / VAULT_N_PATH env vars and return {name: path}."""
|
||||
vaults: Dict[str, str] = {}
|
||||
n = 1
|
||||
while True:
|
||||
name = os.environ.get(f"VAULT_{n}_NAME")
|
||||
path = os.environ.get(f"VAULT_{n}_PATH")
|
||||
if not name or not path:
|
||||
break
|
||||
vaults[name] = path
|
||||
n += 1
|
||||
return vaults
|
||||
|
||||
|
||||
def _extract_tags(post: frontmatter.Post) -> List[str]:
|
||||
"""Extract tags from frontmatter metadata."""
|
||||
tags = post.metadata.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [t.strip().lstrip("#") for t in tags.split(",") if t.strip()]
|
||||
elif isinstance(tags, list):
|
||||
tags = [str(t).strip().lstrip("#") for t in tags]
|
||||
else:
|
||||
tags = []
|
||||
return tags
|
||||
|
||||
|
||||
def _extract_title(post: frontmatter.Post, filepath: Path) -> str:
|
||||
"""Extract title from frontmatter or derive from filename."""
|
||||
title = post.metadata.get("title", "")
|
||||
if not title:
|
||||
title = filepath.stem.replace("-", " ").replace("_", " ")
|
||||
return str(title)
|
||||
|
||||
|
||||
def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
|
||||
"""Synchronously scan a single vault directory."""
|
||||
vault_root = Path(vault_path)
|
||||
files: List[Dict[str, Any]] = []
|
||||
tag_counts: Dict[str, int] = {}
|
||||
|
||||
if not vault_root.exists():
|
||||
logger.warning(f"Vault path does not exist: {vault_path}")
|
||||
return {"files": [], "tags": {}, "path": vault_path}
|
||||
|
||||
for md_file in vault_root.rglob("*.md"):
|
||||
try:
|
||||
relative = md_file.relative_to(vault_root)
|
||||
stat = md_file.stat()
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
|
||||
|
||||
raw = md_file.read_text(encoding="utf-8", errors="replace")
|
||||
post = frontmatter.loads(raw)
|
||||
|
||||
tags = _extract_tags(post)
|
||||
title = _extract_title(post, md_file)
|
||||
content_preview = post.content[:200].strip()
|
||||
|
||||
files.append({
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"content_preview": content_preview,
|
||||
"size": stat.st_size,
|
||||
"modified": modified,
|
||||
})
|
||||
|
||||
for tag in tags:
|
||||
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing {md_file}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Vault '{vault_name}': indexed {len(files)} files, {len(tag_counts)} unique tags")
|
||||
return {"files": files, "tags": tag_counts, "path": vault_path}
|
||||
|
||||
|
||||
async def build_index() -> None:
|
||||
"""Build the full in-memory index for all configured vaults."""
|
||||
global index, vault_config
|
||||
vault_config = load_vault_config()
|
||||
|
||||
if not vault_config:
|
||||
logger.warning("No vaults configured. Set VAULT_N_NAME / VAULT_N_PATH env vars.")
|
||||
return
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
new_index: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
tasks = []
|
||||
for name, path in vault_config.items():
|
||||
tasks.append((name, loop.run_in_executor(None, _scan_vault, name, path)))
|
||||
|
||||
for name, task in tasks:
|
||||
new_index[name] = await task
|
||||
|
||||
index.clear()
|
||||
index.update(new_index)
|
||||
total_files = sum(len(v["files"]) for v in index.values())
|
||||
logger.info(f"Index built: {len(index)} vaults, {total_files} total files")
|
||||
|
||||
|
||||
async def reload_index() -> Dict[str, Any]:
|
||||
"""Force a full re-index and return stats."""
|
||||
await build_index()
|
||||
stats = {}
|
||||
for name, data in index.items():
|
||||
stats[name] = {"file_count": len(data["files"]), "tag_count": len(data["tags"])}
|
||||
return stats
|
||||
|
||||
|
||||
def get_vault_names() -> List[str]:
|
||||
return list(index.keys())
|
||||
|
||||
|
||||
def get_vault_data(vault_name: str) -> Optional[Dict[str, Any]]:
|
||||
return index.get(vault_name)
|
||||
|
||||
|
||||
def find_file_in_index(link_target: str, current_vault: str) -> Optional[Dict[str, str]]:
|
||||
"""Find a file matching a wikilink target. Search current vault first, then all."""
|
||||
target_lower = link_target.lower().strip()
|
||||
if not target_lower.endswith(".md"):
|
||||
target_lower += ".md"
|
||||
|
||||
def _search_vault(vname: str, vdata: Dict[str, Any]):
|
||||
for f in vdata["files"]:
|
||||
fpath = f["path"].lower()
|
||||
fname = fpath.rsplit("/", 1)[-1]
|
||||
if fname == target_lower or fpath == target_lower:
|
||||
return {"vault": vname, "path": f["path"]}
|
||||
return None
|
||||
|
||||
# Search current vault first
|
||||
if current_vault in index:
|
||||
result = _search_vault(current_vault, index[current_vault])
|
||||
if result:
|
||||
return result
|
||||
|
||||
# Search all other vaults
|
||||
for vname, vdata in index.items():
|
||||
if vname == current_vault:
|
||||
continue
|
||||
result = _search_vault(vname, vdata)
|
||||
if result:
|
||||
return result
|
||||
|
||||
return None
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
import re
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import frontmatter
|
||||
import mistune
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from backend.indexer import (
|
||||
build_index,
|
||||
reload_index,
|
||||
index,
|
||||
get_vault_data,
|
||||
find_file_in_index,
|
||||
)
|
||||
from backend.search import search, get_all_tags
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
app = FastAPI(title="ObsiGate", version="1.0.0")
|
||||
|
||||
# Resolve frontend path relative to this file
|
||||
FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Startup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
logger.info("ObsiGate starting — building index...")
|
||||
await build_index()
|
||||
logger.info("ObsiGate ready.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown rendering helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _convert_wikilinks(content: str, current_vault: str) -> str:
|
||||
"""Convert [[wikilinks]] and [[target|display]] to HTML links."""
|
||||
def _replace(match):
|
||||
target = match.group(1).strip()
|
||||
display = match.group(2).strip() if match.group(2) else target
|
||||
found = find_file_in_index(target, current_vault)
|
||||
if found:
|
||||
return (
|
||||
f'<a class="wikilink" href="#" '
|
||||
f'data-vault="{found["vault"]}" '
|
||||
f'data-path="{found["path"]}">{display}</a>'
|
||||
)
|
||||
return f'<span class="wikilink-missing">{display}</span>'
|
||||
|
||||
pattern = r'\[\[([^\]|]+)(?:\|([^\]]+))?\]\]'
|
||||
return re.sub(pattern, _replace, content)
|
||||
|
||||
|
||||
def _render_markdown(raw_md: str, vault_name: str) -> str:
|
||||
"""Render markdown string to HTML with wikilink support."""
|
||||
converted = _convert_wikilinks(raw_md, vault_name)
|
||||
md = mistune.create_markdown(
|
||||
escape=False,
|
||||
plugins=["table", "strikethrough", "footnotes", "task_lists"],
|
||||
)
|
||||
return md(converted)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/vaults")
|
||||
async def api_vaults():
|
||||
"""List configured vaults with file counts."""
|
||||
result = []
|
||||
for name, data in index.items():
|
||||
result.append({
|
||||
"name": name,
|
||||
"file_count": len(data["files"]),
|
||||
"tag_count": len(data["tags"]),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/browse/{vault_name}")
|
||||
async def api_browse(vault_name: str, path: str = ""):
|
||||
"""Browse directories and files in a vault at a given path level."""
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
target = vault_root / path if path else vault_root
|
||||
|
||||
if not target.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Path not found: {path}")
|
||||
|
||||
items = []
|
||||
try:
|
||||
for entry in sorted(target.iterdir(), key=lambda e: (not e.is_dir(), e.name.lower())):
|
||||
# Skip hidden files/dirs
|
||||
if entry.name.startswith("."):
|
||||
continue
|
||||
rel = str(entry.relative_to(vault_root)).replace("\\", "/")
|
||||
if entry.is_dir():
|
||||
# Count .md files recursively
|
||||
md_count = sum(1 for _ in entry.rglob("*.md"))
|
||||
items.append({
|
||||
"name": entry.name,
|
||||
"path": rel,
|
||||
"type": "directory",
|
||||
"children_count": md_count,
|
||||
})
|
||||
elif entry.suffix.lower() == ".md":
|
||||
items.append({
|
||||
"name": entry.name,
|
||||
"path": rel,
|
||||
"type": "file",
|
||||
"size": entry.stat().st_size,
|
||||
})
|
||||
except PermissionError:
|
||||
raise HTTPException(status_code=403, detail="Permission denied")
|
||||
|
||||
return {"vault": vault_name, "path": path, "items": items}
|
||||
|
||||
|
||||
@app.get("/api/file/{vault_name}")
|
||||
async def api_file(vault_name: str, path: str = Query(..., description="Relative path to .md file")):
|
||||
"""Return rendered HTML + metadata for a markdown file."""
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = vault_root / path
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
post = frontmatter.loads(raw)
|
||||
|
||||
# Extract metadata
|
||||
tags = post.metadata.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [t.strip().lstrip("#") for t in tags.split(",") if t.strip()]
|
||||
elif isinstance(tags, list):
|
||||
tags = [str(t).strip().lstrip("#") for t in tags]
|
||||
else:
|
||||
tags = []
|
||||
|
||||
title = post.metadata.get("title", file_path.stem.replace("-", " ").replace("_", " "))
|
||||
|
||||
html_content = _render_markdown(post.content, vault_name)
|
||||
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": str(title),
|
||||
"tags": tags,
|
||||
"frontmatter": dict(post.metadata) if post.metadata else {},
|
||||
"html": html_content,
|
||||
"raw_length": len(raw),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/search")
|
||||
async def api_search(
|
||||
q: str = Query("", description="Search query"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
tag: Optional[str] = Query(None, description="Tag filter"),
|
||||
):
|
||||
"""Full-text search across vaults."""
|
||||
results = search(q, vault_filter=vault, tag_filter=tag)
|
||||
return {"query": q, "vault_filter": vault, "tag_filter": tag, "count": len(results), "results": results}
|
||||
|
||||
|
||||
@app.get("/api/tags")
|
||||
async def api_tags(vault: Optional[str] = Query(None, description="Vault filter")):
|
||||
"""Return all unique tags with counts."""
|
||||
tags = get_all_tags(vault_filter=vault)
|
||||
return {"vault_filter": vault, "tags": tags}
|
||||
|
||||
|
||||
@app.get("/api/index/reload")
|
||||
async def api_reload():
|
||||
"""Force a re-index of all vaults."""
|
||||
stats = await reload_index()
|
||||
return {"status": "ok", "vaults": stats}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files & SPA fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if FRONTEND_DIR.exists():
|
||||
app.mount("/static", StaticFiles(directory=str(FRONTEND_DIR)), name="static")
|
||||
|
||||
@app.get("/{full_path:path}")
|
||||
async def serve_spa(full_path: str):
|
||||
"""Serve the SPA index.html for all non-API routes."""
|
||||
index_file = FRONTEND_DIR / "index.html"
|
||||
if index_file.exists():
|
||||
return HTMLResponse(content=index_file.read_text(encoding="utf-8"))
|
||||
raise HTTPException(status_code=404, detail="Frontend not found")
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi==0.111.0
|
||||
uvicorn[standard]==0.30.0
|
||||
python-frontmatter==1.1.0
|
||||
mistune==3.0.2
|
||||
python-multipart==0.0.9
|
||||
aiofiles==23.2.1
|
||||
@@ -0,0 +1,109 @@
|
||||
import re
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from backend.indexer import index, get_vault_data
|
||||
|
||||
logger = logging.getLogger("obsigate.search")
|
||||
|
||||
|
||||
def _read_file_content(vault_name: str, file_path: str) -> str:
|
||||
"""Read raw markdown content of a file from disk."""
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
return ""
|
||||
vault_root = Path(vault_data["path"])
|
||||
full_path = vault_root / file_path
|
||||
try:
|
||||
return full_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_snippet(content: str, query: str, context_chars: int = 120) -> str:
|
||||
"""Extract a text snippet around the first occurrence of query."""
|
||||
lower_content = content.lower()
|
||||
lower_query = query.lower()
|
||||
pos = lower_content.find(lower_query)
|
||||
if pos == -1:
|
||||
return content[:200].strip()
|
||||
|
||||
start = max(0, pos - context_chars)
|
||||
end = min(len(content), pos + len(query) + context_chars)
|
||||
snippet = content[start:end].strip()
|
||||
|
||||
if start > 0:
|
||||
snippet = "..." + snippet
|
||||
if end < len(content):
|
||||
snippet = snippet + "..."
|
||||
|
||||
return snippet
|
||||
|
||||
|
||||
def search(
|
||||
query: str,
|
||||
vault_filter: str = "all",
|
||||
tag_filter: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Full-text search across indexed vaults.
|
||||
Returns scored results with snippets.
|
||||
"""
|
||||
query = query.strip() if query else ""
|
||||
has_query = len(query) > 0
|
||||
|
||||
if not has_query and not tag_filter:
|
||||
return []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
|
||||
for vault_name, vault_data in index.items():
|
||||
if vault_filter != "all" and vault_name != vault_filter:
|
||||
continue
|
||||
|
||||
for file_info in vault_data["files"]:
|
||||
if tag_filter and tag_filter not in file_info["tags"]:
|
||||
continue
|
||||
|
||||
score = 0
|
||||
snippet = file_info.get("content_preview", "")
|
||||
|
||||
if has_query:
|
||||
# Title match (high weight)
|
||||
if query.lower() in file_info["title"].lower():
|
||||
score += 10
|
||||
|
||||
# Content match
|
||||
content = _read_file_content(vault_name, file_info["path"])
|
||||
if query.lower() in content.lower():
|
||||
score += 1
|
||||
snippet = _extract_snippet(content, query)
|
||||
else:
|
||||
# Tag-only filter: all matching files get score 1
|
||||
score = 1
|
||||
|
||||
if score > 0:
|
||||
results.append({
|
||||
"vault": vault_name,
|
||||
"path": file_info["path"],
|
||||
"title": file_info["title"],
|
||||
"tags": file_info["tags"],
|
||||
"score": score,
|
||||
"snippet": snippet,
|
||||
"modified": file_info["modified"],
|
||||
})
|
||||
|
||||
results.sort(key=lambda x: -x["score"])
|
||||
return results
|
||||
|
||||
|
||||
def get_all_tags(vault_filter: Optional[str] = None) -> Dict[str, int]:
|
||||
"""Aggregate tag counts across vaults."""
|
||||
merged: Dict[str, int] = {}
|
||||
for vault_name, vault_data in index.items():
|
||||
if vault_filter and vault_name != vault_filter:
|
||||
continue
|
||||
for tag, count in vault_data.get("tags", {}).items():
|
||||
merged[tag] = merged.get(tag, 0) + count
|
||||
return dict(sorted(merged.items(), key=lambda x: -x[1]))
|
||||
Reference in New Issue
Block a user