Projets/ObsiGate
1
1
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 16 Wiki Activity

Expand file type support beyond Markdown to include code, config, and build files with syntax highlighting and improved UI filtering

Browse Source
This commit is contained in:
bruno
2026-03-21 11:16:46 -04:00
parent 8a22c1db28
commit 2ed5f65a7a
5 changed files with 738 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/indexer.py
+38 -9
View File
@@ -15,6 +15,18 @@ index: Dict[str, Dict[str, Any]] = {}
# Vault config: {name: path}
vault_config: Dict[str, str] = {}
# Supported text-based file extensions
SUPPORTED_EXTENSIONS = {
".md", ".txt", ".log", ".py", ".js", ".ts", ".jsx", ".tsx",
".sh", ".bash", ".zsh", ".fish", ".bat", ".cmd", ".ps1",
".json", ".yaml", ".yml", ".toml", ".xml", ".csv",
".cfg", ".ini", ".conf", ".env",
".html", ".css", ".scss", ".less",
".java", ".c", ".cpp", ".h", ".hpp", ".cs", ".go", ".rs", ".rb",
".php", ".sql", ".r", ".m", ".swift", ".kt",
".dockerfile", ".makefile", ".cmake",
}
def load_vault_config() -> Dict[str, str]:
"""Read VAULT_N_NAME / VAULT_N_PATH env vars and return {name: path}."""
@@ -60,18 +72,34 @@ def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
logger.warning(f"Vault path does not exist: {vault_path}")
return {"files": [], "tags": {}, "path": vault_path}
for md_file in vault_root.rglob("*.md"):
for fpath in vault_root.rglob("*"):
if not fpath.is_file():
continue
# Skip hidden files and files inside hidden directories
rel_parts = fpath.relative_to(vault_root).parts
if any(part.startswith(".") for part in rel_parts):
continue
ext = fpath.suffix.lower()
# Also match extensionless files named like Dockerfile, Makefile
basename_lower = fpath.name.lower()
if ext not in SUPPORTED_EXTENSIONS and basename_lower not in ("dockerfile", "makefile", "cmakelists.txt"):
continue
try:
relative = md_file.relative_to(vault_root)
stat = md_file.stat()
relative = fpath.relative_to(vault_root)
stat = fpath.stat()
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
raw = md_file.read_text(encoding="utf-8", errors="replace")
post = frontmatter.loads(raw)
raw = fpath.read_text(encoding="utf-8", errors="replace")
tags = _extract_tags(post)
title = _extract_title(post, md_file)
content_preview = post.content[:200].strip()
tags: List[str] = []
title = fpath.stem.replace("-", " ").replace("_", " ")
content_preview = raw[:200].strip()
if ext == ".md":
post = frontmatter.loads(raw)
tags = _extract_tags(post)
title = _extract_title(post, fpath)
content_preview = post.content[:200].strip()
files.append({
"path": str(relative).replace("\\", "/"),
@@ -80,13 +108,14 @@ def _scan_vault(vault_name: str, vault_path: str) -> Dict[str, Any]:
"content_preview": content_preview,
"size": stat.st_size,
"modified": modified,
"extension": ext,
})
for tag in tags:
tag_counts[tag] = tag_counts.get(tag, 0) + 1
except Exception as e:
logger.error(f"Error indexing {md_file}: {e}")
logger.error(f"Error indexing {fpath}: {e}")
continue
logger.info(f"Vault '{vault_name}': indexed {len(files)} files, {len(tag_counts)} unique tags")
backend/main.py
+112 -29
View File
@@ -1,4 +1,5 @@
import re
import html as html_mod
import logging
from pathlib import Path
from typing import Optional
@@ -7,7 +8,7 @@ import frontmatter
import mistune
from fastapi import FastAPI, HTTPException, Query
from fastapi.staticfiles import StaticFiles
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.responses import HTMLResponse, JSONResponse, FileResponse, PlainTextResponse
from backend.indexer import (
build_index,
@@ -15,6 +16,7 @@ from backend.indexer import (
index,
get_vault_data,
find_file_in_index,
SUPPORTED_EXTENSIONS,
)
from backend.search import search, get_all_tags
@@ -111,20 +113,26 @@ async def api_browse(vault_name: str, path: str = ""):
continue
rel = str(entry.relative_to(vault_root)).replace("\\", "/")
if entry.is_dir():
# Count .md files recursively
md_count = sum(1 for _ in entry.rglob("*.md"))
# Count all supported files recursively (skip hidden dirs)
file_count = sum(
1 for f in entry.rglob("*")
if f.is_file()
and not any(p.startswith(".") for p in f.relative_to(entry).parts)
and (f.suffix.lower() in SUPPORTED_EXTENSIONS or f.name.lower() in ("dockerfile", "makefile"))
)
items.append({
"name": entry.name,
"path": rel,
"type": "directory",
"children_count": md_count,
"children_count": file_count,
})
elif entry.suffix.lower() == ".md":
elif entry.suffix.lower() in SUPPORTED_EXTENSIONS or entry.name.lower() in ("dockerfile", "makefile"):
items.append({
"name": entry.name,
"path": rel,
"type": "file",
"size": entry.stat().st_size,
"extension": entry.suffix.lower(),
})
except PermissionError:
raise HTTPException(status_code=403, detail="Permission denied")
@@ -132,9 +140,26 @@ async def api_browse(vault_name: str, path: str = ""):
return {"vault": vault_name, "path": path, "items": items}
@app.get("/api/file/{vault_name}")
async def api_file(vault_name: str, path: str = Query(..., description="Relative path to .md file")):
"""Return rendered HTML + metadata for a markdown file."""
# Map file extensions to highlight.js language hints
EXT_TO_LANG = {
".py": "python", ".js": "javascript", ".ts": "typescript",
".jsx": "jsx", ".tsx": "tsx", ".sh": "bash", ".bash": "bash",
".zsh": "bash", ".fish": "fish", ".bat": "batch", ".cmd": "batch",
".ps1": "powershell", ".json": "json", ".yaml": "yaml", ".yml": "yaml",
".toml": "toml", ".xml": "xml", ".csv": "plaintext",
".cfg": "ini", ".ini": "ini", ".conf": "ini", ".env": "bash",
".html": "html", ".css": "css", ".scss": "scss", ".less": "less",
".java": "java", ".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp",
".cs": "csharp", ".go": "go", ".rs": "rust", ".rb": "ruby",
".php": "php", ".sql": "sql", ".r": "r", ".swift": "swift",
".kt": "kotlin", ".txt": "plaintext", ".log": "plaintext",
".dockerfile": "dockerfile", ".makefile": "makefile", ".cmake": "cmake",
}
@app.get("/api/file/{vault_name}/raw")
async def api_file_raw(vault_name: str, path: str = Query(..., description="Relative path to file")):
"""Return raw file content."""
vault_data = get_vault_data(vault_name)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
@@ -146,30 +171,88 @@ async def api_file(vault_name: str, path: str = Query(..., description="Relative
raise HTTPException(status_code=404, detail=f"File not found: {path}")
raw = file_path.read_text(encoding="utf-8", errors="replace")
post = frontmatter.loads(raw)
return {"vault": vault_name, "path": path, "raw": raw}
# Extract metadata
tags = post.metadata.get("tags", [])
if isinstance(tags, str):
tags = [t.strip().lstrip("#") for t in tags.split(",") if t.strip()]
elif isinstance(tags, list):
tags = [str(t).strip().lstrip("#") for t in tags]
@app.get("/api/file/{vault_name}/download")
async def api_file_download(vault_name: str, path: str = Query(..., description="Relative path to file")):
"""Download a file as attachment."""
vault_data = get_vault_data(vault_name)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
vault_root = Path(vault_data["path"])
file_path = vault_root / path
if not file_path.exists() or not file_path.is_file():
raise HTTPException(status_code=404, detail=f"File not found: {path}")
return FileResponse(
path=str(file_path),
filename=file_path.name,
media_type="application/octet-stream",
)
@app.get("/api/file/{vault_name}")
async def api_file(vault_name: str, path: str = Query(..., description="Relative path to file")):
"""Return rendered HTML + metadata for a file."""
vault_data = get_vault_data(vault_name)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
vault_root = Path(vault_data["path"])
file_path = vault_root / path
if not file_path.exists() or not file_path.is_file():
raise HTTPException(status_code=404, detail=f"File not found: {path}")
raw = file_path.read_text(encoding="utf-8", errors="replace")
ext = file_path.suffix.lower()
if ext == ".md":
post = frontmatter.loads(raw)
# Extract metadata
tags = post.metadata.get("tags", [])
if isinstance(tags, str):
tags = [t.strip().lstrip("#") for t in tags.split(",") if t.strip()]
elif isinstance(tags, list):
tags = [str(t).strip().lstrip("#") for t in tags]
else:
tags = []
title = post.metadata.get("title", file_path.stem.replace("-", " ").replace("_", " "))
html_content = _render_markdown(post.content, vault_name)
return {
"vault": vault_name,
"path": path,
"title": str(title),
"tags": tags,
"frontmatter": dict(post.metadata) if post.metadata else {},
"html": html_content,
"raw_length": len(raw),
"extension": ext,
"is_markdown": True,
}
else:
tags = []
# Non-markdown: wrap in syntax-highlighted code block
lang = EXT_TO_LANG.get(ext, "plaintext")
escaped = html_mod.escape(raw)
html_content = f'<pre><code class="language-{lang}">{escaped}</code></pre>'
title = post.metadata.get("title", file_path.stem.replace("-", " ").replace("_", " "))
html_content = _render_markdown(post.content, vault_name)
return {
"vault": vault_name,
"path": path,
"title": str(title),
"tags": tags,
"frontmatter": dict(post.metadata) if post.metadata else {},
"html": html_content,
"raw_length": len(raw),
}
return {
"vault": vault_name,
"path": path,
"title": file_path.name,
"tags": [],
"frontmatter": {},
"html": html_content,
"raw_length": len(raw),
"extension": ext,
"is_markdown": False,
}
@app.get("/api/search")