Add advanced search engine with inverted index, thread pool execution, configuration API, and comprehensive diagnostics
This commit is contained in:
+178
-5
@@ -1,8 +1,12 @@
|
||||
import asyncio
|
||||
import json as _json
|
||||
import re
|
||||
import html as html_mod
|
||||
import logging
|
||||
import mimetypes
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
@@ -111,11 +115,14 @@ class SearchResultItem(BaseModel):
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
"""Full-text search response."""
|
||||
"""Full-text search response with optional pagination."""
|
||||
query: str
|
||||
vault_filter: str
|
||||
tag_filter: Optional[str]
|
||||
count: int
|
||||
total: int = Field(0, description="Total results before pagination")
|
||||
offset: int = Field(0, description="Current pagination offset")
|
||||
limit: int = Field(200, description="Page size")
|
||||
results: List[SearchResultItem]
|
||||
|
||||
|
||||
@@ -165,6 +172,7 @@ class AdvancedSearchResponse(BaseModel):
|
||||
offset: int
|
||||
limit: int
|
||||
facets: SearchFacets
|
||||
query_time_ms: float = Field(0, description="Server-side query time in milliseconds")
|
||||
|
||||
|
||||
class TitleSuggestion(BaseModel):
|
||||
@@ -210,16 +218,25 @@ class HealthResponse(BaseModel):
|
||||
# Application lifespan (replaces deprecated on_event)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Thread pool for offloading CPU-bound search from the event loop.
|
||||
# Sized to 2 workers so concurrent searches don't starve other requests.
|
||||
_search_executor: Optional[ThreadPoolExecutor] = None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Application lifespan: build index on startup, cleanup on shutdown."""
|
||||
global _search_executor
|
||||
_search_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="search")
|
||||
logger.info("ObsiGate starting \u2014 building index...")
|
||||
await build_index()
|
||||
logger.info("ObsiGate ready.")
|
||||
yield
|
||||
_search_executor.shutdown(wait=False)
|
||||
_search_executor = None
|
||||
|
||||
|
||||
app = FastAPI(title="ObsiGate", version="1.1.0", lifespan=lifespan)
|
||||
app = FastAPI(title="ObsiGate", version="1.2.0", lifespan=lifespan)
|
||||
|
||||
# Resolve frontend path relative to this file
|
||||
FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
|
||||
@@ -687,22 +704,38 @@ async def api_search(
|
||||
q: str = Query("", description="Search query"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
tag: Optional[str] = Query(None, description="Tag filter"),
|
||||
limit: int = Query(50, ge=1, le=200, description="Results per page"),
|
||||
offset: int = Query(0, ge=0, description="Pagination offset"),
|
||||
):
|
||||
"""Full-text search across vaults with relevance scoring.
|
||||
|
||||
Supports combining free-text queries with tag filters.
|
||||
Results are ranked by a multi-factor scoring algorithm.
|
||||
Pagination via ``limit`` and ``offset`` (defaults preserve backward compat).
|
||||
|
||||
Args:
|
||||
q: Free-text search string.
|
||||
vault: Vault name or ``"all"`` to search everywhere.
|
||||
tag: Comma-separated tag names to require.
|
||||
limit: Max results per page (1–200).
|
||||
offset: Pagination offset.
|
||||
|
||||
Returns:
|
||||
``SearchResponse`` with ranked results and snippets.
|
||||
"""
|
||||
results = search(q, vault_filter=vault, tag_filter=tag)
|
||||
return {"query": q, "vault_filter": vault, "tag_filter": tag, "count": len(results), "results": results}
|
||||
loop = asyncio.get_event_loop()
|
||||
# Fetch full result set (capped at DEFAULT_SEARCH_LIMIT internally)
|
||||
all_results = await loop.run_in_executor(
|
||||
_search_executor,
|
||||
partial(search, q, vault_filter=vault, tag_filter=tag),
|
||||
)
|
||||
total = len(all_results)
|
||||
page = all_results[offset: offset + limit]
|
||||
return {
|
||||
"query": q, "vault_filter": vault, "tag_filter": tag,
|
||||
"count": len(page), "total": total, "offset": offset, "limit": limit,
|
||||
"results": page,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/tags", response_model=TagsResponse)
|
||||
@@ -793,7 +826,12 @@ async def api_advanced_search(
|
||||
Returns:
|
||||
``AdvancedSearchResponse`` with scored results, facets, and pagination info.
|
||||
"""
|
||||
return advanced_search(q, vault_filter=vault, tag_filter=tag, limit=limit, offset=offset, sort_by=sort)
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
_search_executor,
|
||||
partial(advanced_search, q, vault_filter=vault, tag_filter=tag,
|
||||
limit=limit, offset=offset, sort_by=sort),
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/suggest", response_model=SuggestResponse)
|
||||
@@ -924,6 +962,141 @@ async def api_attachment_stats(vault: Optional[str] = Query(None, description="V
|
||||
return {"vaults": stats}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config.json"
|
||||
|
||||
_DEFAULT_CONFIG = {
|
||||
"search_workers": 2,
|
||||
"debounce_ms": 300,
|
||||
"results_per_page": 50,
|
||||
"min_query_length": 2,
|
||||
"search_timeout_ms": 30000,
|
||||
"max_content_size": 100000,
|
||||
"snippet_context_chars": 120,
|
||||
"max_snippet_highlights": 5,
|
||||
"title_boost": 3.0,
|
||||
"path_boost": 1.5,
|
||||
"tag_boost": 2.0,
|
||||
"prefix_max_expansions": 50,
|
||||
}
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
"""Load config from disk, merging with defaults."""
|
||||
config = dict(_DEFAULT_CONFIG)
|
||||
if _CONFIG_PATH.exists():
|
||||
try:
|
||||
stored = _json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
config.update(stored)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read config.json: {e}")
|
||||
return config
|
||||
|
||||
|
||||
def _save_config(config: dict) -> None:
|
||||
"""Persist config to disk."""
|
||||
try:
|
||||
_CONFIG_PATH.write_text(
|
||||
_json.dumps(config, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write config.json: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save config: {e}")
|
||||
|
||||
|
||||
@app.get("/api/config")
|
||||
async def api_get_config():
|
||||
"""Return current configuration with defaults for missing keys."""
|
||||
return _load_config()
|
||||
|
||||
|
||||
@app.post("/api/config")
|
||||
async def api_set_config(body: dict = Body(...)):
|
||||
"""Update configuration. Only known keys are accepted.
|
||||
|
||||
Keys matching ``_DEFAULT_CONFIG`` are validated and persisted.
|
||||
Unknown keys are silently ignored.
|
||||
Returns the full merged config after update.
|
||||
"""
|
||||
current = _load_config()
|
||||
updated_keys = []
|
||||
for key, value in body.items():
|
||||
if key in _DEFAULT_CONFIG:
|
||||
expected_type = type(_DEFAULT_CONFIG[key])
|
||||
if isinstance(value, expected_type) or (expected_type is float and isinstance(value, (int, float))):
|
||||
current[key] = value
|
||||
updated_keys.append(key)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid type for '{key}': expected {expected_type.__name__}, got {type(value).__name__}",
|
||||
)
|
||||
_save_config(current)
|
||||
logger.info(f"Config updated: {updated_keys}")
|
||||
return current
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Diagnostics API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/diagnostics")
|
||||
async def api_diagnostics():
|
||||
"""Return index statistics and system diagnostics.
|
||||
|
||||
Includes document counts, token counts, memory estimates,
|
||||
and inverted index status.
|
||||
"""
|
||||
from backend.search import get_inverted_index
|
||||
import sys
|
||||
|
||||
inv = get_inverted_index()
|
||||
|
||||
# Per-vault stats
|
||||
vault_stats = {}
|
||||
total_files = 0
|
||||
total_tags = 0
|
||||
for vname, vdata in index.items():
|
||||
file_count = len(vdata.get("files", []))
|
||||
tag_count = len(vdata.get("tags", {}))
|
||||
vault_stats[vname] = {"file_count": file_count, "tag_count": tag_count}
|
||||
total_files += file_count
|
||||
total_tags += tag_count
|
||||
|
||||
# Memory estimate for inverted index
|
||||
word_index_entries = sum(len(docs) for docs in inv.word_index.values())
|
||||
mem_estimate_mb = round(
|
||||
(sys.getsizeof(inv.word_index) + word_index_entries * 80
|
||||
+ len(inv.doc_info) * 200
|
||||
+ len(inv._sorted_tokens) * 60) / (1024 * 1024), 2
|
||||
)
|
||||
|
||||
return {
|
||||
"index": {
|
||||
"total_files": total_files,
|
||||
"total_tags": total_tags,
|
||||
"vaults": vault_stats,
|
||||
},
|
||||
"inverted_index": {
|
||||
"unique_tokens": len(inv.word_index),
|
||||
"total_postings": word_index_entries,
|
||||
"documents": inv.doc_count,
|
||||
"sorted_tokens": len(inv._sorted_tokens),
|
||||
"is_stale": inv.is_stale(),
|
||||
"memory_estimate_mb": mem_estimate_mb,
|
||||
},
|
||||
"config": _load_config(),
|
||||
"search_executor": {
|
||||
"active": _search_executor is not None,
|
||||
"max_workers": _search_executor._max_workers if _search_executor else 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files & SPA fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user