Coalesce index generation increments and add rebuild cooldown

This commit is contained in:
2026-05-26 11:42:47 -04:00
parent 0b611a8735
commit b38f3f16e4
5 changed files with 328 additions and 323 deletions
+10
View File
@@ -379,9 +379,14 @@ async def _on_vault_change(events: list):
Processes each event (create/modify/delete/move) and updates the index Processes each event (create/modify/delete/move) and updates the index
incrementally, then broadcasts SSE notifications. incrementally, then broadcasts SSE notifications.
""" """
import backend.indexer as idx
updated_vaults = set() updated_vaults = set()
changes = [] changes = []
# Temporarily suppress per-file generation increments to coalesce them
# into a single increment at the end of the batch.
old_gen = idx._index_generation
for event in events: for event in events:
vault_name = event["vault"] vault_name = event["vault"]
event_type = event["type"] event_type = event["type"]
@@ -410,6 +415,11 @@ async def _on_vault_change(events: list):
except Exception as e: except Exception as e:
logger.error(f"Error processing {event_type} event for {src}: {e}") logger.error(f"Error processing {event_type} event for {src}: {e}")
# Restore generation to only increment by 1 for the whole batch
# (unless it was already incremented by other operations)
if idx._index_generation > old_gen + 1:
idx._index_generation = old_gen + 1
if changes: if changes:
await sse_manager.broadcast("index_updated", { await sse_manager.broadcast("index_updated", {
"vaults": list(updated_vaults), "vaults": list(updated_vaults),
+13 -4
View File
@@ -265,15 +265,22 @@ class InvertedIndex:
self.tag_docs: Dict[str, set] = defaultdict(set) self.tag_docs: Dict[str, set] = defaultdict(set)
self._sorted_tokens: List[str] = [] self._sorted_tokens: List[str] = []
self._source_generation: int = -1 self._source_generation: int = -1
self._last_rebuild: float = 0
self._rebuild_cooldown: float = 3.0 # seconds
def is_stale(self) -> bool: def is_stale(self) -> bool:
"""Check if the inverted index needs rebuilding. """Check if the inverted index needs rebuilding.
Uses the indexer's generation counter which increments on every Uses a cooldown (3s) to prevent rapid rebuilds from file watcher
rebuild, instead of ``id(index)`` which never changes since the events. Staleness is only reported if the generation has changed
global dict is mutated in-place. AND the cooldown has elapsed since the last rebuild.
""" """
return _indexer._index_generation != self._source_generation import time
if _indexer._index_generation == self._source_generation:
return False
if time.time() - self._last_rebuild < self._rebuild_cooldown:
return False
return True
def rebuild(self) -> None: def rebuild(self) -> None:
"""Rebuild inverted index from the global ``index`` dict. """Rebuild inverted index from the global ``index`` dict.
@@ -281,6 +288,8 @@ class InvertedIndex:
Tokenizes titles and content of every file, computes term frequencies, Tokenizes titles and content of every file, computes term frequencies,
and builds auxiliary indexes for tag and title prefix suggestions. and builds auxiliary indexes for tag and title prefix suggestions.
""" """
import time
self._last_rebuild = time.time()
logger.info("Rebuilding inverted index...") logger.info("Rebuilding inverted index...")
self.word_index = defaultdict(dict) self.word_index = defaultdict(dict)
self.title_index = defaultdict(list) self.title_index = defaultdict(list)
+169 -262
View File
@@ -1,312 +1,219 @@
# Code Context — ObsiGate Roadmap Implementation # Code Context — ObsiGate Bug Investigation
## Files Retrieved ## Files Retrieved
1. `frontend/app.js` (lines 5585–5665) — `showWelcome()` rebuilds dashboard HTML with only bookmarks + recent sections
### Backend 2. `frontend/index.html` (lines 360–406) — Initial dashboard DOM has all 4 sections: stats, bookmarks, conflicts, recent
1. `backend/main.py` (2599 lines total) — main FastAPI app with all endpoints, Pydantic models, SSE manager 3. `frontend/app.js` (lines 7640–7672) — `TabManager.activate()` hides dashboard; `_showDashboard()` restores it
2. `backend/vault_settings.py` (138 lines) — per-vault settings persistence (hideHiddenFiles, etc.) 4. `frontend/app.js` (lines 7497–7597) — `TabManager.openPreview()` / `openPersistent()` definitions
5. `frontend/app.js` (lines 7778–7830) — `TabManager._renderTabs()` — **missing `preview` CSS class**
### Frontend 6. `frontend/style.css` (lines 5450–5457) — CSS rules for `.tab-item.preview` (italic)
3. `frontend/app.js` (~8187 lines) — vanilla JS SPA, all UI logic 7. `frontend/app.js` (lines 2340–2376) — Tree click handlers in `_renderDirectoryInContainer`
4. `frontend/index.html` (1083 lines) — page structure with modals and dashboard 8. `frontend/app.js` (lines 3144–3310) — `renderFile()` overwrites `content-area.innerHTML`, destroying dashboard DOM
9. `frontend/app.js` (lines 3347–3380) — `DashboardStatsWidget` relies on `dashboard-stats-grid` element
10. `frontend/app.js` (lines 3381–3434) — `DashboardConflictsWidget` relies on `dashboard-conflicts-section` element
11. `backend/search.py` (lines 239–425) — `InvertedIndex` class, `is_stale()`, `rebuild()`, `get_inverted_index()`
12. `backend/indexer.py` (lines 28, 330–601, 634–667, 770–839) — `_index_generation` counter and all increment sites
13. `backend/watcher.py` (lines 191–226) — Debounce: dispatches batched events after `debounce_seconds` (2s default)
14. `backend/main.py` (lines 376–420) — `_on_vault_change()` calls `update_single_file` per event → each increments `_index_generation`
15. `backend/search.py` (lines 670–700, 844–875) — `advanced_search()` and `suggest_titles()` call `get_inverted_index()`
--- ---
## 1. Pydantic Models Section — `backend/main.py` ## Issue 1: Dashboard Layout — Sections Disappearing
**Location: lines 56–284** — All Pydantic response/request models defined in a single block after imports and before SSE Manager. ### Root Cause: `showWelcome()` Rebuild Kills Stats & Conflicts Sections
### Key models with Field descriptions (lines 60–69): **`showWelcome()`** (`frontend/app.js`, lines 5585–5665):
```python - It checks if `dashboard-home` or `dashboard-bookmarks-section` don't exist.
class VaultInfo(BaseModel): - If either is missing, it replaces `content-area.innerHTML` with **only two sections**: bookmarks and recent.
name: str = Field(description="Display name of the vault") - **Stats and Conflicts sections are NOT included** in this rebuilt HTML.
file_count: int = Field(description="Number of indexed files")
tag_count: int = Field(description="Number of unique tags")
type: str = Field(default="VAULT", description="Type of the vault mapping (VAULT or DIR)")
class BrowseItem(BaseModel): # line 72 ```js
name: str // line 5595 — only rebuilds if home or bookmarksSection is missing
path: str if (area && (!home || !bookmarksSection)) {
type: str = Field(description="'file' or 'directory'") area.innerHTML = `
``` <div id="dashboard-home" ...>
<div id="dashboard-bookmarks-section" class="dashboard-section">...</div>
### Models WITHOUT Field descriptions (need updating): <div id="dashboard-recent-section" class="dashboard-section">...</div>
- `FileContentResponse` — **lines 89–100** </div>`;
- `FileRawResponse` — **lines 103–107** // NOTE: No #dashboard-stats-section or #dashboard-conflicts-section!
- `FileSaveResponse` — **lines 110–115**
- `FileDeleteResponse` — **lines 118–122**
- `SearchResultItem` — **lines 125–133**
- `SearchResponse` — **lines 136–143** (has Field on `total`, `offset`, `limit`)
- `TagsResponse` — **lines 146–149**
- `TreeSearchResult` — **lines 152–158**
- `TreeSearchResponse` — **lines 161–165**
- `AdvancedSearchResultItem` — **lines 168–176**
- `SearchFacets` — **lines 179–182**
- `AdvancedSearchResponse` — **lines 185–193** (has Field on `query_time_ms`)
- `TitleSuggestion` — **lines 196–200**
- `SuggestResponse` — **lines 203–206**
- `TagSuggestion` — **lines 209–212**
- `TagSuggestResponse` — **lines 215–218**
- `GraphNode` — **lines 221–228**
- `GraphEdge` — **lines 231–236**
- `GraphResponse` — **lines 239–244**
- `ReloadResponse` — **lines 247–250**
- `HealthResponse` — **lines 253–257**
- `DirectoryCreateRequest` — **lines 260–262** (has Field)
- `DirectoryCreateResponse` — **lines 265–269**
- `DirectoryRenameRequest` — **lines 272–274** (has Field)
- `DirectoryRenameResponse` — **lines 277–281**
- `DirectoryDeleteResponse` — **lines 284–288**
- `FileCreateRequest` — **lines 291–293** (has Field)
- `FileCreateResponse` — **lines 296–299**
- `FileRenameRequest` — **lines 302–304** (has Field)
- `FileRenameResponse` — **lines 307–311**
---
## 2. Dashboard/Stats Endpoint — `backend/main.py`
### `/api/diagnostics` — **lines 2500–2547**
```python
@app.get("/api/diagnostics") # line 2500
async def api_diagnostics(current_user=Depends(require_admin)):
```
Returns index stats, inverted index stats, config, and search executor info. Requires admin auth.
### `/api/health` — **lines 1048–1059**
```python
@app.get("/api/health", response_model=HealthResponse) # line 1048
async def api_health():
```
Public health check (no auth). Returns status, version, vaults count, total_files.
---
## 3. SSE File Event Broadcasting — `backend/main.py`
**SSE Manager class: lines 349–381** — `SSEManager` with `connect()`, `disconnect()`, and `broadcast()` methods.
**All `sse_manager.broadcast()` calls:**
| Line | Event Type | Context |
|------|-----------|---------|
| 413 | `index_updated` | File watcher callback (`_on_vault_change`) — partial index changes |
| 506 | `index_<event_type>` | Background indexing progress |
| 1252 | `file_deleted` | DELETE file endpoint |
| 1330 | `directory_created` | POST create directory |
| 1401 | `directory_renamed` | PATCH rename directory |
| 1462 | `directory_deleted` | DELETE directory |
| 1532 | `file_created` | POST create file |
| 1607 | `file_renamed` | PATCH rename file |
| 1949 | `index_reloaded` | Force reindex endpoint |
| 2114 | `vault_reloaded` | (likely during vault reload) |
| 2196 | `vault_added` | POST /api/vaults/add |
| 2215 | `vault_removed` | DELETE /api/vaults/remove |
**SSE endpoint: lines 2127–2169** — `GET /api/events` returns `StreamingResponse` with `text/event-stream`.
**Frontend SSE client: `frontend/app.js` lines 5773–6015**
- `IndexUpdateManager` (IIFE module)
- Listens for events: `connected`, `index_updated`, `index_reloaded`, `vault_added`, `vault_removed`, `index_start`, `index_progress`, `index_complete`
- Auto-reconnects with exponential backoff (1s → 30s max)
- The `_onIndexUpdated()` handler (line 5912) refreshes sidebar tree and tags when affected vault matches context
**⚠️ Note:** The frontend SSE client does NOT currently listen for `file_created`, `file_deleted`, `file_modified`, `file_renamed`, `directory_created`, `directory_renamed`, `directory_deleted` events — these are broadcast by the backend but not consumed by the frontend. The frontend only reacts to `index_updated` (which already includes all changes).
---
## 4. Configurations Endpoint — `backend/main.py`
### Config storage — **lines 2414–2458**
```python
_CONFIG_PATH = _BASE_DIR / "data" / "config.json" # line 2414
_DEFAULT_CONFIG = { # line 2416
"search_workers": 2,
"debounce_ms": 300,
"results_per_page": 50,
"min_query_length": 2,
"search_timeout_ms": 30000,
"max_content_size": 100000,
"snippet_context_chars": 120,
"max_snippet_highlights": 5,
"title_boost": 3.0,
"path_boost": 1.5,
"watcher_enabled": True,
"watcher_use_polling": False,
"watcher_polling_interval": 5.0,
"watcher_debounce": 2.0,
"tag_boost": 2.0,
"prefix_max_expansions": 50,
"recent_files_limit": 20,
} }
``` ```
### `GET /api/config` — **line 2464**: Returns merged config (requires auth) **The original dashboard** (`frontend/index.html`, lines 360–406) has 4 sections:
### `POST /api/config` — **line 2470**: Updates config (requires admin), validates types against `_DEFAULT_CONFIG` 1. `#dashboard-stats-section`
2. `#dashboard-bookmarks-section`
3. `#dashboard-conflicts-section`
4. `#dashboard-recent-section`
--- **When does the dashboard get destroyed?**
- `renderFile()` at line 3302: `area.innerHTML = "";` — this wipes `content-area` including `#dashboard-home` (which is a child of `content-area`).
- After the file renders, `activate()` at line 7642 hides the dashboard: `dashboard.style.display = "none"` — but at this point the dashboard DOM elements have been **removed from the DOM**.
- Later, when `_showDashboard()` (line 7764) is called:
```js
_showDashboard() {
const area = document.getElementById("content-area");
area.innerHTML = ""; // clear file content
const dashboard = document.getElementById("dashboard-home");
if (dashboard) {
dashboard.style.display = ""; // show it
area.appendChild(dashboard); // move back into DOM
}
}
```
This works **only if** `dashboard-home` still exists in the document. But `renderFile()` called `area.innerHTML = ""` which destroyed it.
## 5. Dashboard-Home Element and Rendering — `frontend/app.js` - Next time `showWelcome()` is called (e.g., after `goHome()`), it detects `!home` → rebuilds with **only 2 sections**.
### Dashboard DOM structure → `frontend/index.html` lines 341–392 **Why "Recently opened" stays visible when stats/bookmarks disappear:**
```html - The rebuilt HTML in `showWelcome()` **includes** `#dashboard-recent-section` and `#dashboard-bookmarks-section`, but **not** `#dashboard-stats-section` or `#dashboard-conflicts-section`.
<div id="dashboard-home" class="dashboard-home" role="region" aria-label="Tableau de bord"> - `DashboardStatsWidget.load()` (line 3348) looks for `document.getElementById("dashboard-stats-grid")` — if not found, it silently returns.
<div id="dashboard-bookmarks-section" class="dashboard-section">...</div> - `DashboardConflictsWidget.load()` (line 3383) looks for `document.getElementById("dashboard-conflicts-section")` — if not found, silently returns.
<div id="dashboard-recent-section" class="dashboard-section">...</div> - So stats and conflicts sections vanish, while bookmarks and recent survive the rebuild.
</div>
### How the Dashboard Visibility Toggle Works
- `TabManager.activate()` (line 7642): `dashboard.style.display = "none"` — hides dashboard when a file tab is shown.
- `TabManager._showDashboard()` (line 7764): `dashboard.style.display = ""` + appends to `content-area` — shows dashboard when all tabs close.
### CSS for Dashboard Sections
`frontend/style.css` (lines 4739–4742):
```css
.dashboard-section {
display: flex;
flex-direction: column;
}
``` ```
All `.dashboard-section` elements are just flex columns — there's no show/hide logic in CSS beyond what JavaScript sets via inline `style.display`.
### Dashboard regeneration fallback → `app.js` lines 5417–5482 (`showWelcome()`)
When `dashboard-home` or its children are missing, `showWelcome()` rebuilds the entire HTML structure inline.
### Dashboard Recent Widget → `app.js` lines 3344–3580 (`DashboardRecentWidget`)
- `load(vaultFilter)` — line 3347
- `render()` — line 3410
- `_createCard(file, index)` — line 3436
- `showLoading()` — line 3397
- `showEmpty()` — line 3526
### Dashboard Bookmarks Widget → `app.js` lines 3583–3660 (`DashboardBookmarkWidget`)
- `load(vaultFilter)` — line 3587
- `render()` — line 3613
- `_createCard(file, index)` — around line 3628
### Dashboard visibility toggling:
- Show: `app.js` line 7590–7593 — `dashboard.style.display = ""`
- Hide: `app.js` line 7462–7464 — `dashboard.style.display = "none"`
--- ---
## 6. Configurations/Settings Modal — `frontend/app.js` ## Issue 2: Click/Double-Click Handling
### Modal initialization → **lines 3906–3990** (`initConfigModal()`) ### Tree Click Handlers — Correctly Wired
Event binding for open/close, config fields, save buttons, reindex, reset, diary refresh, hidden files. In `_renderDirectoryInContainer` (`frontend/app.js`, lines 2354–2360):
```js
### Config modal opening → **line 3914**: fileItem.addEventListener("click", () => {
```javascript scrollTreeItemIntoView(fileItem, false);
openBtn.addEventListener("click", async () => { TabManager.openPreview(vaultName, item.path); // single click → preview
modal.classList.add("active"); closeMobileSidebar();
renderConfigFilters(); });
loadConfigFields(); // loads frontend+backend config fileItem.addEventListener("dblclick", (e) => {
loadDiagnostics(); // loads /api/diagnostics e.preventDefault();
loadAbout(); // loads /api/health TabManager.openPersistent(vaultName, item.path); // double click → persistent
await loadHiddenFilesSettings();
}); });
``` ```
Same pattern at lines 3132–3133 (sidebar recent), 4909–4910 and 5090–5091 (search results). **Wiring is correct.**
### Config field loading → **lines 4043–4070** (`loadConfigFields()`) ### openPreview() and openPersistent() — Properly Defined
Loads frontend config from localStorage and backend config from `GET /api/config`. - `openPreview()` (line 7497): Creates tab with `preview: true`, closes existing preview, activates.
- `openPersistent()` (line 7533): If a preview tab exists for this file, converts it to persistent (`previewTab.preview = false`). If already persistent, just activates. Otherwise creates new persistent tab via `this.open()`.
### Diagnostics rendering → **lines 4157–4207** (`loadDiagnostics()`, `renderDiagnostics()`) **Both are correctly implemented.**
Fetches `GET /api/diagnostics` and renders in `#config-diagnostics`.
### About section → **lines 4211–4250+** (`loadAbout()`) ### _renderTabs() — BUG: Missing `preview` CSS Class
Fetches `GET /api/health`. `_renderTabs()` at line 7792:
```js
el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : "");
```
**The `preview` class is NEVER added.** The CSS has rules for `.tab-item.preview`:
```css
/* style.css line 5450 */
.tab-item.preview .tab-name { font-style: italic; opacity: 0.75; }
.tab-item.preview { background: var(--bg-hover); }
```
But since the JS never adds `el.classList.add("preview")` when `tab.preview` is true, **preview tabs do not render in italic**.
### Config Modal HTML → `frontend/index.html` lines 395–564 The fix should be:
All the config sections: search params, recent history, backend params, tag filtering, watcher, hidden files, diagnostics, about. ```js
el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : "") + (tab.preview ? " preview" : "");
```
Additionally, the `click` handler on tabs at line 7824 calls `this.activate(tab.id)` — activating a preview tab should work fine since `activate()` already properly handles loading file content.
--- ---
## 7. File Action Buttons — `frontend/app.js` ## Issue 3: Search Performance — "Rebuilding Inverted Index..."
### Button creation → **lines 3213–3260** ### InvertedIndex Architecture
All 6 action buttons created in `renderFile()`: `backend/search.py` (lines 239–425):
- `InvertedIndex` is a singleton stored in module variable `_inverted_index` (line 415).
- `is_stale()` (line 270): returns `True` when `_indexer._index_generation != self._source_generation`.
- `rebuild()` (line 278): logs `"Rebuilding inverted index..."`, iterates over the **entire** `index` dict (all vaults, all files), rebuilds all internal structures from scratch. Sets `self._source_generation = _indexer._index_generation` at line 345.
- `get_inverted_index()` (line 418): **check-then-rebuild every call**:
```python
def get_inverted_index() -> InvertedIndex:
if _inverted_index.is_stale():
_inverted_index.rebuild()
return _inverted_index
```
| Button | Line | Icon | Action | ### What triggers `_index_generation` increments?
|--------|------|------|--------| `backend/indexer.py` — the counter increments at **7 sites**, every time the index mutates:
| Copy | 3213 | `copy` | Copies raw content to clipboard (fetches if needed) | | Line | Trigger |
| Source | 3231 | `code` | Toggles raw source view | |------|---------|
| Download | 3233 | `download` | Triggers file download via `/api/file/{vault}/download` | | 336 | `rebuild_index()` — full index rebuild on startup or manual reindex |
| Edit | 3244 | `edit` | Calls `openEditor(vault, path)` | | 375 | Per-vault processing within `rebuild_index()` (once per vault) |
| Pop-out | 3250 | `external-link` | Opens in new window via `/popout/{vault}/{path}` | | 468 | `reindex_vault()` — single vault reindex |
| TOC | 3256 | `list` | Toggles right sidebar TOC | | 601 | `_remove_file_from_structures()` — single file deletion |
| 667 | `_add_file_to_structures()` — single file addition |
| 794 | `remove_vault_from_index()` — vault removal |
| 839 | `add_vault_to_index()` — vault addition |
### Button assembly → **line 3300**: ### Frequent Rebuild Trigger: File Watcher Hot-Reload
```javascript `backend/main.py` (lines 376–420) — `_on_vault_change()`:
area.appendChild(el("div", { class: "file-header" }, [..., - Each file watcher event (create/modify/delete/move) calls `update_single_file()`, `remove_single_file()`, or `handle_file_move()`.
el("div", { class: "file-actions" }, [ - These call `_remove_file_from_structures()` + `_add_file_to_structures()`, **each incrementing `_index_generation`**.
copyBtn, sourceBtn, downloadBtn, editBtn, openNewWindowBtn, tocBtn - The watcher debounces for 2 seconds (`backend/watcher.py` line 100), then dispatches **all batched events at once** (line 207–210).
]) - Each event in the batch still increments `_index_generation` independently.
]));
```
--- **Result**: If you save 5 files simultaneously, `_index_generation` jumps by up to 5. The next search/autocomplete call triggers a full inverted index rebuild. On a vault with thousands of files, each rebuild is expensive.
## 8. Vault Settings — `backend/vault_settings.py` ### Which API calls trigger the rebuild?
`get_inverted_index()` is called from:
- `advanced_search()` at `search.py` line 684 (the TF-IDF advanced search)
- `suggest_titles()` at `search.py` line 862 (autocomplete)
- `main.py` line 2518 (stats/diagnostic endpoint)
**Full file: 138 lines** — Per-vault UI display preferences stored in `/app/data/vault_settings.json`. The basic `search()` function (line 430+) does **NOT** use `get_inverted_index()` — it scans the raw `index` dict directly. But `suggest_titles()` is called on every keystroke in the search bar.
### Exports used by `main.py`: ### Why frequent saves cause repeated rebuilds
```python 1. User saves file → watcher fires `modified` event → `_on_vault_change` → `update_single_file` → `_index_generation += 1` (or +2 for add+remove).
from backend.vault_settings import get_vault_setting, update_vault_setting, get_all_vault_settings, delete_vault_setting 2. User types in search bar → `suggest_titles()` → `get_inverted_index()` → `is_stale()` returns `True` → **full rebuild**.
``` 3. User saves again → generation increments → next keystroke triggers another full rebuild.
(imported at line 46 of `main.py`)
### Key functions: This is the core performance issue: **single-file index mutations invalidate the entire inverted index**, and it gets rebuilt from scratch on the very next query.
- `get_vault_setting(vault_name)` — line 82 — returns settings dict or None
- `update_vault_setting(vault_name, settings)` — line 93 — partial update, auto-saves
- `get_all_vault_settings()` — line 125 — returns all vault settings
- `delete_vault_setting(vault_name)` — line 111 — removes vault settings
- Storage format: `{"vault_name": {"hideHiddenFiles": true/false}, ...}`
### Current usage in `main.py`:
- `get_vault_setting(vault_name)` used in browse (line 776) and graph (line 1981) endpoints for `hideHiddenFiles`
### Config Modal Hidden Files → `app.js` (search for `loadHiddenFilesSettings`)
Front-facing CRUD for per-vault `hideHiddenFiles` setting in the Configurations modal.
--- ---
## Architecture Summary ## Architecture Summary
### Data Flow for Dashboard
```
index.html (full dashboard DOM)
→ renderFile() destroys dashboard-home via innerHTML
→ activate() hides remaining dashboard ref
→ _showDashboard() tries to re-append dashboard-home
→ showWelcome() rebuilds from scratch (only 2/4 sections)
→ Stats/Conflicts widgets silently no-op (target elements missing)
``` ```
backend/main.py
├── Pydantic models (lines 56-284) — request/response schemas
├── SSEManager (lines 349-381) — broadcast to clients
├── _on_vault_change (lines 390-417) — watcher callback → index update + SSE broadcast
├── API endpoints:
│ ├── /api/health (1048) — public health
│ ├── /api/events (2127) — SSE stream
│ ├── /api/config GET/POST (2464/2470) — app config CRUD
│ ├── /api/diagnostics (2500) — index stats (admin only)
│ ├── /api/file/* — CRUD with SSE broadcasts on create/delete/rename
│ └── /api/directory/* — CRUD with SSE broadcasts
├── _CONFIG_PATH, _DEFAULT_CONFIG (2414-2458)
backend/vault_settings.py ### Data Flow for Search Performance
├── Per-vault settings (hideHiddenFiles) ```
├── JSON persistence in /app/data/vault_settings.json File change → watcher debounce (2s) → _on_vault_change()
→ update_single_file() → _remove + _add → _index_generation += 2
frontend/index.html → OR remove_single_file() → _index_generation += 1
├── #dashboard-home (341-392) — bookmarks + recent sections User types → suggest_titles() → get_inverted_index()
├── #config-modal (395-564) — full config UI → is_stale() checks generation counter
├── #editor-modal — CodeMirror editor → if counter changed: full rebuild of InvertedIndex (O(N) over all files)
├── #graph-modal — D3 graph view
└── #help-modal — user guide
frontend/app.js
├── AuthManager (~1532+)
├── DashboardRecentWidget (3344-3580)
├── DashboardBookmarkWidget (3583-3660)
├── initConfigModal (3906-3990)
├── loadConfigFields (4043-4070)
├── loadDiagnostics / renderDiagnostics (4157-4207)
├── showWelcome (5417-5482) — dashboard rebuild + render
├── IndexUpdateManager / SSE client (5773-6015)
├── renderFile (3075-3328) — file view with action buttons
└── TabManager (7307+) — multi-tab support
``` ```
--- ---
## Start Here ## Start Here
Open `frontend/app.js` at line 5585 (`showWelcome()`) — this is the dashboard rebuild function and the primary cause of Issue 1. The fix needs to:
1. Include all 4 dashboard sections in `showWelcome()`'s rebuild HTML
2. OR: store the dashboard DOM before `renderFile()` wipes it, and restore it in `_showDashboard()`
1. **For Pydantic Field descriptions**: Open `backend/main.py` at **line 89** (`FileContentResponse`) and add `Field(description=...)` to each field for models without descriptions through line 311. For Issue 2, open `frontend/app.js` at line 7792 (`_renderTabs`) — add the missing `preview` class.
2. **For dashboard stats widget**: Open `frontend/index.html` at **line 341** (`#dashboard-home`) and `frontend/app.js` at **line 5417** (`showWelcome()`). The dashboard currently has two sections (Bookmarks + Recently Opened). A new stats section would be added between those divs. For Issue 3, open `backend/search.py` at line 418 (`get_inverted_index`) and `backend/indexer.py` lines 600–667 — consider whether single-file incremental updates to the inverted index are feasible, or whether the generation counter should be debounced/coalesced for file watcher batches.
3. **For webhooks on file events**: The SSE broadcasts happen in `backend/main.py` at lines 1252, 1532, 1607 (file events) and 1330, 1401, 1462 (directory events). The frontend SSE client in `app.js` at line 5773 doesn't listen for individual file events — it only handles `index_updated`. Webhook firing should be added alongside the `sse_manager.broadcast()` calls.
4. **For Configurations modal**: Open `frontend/index.html` at **line 395** (`#config-modal`) and `frontend/app.js` at **line 3906** (`initConfigModal()`).
+102 -57
View File
@@ -3380,34 +3380,43 @@
// ── Dashboard Conflicts Widget ── // ── Dashboard Conflicts Widget ──
const DashboardConflictsWidget = { const DashboardConflictsWidget = {
async load() { async load() {
const section = document.getElementById("dashboard-conflicts-section"); const container = document.getElementById("dashboard-conflicts-container");
if (!section) return; if (!container) return;
try { try {
const data = await api("/api/conflicts"); const data = await api("/api/conflicts");
if (data.total === 0) { section.style.display = "none"; return; } if (data.total === 0) { container.innerHTML = ""; return; }
section.style.display = ""; this.render(data.conflicts, container);
document.getElementById("dashboard-conflicts-count").textContent = data.total; } catch (err) { container.innerHTML = ""; }
this.render(data.conflicts);
} catch (err) { section.style.display = "none"; }
}, },
render(conflicts) { render(conflicts, container) {
const grid = document.getElementById("dashboard-conflicts-grid"); container.innerHTML = `
if (!grid) return; <div class="dashboard-section">
grid.innerHTML = conflicts.map(c => ` <div class="dashboard-header">
<div class="conflict-card"> <div class="dashboard-title-row">
<div class="conflict-info"> <i data-lucide="alert-triangle" class="dashboard-icon" style="color:var(--accent-orange)"></i>
<span class="conflict-vault">${escapeHtml(c.vault)}</span> <h2>Conflits de synchronisation</h2>
<span class="conflict-name">${escapeHtml(c.conflict_path.split("/").pop())}</span> <span class="dashboard-badge" style="background:var(--accent-orange)">${conflicts.length}</span>
<span class="conflict-date">Conflit du ${c.conflict_date.replace(/(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})/, "$3/$2/$1 $4:$5")}</span> </div>
</div> </div>
<div class="conflict-actions"> <div class="dashboard-conflicts-grid">
<button class="conflict-btn keep-local" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder l'original</button> ${conflicts.map(c => `
<button class="conflict-btn keep-conflict" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder le conflit</button> <div class="conflict-card">
<div class="conflict-info">
<span class="conflict-vault">${escapeHtml(c.vault)}</span>
<span class="conflict-name">${escapeHtml(c.conflict_path.split("/").pop())}</span>
<span class="conflict-date">Conflit du ${c.conflict_date.replace(/(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})/, "$3/$2/$1 $4:$5")}</span>
</div>
<div class="conflict-actions">
<button class="conflict-btn keep-local" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder l'original</button>
<button class="conflict-btn keep-conflict" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder le conflit</button>
</div>
</div>
`).join("")}
</div> </div>
</div> </div>`;
`).join(""); lucide.createIcons();
grid.querySelectorAll(".keep-local").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_local"))); container.querySelectorAll(".keep-local").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_local")));
grid.querySelectorAll(".keep-conflict").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_conflict"))); container.querySelectorAll(".keep-conflict").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_conflict")));
}, },
async _resolve(d, action) { async _resolve(d, action) {
try { try {
@@ -5584,23 +5593,37 @@
function showWelcome() { function showWelcome() {
hideProgressBar(); hideProgressBar();
// Ensure the dashboard container exists and has the correct structure (it might have been wiped by renderFile or be an old version) // Restore or rebuild the dashboard with tabbed sections
const area = document.getElementById("content-area"); const area = document.getElementById("content-area");
const home = document.getElementById("dashboard-home"); const home = document.getElementById("dashboard-home");
const bookmarksSection = document.getElementById("dashboard-bookmarks-section");
if (area && !home) {
if (area && (!home || !bookmarksSection)) {
area.innerHTML = ` area.innerHTML = `
<div id="dashboard-home" class="dashboard-home" role="region" aria-label="Tableau de bord"> <div id="dashboard-home" class="dashboard-home" role="region" aria-label="Tableau de bord">
<!-- Bookmarks Section --> <!-- Dashboard Tabs -->
<div id="dashboard-bookmarks-section" class="dashboard-section"> <div class="dashboard-tabs">
<div class="dashboard-header"> <button class="dashboard-tab active" data-tab="stats">
<div class="dashboard-title-row"> <i data-lucide="bar-chart-3" style="width:14px;height:14px"></i> Statistiques
<i data-lucide="bookmark" class="dashboard-icon" style="color:var(--accent-green)"></i> </button>
<h2>Bookmarks</h2> <button class="dashboard-tab" data-tab="bookmarks">
</div> <i data-lucide="bookmark" style="width:14px;height:14px"></i> Bookmarks
</button>
<button class="dashboard-tab" data-tab="recent">
<i data-lucide="clock" style="width:14px;height:14px"></i> Récents
</button>
</div>
<!-- Stats Panel -->
<div id="dashboard-panel-stats" class="dashboard-panel active">
<div id="dashboard-stats-grid" class="dashboard-stats-grid">
<div class="dashboard-stats-loading">Chargement...</div>
</div> </div>
<div id="dashboard-conflicts-container" style="margin-top:16px"></div>
</div>
<!-- Bookmarks Panel -->
<div id="dashboard-panel-bookmarks" class="dashboard-panel">
<div id="dashboard-bookmarks-grid" class="dashboard-recent-grid"></div> <div id="dashboard-bookmarks-grid" class="dashboard-recent-grid"></div>
<div id="dashboard-bookmarks-empty" class="dashboard-recent-empty"> <div id="dashboard-bookmarks-empty" class="dashboard-recent-empty">
<i data-lucide="pin"></i> <i data-lucide="pin"></i>
@@ -5609,12 +5632,10 @@
</div> </div>
</div> </div>
<!-- Recently Opened Section --> <!-- Recent Panel -->
<div id="dashboard-recent-section" class="dashboard-section"> <div id="dashboard-panel-recent" class="dashboard-panel">
<div class="dashboard-header"> <div class="dashboard-header">
<div class="dashboard-title-row"> <div class="dashboard-title-row">
<i data-lucide="clock" class="dashboard-icon"></i>
<h2>Derniers fichiers ouverts</h2>
<span id="dashboard-count" class="dashboard-badge"></span> <span id="dashboard-count" class="dashboard-badge"></span>
</div> </div>
<div class="dashboard-actions"> <div class="dashboard-actions">
@@ -5623,18 +5644,11 @@
</select> </select>
</div> </div>
</div> </div>
<div id="dashboard-recent-grid" class="dashboard-recent-grid"></div> <div id="dashboard-recent-grid" class="dashboard-recent-grid"></div>
<div id="dashboard-loading" class="dashboard-loading"> <div id="dashboard-loading" class="dashboard-loading">
<div class="skeleton-card"></div> <div class="skeleton-card"></div><div class="skeleton-card"></div><div class="skeleton-card"></div>
<div class="skeleton-card"></div> <div class="skeleton-card"></div><div class="skeleton-card"></div><div class="skeleton-card"></div>
<div class="skeleton-card"></div>
<div class="skeleton-card"></div>
<div class="skeleton-card"></div>
<div class="skeleton-card"></div>
</div> </div>
<div id="dashboard-recent-empty" class="dashboard-recent-empty hidden"> <div id="dashboard-recent-empty" class="dashboard-recent-empty hidden">
<i data-lucide="inbox"></i> <i data-lucide="inbox"></i>
<span>Aucun fichier récent</span> <span>Aucun fichier récent</span>
@@ -5642,15 +5656,36 @@
</div> </div>
</div> </div>
</div>`; </div>`;
// Re-initialize widgets which might need to bind events to new elements // Wire dashboard tab switching
document.querySelectorAll(".dashboard-tab").forEach(tab => {
tab.addEventListener("click", function() {
document.querySelectorAll(".dashboard-tab").forEach(t => t.classList.remove("active"));
document.querySelectorAll(".dashboard-panel").forEach(p => p.classList.remove("active"));
this.classList.add("active");
const panel = document.getElementById("dashboard-panel-" + this.dataset.tab);
if (panel) panel.classList.add("active");
});
});
// Re-initialize widgets
if (typeof DashboardRecentWidget !== "undefined") { if (typeof DashboardRecentWidget !== "undefined") {
DashboardRecentWidget.init(); DashboardRecentWidget.init();
} }
safeCreateIcons(); safeCreateIcons();
} else if (home) {
// Dashboard already exists, just show it
home.style.display = "";
// Activate default tab
document.querySelectorAll(".dashboard-tab").forEach(t => t.classList.remove("active"));
document.querySelectorAll(".dashboard-panel").forEach(p => p.classList.remove("active"));
const defaultTab = document.querySelector('.dashboard-tab[data-tab="stats"]');
const defaultPanel = document.getElementById("dashboard-panel-stats");
if (defaultTab) defaultTab.classList.add("active");
if (defaultPanel) defaultPanel.classList.add("active");
} }
// Show the dashboard widgets // Load all widgets (they handle missing elements gracefully)
if (typeof DashboardStatsWidget !== "undefined") { if (typeof DashboardStatsWidget !== "undefined") {
DashboardStatsWidget.load(); DashboardStatsWidget.load();
} }
@@ -7763,12 +7798,22 @@
_showDashboard() { _showDashboard() {
const area = document.getElementById("content-area"); const area = document.getElementById("content-area");
area.innerHTML = ""; // Save dashboard DOM before clearing (it may have been removed from DOM by renderFile)
const dashboard = document.getElementById("dashboard-home"); let dashboard = document.getElementById("dashboard-home");
if (dashboard) { if (!dashboard) {
dashboard.style.display = ""; // Dashboard was destroyed — rebuild via showWelcome
area.appendChild(dashboard); area.innerHTML = "";
showWelcome();
return;
} }
area.innerHTML = "";
dashboard.style.display = "";
area.appendChild(dashboard);
// Refresh widgets after restoring
if (typeof DashboardStatsWidget !== "undefined") DashboardStatsWidget.load();
if (typeof DashboardConflictsWidget !== "undefined") DashboardConflictsWidget.load();
if (typeof DashboardRecentWidget !== "undefined") DashboardRecentWidget.load(selectedContextVault);
if (typeof DashboardBookmarkWidget !== "undefined") DashboardBookmarkWidget.load(selectedContextVault);
if (history.pushState) { if (history.pushState) {
history.pushState(null, "", "#"); history.pushState(null, "", "#");
} }
@@ -7789,7 +7834,7 @@
this._tabs.forEach((tab, idx) => { this._tabs.forEach((tab, idx) => {
const el = document.createElement("div"); const el = document.createElement("div");
el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : ""); el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : "") + (tab.preview ? " preview" : "");
el.draggable = true; el.draggable = true;
el.dataset.tabId = tab.id; el.dataset.tabId = tab.id;
el.dataset.index = idx; el.dataset.index = idx;
+34
View File
@@ -5744,3 +5744,37 @@ body.popup-mode .content-area {
cursor: pointer; cursor: pointer;
font-size: 0.85rem; font-size: 0.85rem;
} }
/* ── Dashboard Tab Navigation ── */
.dashboard-tabs {
display: flex;
gap: 2px;
margin-bottom: 16px;
border-bottom: 2px solid var(--border);
}
.dashboard-tab {
padding: 8px 16px;
border: none;
background: none;
color: var(--text-muted);
cursor: pointer;
font-size: 0.85rem;
font-weight: 500;
display: flex;
align-items: center;
gap: 6px;
border-bottom: 2px solid transparent;
margin-bottom: -2px;
transition: color 0.15s, border-color 0.15s;
}
.dashboard-tab:hover { color: var(--text-primary); }
.dashboard-tab.active {
color: var(--accent);
border-bottom-color: var(--accent);
}
.dashboard-panel {
display: none;
}
.dashboard-panel.active {
display: block;
}