Coalesce index generation increments and add rebuild cooldown

This commit is contained in:
Bruno Charest 2026-05-26 11:42:47 -04:00
parent 0b611a8735
commit b38f3f16e4
5 changed files with 328 additions and 323 deletions

View File

@ -379,9 +379,14 @@ async def _on_vault_change(events: list):
Processes each event (create/modify/delete/move) and updates the index Processes each event (create/modify/delete/move) and updates the index
incrementally, then broadcasts SSE notifications. incrementally, then broadcasts SSE notifications.
""" """
import backend.indexer as idx
updated_vaults = set() updated_vaults = set()
changes = [] changes = []
# Temporarily suppress per-file generation increments to coalesce them
# into a single increment at the end of the batch.
old_gen = idx._index_generation
for event in events: for event in events:
vault_name = event["vault"] vault_name = event["vault"]
event_type = event["type"] event_type = event["type"]
@ -410,6 +415,11 @@ async def _on_vault_change(events: list):
except Exception as e: except Exception as e:
logger.error(f"Error processing {event_type} event for {src}: {e}") logger.error(f"Error processing {event_type} event for {src}: {e}")
# Restore generation to only increment by 1 for the whole batch
# (unless it was already incremented by other operations)
if idx._index_generation > old_gen + 1:
idx._index_generation = old_gen + 1
if changes: if changes:
await sse_manager.broadcast("index_updated", { await sse_manager.broadcast("index_updated", {
"vaults": list(updated_vaults), "vaults": list(updated_vaults),

View File

@ -265,15 +265,22 @@ class InvertedIndex:
self.tag_docs: Dict[str, set] = defaultdict(set) self.tag_docs: Dict[str, set] = defaultdict(set)
self._sorted_tokens: List[str] = [] self._sorted_tokens: List[str] = []
self._source_generation: int = -1 self._source_generation: int = -1
self._last_rebuild: float = 0
self._rebuild_cooldown: float = 3.0 # seconds
def is_stale(self) -> bool: def is_stale(self) -> bool:
"""Check if the inverted index needs rebuilding. """Check if the inverted index needs rebuilding.
Uses the indexer's generation counter which increments on every Uses a cooldown (3s) to prevent rapid rebuilds from file watcher
rebuild, instead of ``id(index)`` which never changes since the events. Staleness is only reported if the generation has changed
global dict is mutated in-place. AND the cooldown has elapsed since the last rebuild.
""" """
return _indexer._index_generation != self._source_generation import time
if _indexer._index_generation == self._source_generation:
return False
if time.time() - self._last_rebuild < self._rebuild_cooldown:
return False
return True
def rebuild(self) -> None: def rebuild(self) -> None:
"""Rebuild inverted index from the global ``index`` dict. """Rebuild inverted index from the global ``index`` dict.
@ -281,6 +288,8 @@ class InvertedIndex:
Tokenizes titles and content of every file, computes term frequencies, Tokenizes titles and content of every file, computes term frequencies,
and builds auxiliary indexes for tag and title prefix suggestions. and builds auxiliary indexes for tag and title prefix suggestions.
""" """
import time
self._last_rebuild = time.time()
logger.info("Rebuilding inverted index...") logger.info("Rebuilding inverted index...")
self.word_index = defaultdict(dict) self.word_index = defaultdict(dict)
self.title_index = defaultdict(list) self.title_index = defaultdict(list)

View File

@ -1,312 +1,219 @@
# Code Context — ObsiGate Roadmap Implementation # Code Context — ObsiGate Bug Investigation
## Files Retrieved ## Files Retrieved
1. `frontend/app.js` (lines 55855665) — `showWelcome()` rebuilds dashboard HTML with only bookmarks + recent sections
### Backend 2. `frontend/index.html` (lines 360406) — Initial dashboard DOM has all 4 sections: stats, bookmarks, conflicts, recent
1. `backend/main.py` (2599 lines total) — main FastAPI app with all endpoints, Pydantic models, SSE manager 3. `frontend/app.js` (lines 76407672) — `TabManager.activate()` hides dashboard; `_showDashboard()` restores it
2. `backend/vault_settings.py` (138 lines) — per-vault settings persistence (hideHiddenFiles, etc.) 4. `frontend/app.js` (lines 74977597) — `TabManager.openPreview()` / `openPersistent()` definitions
5. `frontend/app.js` (lines 77787830) — `TabManager._renderTabs()` — **missing `preview` CSS class**
### Frontend 6. `frontend/style.css` (lines 54505457) — CSS rules for `.tab-item.preview` (italic)
3. `frontend/app.js` (~8187 lines) — vanilla JS SPA, all UI logic 7. `frontend/app.js` (lines 23402376) — Tree click handlers in `_renderDirectoryInContainer`
4. `frontend/index.html` (1083 lines) — page structure with modals and dashboard 8. `frontend/app.js` (lines 31443310) — `renderFile()` overwrites `content-area.innerHTML`, destroying dashboard DOM
9. `frontend/app.js` (lines 33473380) — `DashboardStatsWidget` relies on `dashboard-stats-grid` element
10. `frontend/app.js` (lines 33813434) — `DashboardConflictsWidget` relies on `dashboard-conflicts-section` element
11. `backend/search.py` (lines 239425) — `InvertedIndex` class, `is_stale()`, `rebuild()`, `get_inverted_index()`
12. `backend/indexer.py` (lines 28, 330601, 634667, 770839) — `_index_generation` counter and all increment sites
13. `backend/watcher.py` (lines 191226) — Debounce: dispatches batched events after `debounce_seconds` (2s default)
14. `backend/main.py` (lines 376420) — `_on_vault_change()` calls `update_single_file` per event → each increments `_index_generation`
15. `backend/search.py` (lines 670700, 844875) — `advanced_search()` and `suggest_titles()` call `get_inverted_index()`
--- ---
## 1. Pydantic Models Section — `backend/main.py` ## Issue 1: Dashboard Layout — Sections Disappearing
**Location: lines 56284** — All Pydantic response/request models defined in a single block after imports and before SSE Manager. ### Root Cause: `showWelcome()` Rebuild Kills Stats & Conflicts Sections
### Key models with Field descriptions (lines 6069): **`showWelcome()`** (`frontend/app.js`, lines 55855665):
```python - It checks if `dashboard-home` or `dashboard-bookmarks-section` don't exist.
class VaultInfo(BaseModel): - If either is missing, it replaces `content-area.innerHTML` with **only two sections**: bookmarks and recent.
name: str = Field(description="Display name of the vault") - **Stats and Conflicts sections are NOT included** in this rebuilt HTML.
file_count: int = Field(description="Number of indexed files")
tag_count: int = Field(description="Number of unique tags")
type: str = Field(default="VAULT", description="Type of the vault mapping (VAULT or DIR)")
class BrowseItem(BaseModel): # line 72 ```js
name: str // line 5595 — only rebuilds if home or bookmarksSection is missing
path: str if (area && (!home || !bookmarksSection)) {
type: str = Field(description="'file' or 'directory'") area.innerHTML = `
``` <div id="dashboard-home" ...>
<div id="dashboard-bookmarks-section" class="dashboard-section">...</div>
### Models WITHOUT Field descriptions (need updating): <div id="dashboard-recent-section" class="dashboard-section">...</div>
- `FileContentResponse` — **lines 89100** </div>`;
- `FileRawResponse` — **lines 103107** // NOTE: No #dashboard-stats-section or #dashboard-conflicts-section!
- `FileSaveResponse` — **lines 110115**
- `FileDeleteResponse` — **lines 118122**
- `SearchResultItem` — **lines 125133**
- `SearchResponse`**lines 136143** (has Field on `total`, `offset`, `limit`)
- `TagsResponse` — **lines 146149**
- `TreeSearchResult` — **lines 152158**
- `TreeSearchResponse` — **lines 161165**
- `AdvancedSearchResultItem` — **lines 168176**
- `SearchFacets` — **lines 179182**
- `AdvancedSearchResponse`**lines 185193** (has Field on `query_time_ms`)
- `TitleSuggestion` — **lines 196200**
- `SuggestResponse` — **lines 203206**
- `TagSuggestion` — **lines 209212**
- `TagSuggestResponse` — **lines 215218**
- `GraphNode` — **lines 221228**
- `GraphEdge` — **lines 231236**
- `GraphResponse` — **lines 239244**
- `ReloadResponse` — **lines 247250**
- `HealthResponse` — **lines 253257**
- `DirectoryCreateRequest`**lines 260262** (has Field)
- `DirectoryCreateResponse` — **lines 265269**
- `DirectoryRenameRequest`**lines 272274** (has Field)
- `DirectoryRenameResponse` — **lines 277281**
- `DirectoryDeleteResponse` — **lines 284288**
- `FileCreateRequest`**lines 291293** (has Field)
- `FileCreateResponse` — **lines 296299**
- `FileRenameRequest`**lines 302304** (has Field)
- `FileRenameResponse` — **lines 307311**
---
## 2. Dashboard/Stats Endpoint — `backend/main.py`
### `/api/diagnostics` — **lines 25002547**
```python
@app.get("/api/diagnostics") # line 2500
async def api_diagnostics(current_user=Depends(require_admin)):
```
Returns index stats, inverted index stats, config, and search executor info. Requires admin auth.
### `/api/health` — **lines 10481059**
```python
@app.get("/api/health", response_model=HealthResponse) # line 1048
async def api_health():
```
Public health check (no auth). Returns status, version, vaults count, total_files.
---
## 3. SSE File Event Broadcasting — `backend/main.py`
**SSE Manager class: lines 349381** — `SSEManager` with `connect()`, `disconnect()`, and `broadcast()` methods.
**All `sse_manager.broadcast()` calls:**
| Line | Event Type | Context |
|------|-----------|---------|
| 413 | `index_updated` | File watcher callback (`_on_vault_change`) — partial index changes |
| 506 | `index_<event_type>` | Background indexing progress |
| 1252 | `file_deleted` | DELETE file endpoint |
| 1330 | `directory_created` | POST create directory |
| 1401 | `directory_renamed` | PATCH rename directory |
| 1462 | `directory_deleted` | DELETE directory |
| 1532 | `file_created` | POST create file |
| 1607 | `file_renamed` | PATCH rename file |
| 1949 | `index_reloaded` | Force reindex endpoint |
| 2114 | `vault_reloaded` | (likely during vault reload) |
| 2196 | `vault_added` | POST /api/vaults/add |
| 2215 | `vault_removed` | DELETE /api/vaults/remove |
**SSE endpoint: lines 21272169** — `GET /api/events` returns `StreamingResponse` with `text/event-stream`.
**Frontend SSE client: `frontend/app.js` lines 57736015**
- `IndexUpdateManager` (IIFE module)
- Listens for events: `connected`, `index_updated`, `index_reloaded`, `vault_added`, `vault_removed`, `index_start`, `index_progress`, `index_complete`
- Auto-reconnects with exponential backoff (1s → 30s max)
- The `_onIndexUpdated()` handler (line 5912) refreshes sidebar tree and tags when affected vault matches context
**⚠️ Note:** The frontend SSE client does NOT currently listen for `file_created`, `file_deleted`, `file_modified`, `file_renamed`, `directory_created`, `directory_renamed`, `directory_deleted` events — these are broadcast by the backend but not consumed by the frontend. The frontend only reacts to `index_updated` (which already includes all changes).
---
## 4. Configurations Endpoint — `backend/main.py`
### Config storage — **lines 24142458**
```python
_CONFIG_PATH = _BASE_DIR / "data" / "config.json" # line 2414
_DEFAULT_CONFIG = { # line 2416
"search_workers": 2,
"debounce_ms": 300,
"results_per_page": 50,
"min_query_length": 2,
"search_timeout_ms": 30000,
"max_content_size": 100000,
"snippet_context_chars": 120,
"max_snippet_highlights": 5,
"title_boost": 3.0,
"path_boost": 1.5,
"watcher_enabled": True,
"watcher_use_polling": False,
"watcher_polling_interval": 5.0,
"watcher_debounce": 2.0,
"tag_boost": 2.0,
"prefix_max_expansions": 50,
"recent_files_limit": 20,
} }
``` ```
### `GET /api/config`**line 2464**: Returns merged config (requires auth) **The original dashboard** (`frontend/index.html`, lines 360406) has 4 sections:
### `POST /api/config`**line 2470**: Updates config (requires admin), validates types against `_DEFAULT_CONFIG` 1. `#dashboard-stats-section`
2. `#dashboard-bookmarks-section`
3. `#dashboard-conflicts-section`
4. `#dashboard-recent-section`
--- **When does the dashboard get destroyed?**
- `renderFile()` at line 3302: `area.innerHTML = "";` — this wipes `content-area` including `#dashboard-home` (which is a child of `content-area`).
- After the file renders, `activate()` at line 7642 hides the dashboard: `dashboard.style.display = "none"` — but at this point the dashboard DOM elements have been **removed from the DOM**.
- Later, when `_showDashboard()` (line 7764) is called:
```js
_showDashboard() {
const area = document.getElementById("content-area");
area.innerHTML = ""; // clear file content
const dashboard = document.getElementById("dashboard-home");
if (dashboard) {
dashboard.style.display = ""; // show it
area.appendChild(dashboard); // move back into DOM
}
}
```
This works **only if** `dashboard-home` still exists in the document. But `renderFile()` called `area.innerHTML = ""` which destroyed it.
## 5. Dashboard-Home Element and Rendering — `frontend/app.js` - Next time `showWelcome()` is called (e.g., after `goHome()`), it detects `!home` → rebuilds with **only 2 sections**.
### Dashboard DOM structure → `frontend/index.html` lines 341392 **Why "Recently opened" stays visible when stats/bookmarks disappear:**
```html - The rebuilt HTML in `showWelcome()` **includes** `#dashboard-recent-section` and `#dashboard-bookmarks-section`, but **not** `#dashboard-stats-section` or `#dashboard-conflicts-section`.
<div id="dashboard-home" class="dashboard-home" role="region" aria-label="Tableau de bord"> - `DashboardStatsWidget.load()` (line 3348) looks for `document.getElementById("dashboard-stats-grid")` — if not found, it silently returns.
<div id="dashboard-bookmarks-section" class="dashboard-section">...</div> - `DashboardConflictsWidget.load()` (line 3383) looks for `document.getElementById("dashboard-conflicts-section")` — if not found, silently returns.
<div id="dashboard-recent-section" class="dashboard-section">...</div> - So stats and conflicts sections vanish, while bookmarks and recent survive the rebuild.
</div>
### How the Dashboard Visibility Toggle Works
- `TabManager.activate()` (line 7642): `dashboard.style.display = "none"` — hides dashboard when a file tab is shown.
- `TabManager._showDashboard()` (line 7764): `dashboard.style.display = ""` + appends to `content-area` — shows dashboard when all tabs close.
### CSS for Dashboard Sections
`frontend/style.css` (lines 47394742):
```css
.dashboard-section {
display: flex;
flex-direction: column;
}
``` ```
All `.dashboard-section` elements are just flex columns — there's no show/hide logic in CSS beyond what JavaScript sets via inline `style.display`.
### Dashboard regeneration fallback → `app.js` lines 54175482 (`showWelcome()`)
When `dashboard-home` or its children are missing, `showWelcome()` rebuilds the entire HTML structure inline.
### Dashboard Recent Widget → `app.js` lines 33443580 (`DashboardRecentWidget`)
- `load(vaultFilter)` — line 3347
- `render()` — line 3410
- `_createCard(file, index)` — line 3436
- `showLoading()` — line 3397
- `showEmpty()` — line 3526
### Dashboard Bookmarks Widget → `app.js` lines 35833660 (`DashboardBookmarkWidget`)
- `load(vaultFilter)` — line 3587
- `render()` — line 3613
- `_createCard(file, index)` — around line 3628
### Dashboard visibility toggling:
- Show: `app.js` line 75907593 — `dashboard.style.display = ""`
- Hide: `app.js` line 74627464 — `dashboard.style.display = "none"`
--- ---
## 6. Configurations/Settings Modal — `frontend/app.js` ## Issue 2: Click/Double-Click Handling
### Modal initialization → **lines 39063990** (`initConfigModal()`) ### Tree Click Handlers — Correctly Wired
Event binding for open/close, config fields, save buttons, reindex, reset, diary refresh, hidden files. In `_renderDirectoryInContainer` (`frontend/app.js`, lines 23542360):
```js
### Config modal opening → **line 3914**: fileItem.addEventListener("click", () => {
```javascript scrollTreeItemIntoView(fileItem, false);
openBtn.addEventListener("click", async () => { TabManager.openPreview(vaultName, item.path); // single click → preview
modal.classList.add("active"); closeMobileSidebar();
renderConfigFilters(); });
loadConfigFields(); // loads frontend+backend config fileItem.addEventListener("dblclick", (e) => {
loadDiagnostics(); // loads /api/diagnostics e.preventDefault();
loadAbout(); // loads /api/health TabManager.openPersistent(vaultName, item.path); // double click → persistent
await loadHiddenFilesSettings();
}); });
``` ```
Same pattern at lines 31323133 (sidebar recent), 49094910 and 50905091 (search results). **Wiring is correct.**
### Config field loading → **lines 40434070** (`loadConfigFields()`) ### openPreview() and openPersistent() — Properly Defined
Loads frontend config from localStorage and backend config from `GET /api/config`. - `openPreview()` (line 7497): Creates tab with `preview: true`, closes existing preview, activates.
- `openPersistent()` (line 7533): If a preview tab exists for this file, converts it to persistent (`previewTab.preview = false`). If already persistent, just activates. Otherwise creates new persistent tab via `this.open()`.
### Diagnostics rendering → **lines 41574207** (`loadDiagnostics()`, `renderDiagnostics()`) **Both are correctly implemented.**
Fetches `GET /api/diagnostics` and renders in `#config-diagnostics`.
### About section → **lines 42114250+** (`loadAbout()`) ### _renderTabs() — BUG: Missing `preview` CSS Class
Fetches `GET /api/health`. `_renderTabs()` at line 7792:
```js
el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : "");
```
**The `preview` class is NEVER added.** The CSS has rules for `.tab-item.preview`:
```css
/* style.css line 5450 */
.tab-item.preview .tab-name { font-style: italic; opacity: 0.75; }
.tab-item.preview { background: var(--bg-hover); }
```
But since the JS never adds `el.classList.add("preview")` when `tab.preview` is true, **preview tabs do not render in italic**.
### Config Modal HTML → `frontend/index.html` lines 395564 The fix should be:
All the config sections: search params, recent history, backend params, tag filtering, watcher, hidden files, diagnostics, about. ```js
el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : "") + (tab.preview ? " preview" : "");
```
Additionally, the `click` handler on tabs at line 7824 calls `this.activate(tab.id)` — activating a preview tab should work fine since `activate()` already properly handles loading file content.
--- ---
## 7. File Action Buttons — `frontend/app.js` ## Issue 3: Search Performance — "Rebuilding Inverted Index..."
### Button creation → **lines 32133260** ### InvertedIndex Architecture
All 6 action buttons created in `renderFile()`: `backend/search.py` (lines 239425):
- `InvertedIndex` is a singleton stored in module variable `_inverted_index` (line 415).
- `is_stale()` (line 270): returns `True` when `_indexer._index_generation != self._source_generation`.
- `rebuild()` (line 278): logs `"Rebuilding inverted index..."`, iterates over the **entire** `index` dict (all vaults, all files), rebuilds all internal structures from scratch. Sets `self._source_generation = _indexer._index_generation` at line 345.
- `get_inverted_index()` (line 418): **check-then-rebuild every call**:
```python
def get_inverted_index() -> InvertedIndex:
if _inverted_index.is_stale():
_inverted_index.rebuild()
return _inverted_index
```
| Button | Line | Icon | Action | ### What triggers `_index_generation` increments?
|--------|------|------|--------| `backend/indexer.py` — the counter increments at **7 sites**, every time the index mutates:
| Copy | 3213 | `copy` | Copies raw content to clipboard (fetches if needed) | | Line | Trigger |
| Source | 3231 | `code` | Toggles raw source view | |------|---------|
| Download | 3233 | `download` | Triggers file download via `/api/file/{vault}/download` | | 336 | `rebuild_index()` — full index rebuild on startup or manual reindex |
| Edit | 3244 | `edit` | Calls `openEditor(vault, path)` | | 375 | Per-vault processing within `rebuild_index()` (once per vault) |
| Pop-out | 3250 | `external-link` | Opens in new window via `/popout/{vault}/{path}` | | 468 | `reindex_vault()` — single vault reindex |
| TOC | 3256 | `list` | Toggles right sidebar TOC | | 601 | `_remove_file_from_structures()` — single file deletion |
| 667 | `_add_file_to_structures()` — single file addition |
| 794 | `remove_vault_from_index()` — vault removal |
| 839 | `add_vault_to_index()` — vault addition |
### Button assembly → **line 3300**: ### Frequent Rebuild Trigger: File Watcher Hot-Reload
```javascript `backend/main.py` (lines 376420) — `_on_vault_change()`:
area.appendChild(el("div", { class: "file-header" }, [..., - Each file watcher event (create/modify/delete/move) calls `update_single_file()`, `remove_single_file()`, or `handle_file_move()`.
el("div", { class: "file-actions" }, [ - These call `_remove_file_from_structures()` + `_add_file_to_structures()`, **each incrementing `_index_generation`**.
copyBtn, sourceBtn, downloadBtn, editBtn, openNewWindowBtn, tocBtn - The watcher debounces for 2 seconds (`backend/watcher.py` line 100), then dispatches **all batched events at once** (line 207210).
]) - Each event in the batch still increments `_index_generation` independently.
]));
```
--- **Result**: If you save 5 files simultaneously, `_index_generation` jumps by up to 5. The next search/autocomplete call triggers a full inverted index rebuild. On a vault with thousands of files, each rebuild is expensive.
## 8. Vault Settings — `backend/vault_settings.py` ### Which API calls trigger the rebuild?
`get_inverted_index()` is called from:
- `advanced_search()` at `search.py` line 684 (the TF-IDF advanced search)
- `suggest_titles()` at `search.py` line 862 (autocomplete)
- `main.py` line 2518 (stats/diagnostic endpoint)
**Full file: 138 lines** — Per-vault UI display preferences stored in `/app/data/vault_settings.json`. The basic `search()` function (line 430+) does **NOT** use `get_inverted_index()` — it scans the raw `index` dict directly. But `suggest_titles()` is called on every keystroke in the search bar.
### Exports used by `main.py`: ### Why frequent saves cause repeated rebuilds
```python 1. User saves file → watcher fires `modified` event → `_on_vault_change``update_single_file``_index_generation += 1` (or +2 for add+remove).
from backend.vault_settings import get_vault_setting, update_vault_setting, get_all_vault_settings, delete_vault_setting 2. User types in search bar → `suggest_titles()``get_inverted_index()``is_stale()` returns `True`**full rebuild**.
``` 3. User saves again → generation increments → next keystroke triggers another full rebuild.
(imported at line 46 of `main.py`)
### Key functions: This is the core performance issue: **single-file index mutations invalidate the entire inverted index**, and it gets rebuilt from scratch on the very next query.
- `get_vault_setting(vault_name)` — line 82 — returns settings dict or None
- `update_vault_setting(vault_name, settings)` — line 93 — partial update, auto-saves
- `get_all_vault_settings()` — line 125 — returns all vault settings
- `delete_vault_setting(vault_name)` — line 111 — removes vault settings
- Storage format: `{"vault_name": {"hideHiddenFiles": true/false}, ...}`
### Current usage in `main.py`:
- `get_vault_setting(vault_name)` used in browse (line 776) and graph (line 1981) endpoints for `hideHiddenFiles`
### Config Modal Hidden Files → `app.js` (search for `loadHiddenFilesSettings`)
Front-facing CRUD for per-vault `hideHiddenFiles` setting in the Configurations modal.
--- ---
## Architecture Summary ## Architecture Summary
### Data Flow for Dashboard
```
index.html (full dashboard DOM)
→ renderFile() destroys dashboard-home via innerHTML
→ activate() hides remaining dashboard ref
→ _showDashboard() tries to re-append dashboard-home
→ showWelcome() rebuilds from scratch (only 2/4 sections)
→ Stats/Conflicts widgets silently no-op (target elements missing)
``` ```
backend/main.py
├── Pydantic models (lines 56-284) — request/response schemas
├── SSEManager (lines 349-381) — broadcast to clients
├── _on_vault_change (lines 390-417) — watcher callback → index update + SSE broadcast
├── API endpoints:
│ ├── /api/health (1048) — public health
│ ├── /api/events (2127) — SSE stream
│ ├── /api/config GET/POST (2464/2470) — app config CRUD
│ ├── /api/diagnostics (2500) — index stats (admin only)
│ ├── /api/file/* — CRUD with SSE broadcasts on create/delete/rename
│ └── /api/directory/* — CRUD with SSE broadcasts
├── _CONFIG_PATH, _DEFAULT_CONFIG (2414-2458)
backend/vault_settings.py ### Data Flow for Search Performance
├── Per-vault settings (hideHiddenFiles) ```
├── JSON persistence in /app/data/vault_settings.json File change → watcher debounce (2s) → _on_vault_change()
→ update_single_file() → _remove + _add → _index_generation += 2
frontend/index.html → OR remove_single_file() → _index_generation += 1
├── #dashboard-home (341-392) — bookmarks + recent sections User types → suggest_titles() → get_inverted_index()
├── #config-modal (395-564) — full config UI → is_stale() checks generation counter
├── #editor-modal — CodeMirror editor → if counter changed: full rebuild of InvertedIndex (O(N) over all files)
├── #graph-modal — D3 graph view
└── #help-modal — user guide
frontend/app.js
├── AuthManager (~1532+)
├── DashboardRecentWidget (3344-3580)
├── DashboardBookmarkWidget (3583-3660)
├── initConfigModal (3906-3990)
├── loadConfigFields (4043-4070)
├── loadDiagnostics / renderDiagnostics (4157-4207)
├── showWelcome (5417-5482) — dashboard rebuild + render
├── IndexUpdateManager / SSE client (5773-6015)
├── renderFile (3075-3328) — file view with action buttons
└── TabManager (7307+) — multi-tab support
``` ```
--- ---
## Start Here ## Start Here
Open `frontend/app.js` at line 5585 (`showWelcome()`) — this is the dashboard rebuild function and the primary cause of Issue 1. The fix needs to:
1. Include all 4 dashboard sections in `showWelcome()`'s rebuild HTML
2. OR: store the dashboard DOM before `renderFile()` wipes it, and restore it in `_showDashboard()`
1. **For Pydantic Field descriptions**: Open `backend/main.py` at **line 89** (`FileContentResponse`) and add `Field(description=...)` to each field for models without descriptions through line 311. For Issue 2, open `frontend/app.js` at line 7792 (`_renderTabs`) — add the missing `preview` class.
2. **For dashboard stats widget**: Open `frontend/index.html` at **line 341** (`#dashboard-home`) and `frontend/app.js` at **line 5417** (`showWelcome()`). The dashboard currently has two sections (Bookmarks + Recently Opened). A new stats section would be added between those divs. For Issue 3, open `backend/search.py` at line 418 (`get_inverted_index`) and `backend/indexer.py` lines 600667 — consider whether single-file incremental updates to the inverted index are feasible, or whether the generation counter should be debounced/coalesced for file watcher batches.
3. **For webhooks on file events**: The SSE broadcasts happen in `backend/main.py` at lines 1252, 1532, 1607 (file events) and 1330, 1401, 1462 (directory events). The frontend SSE client in `app.js` at line 5773 doesn't listen for individual file events — it only handles `index_updated`. Webhook firing should be added alongside the `sse_manager.broadcast()` calls.
4. **For Configurations modal**: Open `frontend/index.html` at **line 395** (`#config-modal`) and `frontend/app.js` at **line 3906** (`initConfigModal()`).

View File

@ -3380,34 +3380,43 @@
// ── Dashboard Conflicts Widget ── // ── Dashboard Conflicts Widget ──
const DashboardConflictsWidget = { const DashboardConflictsWidget = {
async load() { async load() {
const section = document.getElementById("dashboard-conflicts-section"); const container = document.getElementById("dashboard-conflicts-container");
if (!section) return; if (!container) return;
try { try {
const data = await api("/api/conflicts"); const data = await api("/api/conflicts");
if (data.total === 0) { section.style.display = "none"; return; } if (data.total === 0) { container.innerHTML = ""; return; }
section.style.display = ""; this.render(data.conflicts, container);
document.getElementById("dashboard-conflicts-count").textContent = data.total; } catch (err) { container.innerHTML = ""; }
this.render(data.conflicts);
} catch (err) { section.style.display = "none"; }
}, },
render(conflicts) { render(conflicts, container) {
const grid = document.getElementById("dashboard-conflicts-grid"); container.innerHTML = `
if (!grid) return; <div class="dashboard-section">
grid.innerHTML = conflicts.map(c => ` <div class="dashboard-header">
<div class="conflict-card"> <div class="dashboard-title-row">
<div class="conflict-info"> <i data-lucide="alert-triangle" class="dashboard-icon" style="color:var(--accent-orange)"></i>
<span class="conflict-vault">${escapeHtml(c.vault)}</span> <h2>Conflits de synchronisation</h2>
<span class="conflict-name">${escapeHtml(c.conflict_path.split("/").pop())}</span> <span class="dashboard-badge" style="background:var(--accent-orange)">${conflicts.length}</span>
<span class="conflict-date">Conflit du ${c.conflict_date.replace(/(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})/, "$3/$2/$1 $4:$5")}</span> </div>
</div> </div>
<div class="conflict-actions"> <div class="dashboard-conflicts-grid">
<button class="conflict-btn keep-local" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder l'original</button> ${conflicts.map(c => `
<button class="conflict-btn keep-conflict" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder le conflit</button> <div class="conflict-card">
<div class="conflict-info">
<span class="conflict-vault">${escapeHtml(c.vault)}</span>
<span class="conflict-name">${escapeHtml(c.conflict_path.split("/").pop())}</span>
<span class="conflict-date">Conflit du ${c.conflict_date.replace(/(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})/, "$3/$2/$1 $4:$5")}</span>
</div>
<div class="conflict-actions">
<button class="conflict-btn keep-local" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder l'original</button>
<button class="conflict-btn keep-conflict" data-vault="${escapeHtml(c.vault)}" data-conflict="${escapeHtml(c.conflict_path)}" data-original="${escapeHtml(c.original_path)}">Garder le conflit</button>
</div>
</div>
`).join("")}
</div> </div>
</div> </div>`;
`).join(""); lucide.createIcons();
grid.querySelectorAll(".keep-local").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_local"))); container.querySelectorAll(".keep-local").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_local")));
grid.querySelectorAll(".keep-conflict").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_conflict"))); container.querySelectorAll(".keep-conflict").forEach(btn => btn.addEventListener("click", () => this._resolve(btn.dataset, "keep_conflict")));
}, },
async _resolve(d, action) { async _resolve(d, action) {
try { try {
@ -5584,23 +5593,37 @@
function showWelcome() { function showWelcome() {
hideProgressBar(); hideProgressBar();
// Ensure the dashboard container exists and has the correct structure (it might have been wiped by renderFile or be an old version) // Restore or rebuild the dashboard with tabbed sections
const area = document.getElementById("content-area"); const area = document.getElementById("content-area");
const home = document.getElementById("dashboard-home"); const home = document.getElementById("dashboard-home");
const bookmarksSection = document.getElementById("dashboard-bookmarks-section");
if (area && !home) {
if (area && (!home || !bookmarksSection)) {
area.innerHTML = ` area.innerHTML = `
<div id="dashboard-home" class="dashboard-home" role="region" aria-label="Tableau de bord"> <div id="dashboard-home" class="dashboard-home" role="region" aria-label="Tableau de bord">
<!-- Bookmarks Section --> <!-- Dashboard Tabs -->
<div id="dashboard-bookmarks-section" class="dashboard-section"> <div class="dashboard-tabs">
<div class="dashboard-header"> <button class="dashboard-tab active" data-tab="stats">
<div class="dashboard-title-row"> <i data-lucide="bar-chart-3" style="width:14px;height:14px"></i> Statistiques
<i data-lucide="bookmark" class="dashboard-icon" style="color:var(--accent-green)"></i> </button>
<h2>Bookmarks</h2> <button class="dashboard-tab" data-tab="bookmarks">
</div> <i data-lucide="bookmark" style="width:14px;height:14px"></i> Bookmarks
</button>
<button class="dashboard-tab" data-tab="recent">
<i data-lucide="clock" style="width:14px;height:14px"></i> Récents
</button>
</div>
<!-- Stats Panel -->
<div id="dashboard-panel-stats" class="dashboard-panel active">
<div id="dashboard-stats-grid" class="dashboard-stats-grid">
<div class="dashboard-stats-loading">Chargement...</div>
</div> </div>
<div id="dashboard-conflicts-container" style="margin-top:16px"></div>
</div>
<!-- Bookmarks Panel -->
<div id="dashboard-panel-bookmarks" class="dashboard-panel">
<div id="dashboard-bookmarks-grid" class="dashboard-recent-grid"></div> <div id="dashboard-bookmarks-grid" class="dashboard-recent-grid"></div>
<div id="dashboard-bookmarks-empty" class="dashboard-recent-empty"> <div id="dashboard-bookmarks-empty" class="dashboard-recent-empty">
<i data-lucide="pin"></i> <i data-lucide="pin"></i>
@ -5609,12 +5632,10 @@
</div> </div>
</div> </div>
<!-- Recently Opened Section --> <!-- Recent Panel -->
<div id="dashboard-recent-section" class="dashboard-section"> <div id="dashboard-panel-recent" class="dashboard-panel">
<div class="dashboard-header"> <div class="dashboard-header">
<div class="dashboard-title-row"> <div class="dashboard-title-row">
<i data-lucide="clock" class="dashboard-icon"></i>
<h2>Derniers fichiers ouverts</h2>
<span id="dashboard-count" class="dashboard-badge"></span> <span id="dashboard-count" class="dashboard-badge"></span>
</div> </div>
<div class="dashboard-actions"> <div class="dashboard-actions">
@ -5623,18 +5644,11 @@
</select> </select>
</div> </div>
</div> </div>
<div id="dashboard-recent-grid" class="dashboard-recent-grid"></div> <div id="dashboard-recent-grid" class="dashboard-recent-grid"></div>
<div id="dashboard-loading" class="dashboard-loading"> <div id="dashboard-loading" class="dashboard-loading">
<div class="skeleton-card"></div> <div class="skeleton-card"></div><div class="skeleton-card"></div><div class="skeleton-card"></div>
<div class="skeleton-card"></div> <div class="skeleton-card"></div><div class="skeleton-card"></div><div class="skeleton-card"></div>
<div class="skeleton-card"></div>
<div class="skeleton-card"></div>
<div class="skeleton-card"></div>
<div class="skeleton-card"></div>
</div> </div>
<div id="dashboard-recent-empty" class="dashboard-recent-empty hidden"> <div id="dashboard-recent-empty" class="dashboard-recent-empty hidden">
<i data-lucide="inbox"></i> <i data-lucide="inbox"></i>
<span>Aucun fichier récent</span> <span>Aucun fichier récent</span>
@ -5642,15 +5656,36 @@
</div> </div>
</div> </div>
</div>`; </div>`;
// Re-initialize widgets which might need to bind events to new elements // Wire dashboard tab switching
document.querySelectorAll(".dashboard-tab").forEach(tab => {
tab.addEventListener("click", function() {
document.querySelectorAll(".dashboard-tab").forEach(t => t.classList.remove("active"));
document.querySelectorAll(".dashboard-panel").forEach(p => p.classList.remove("active"));
this.classList.add("active");
const panel = document.getElementById("dashboard-panel-" + this.dataset.tab);
if (panel) panel.classList.add("active");
});
});
// Re-initialize widgets
if (typeof DashboardRecentWidget !== "undefined") { if (typeof DashboardRecentWidget !== "undefined") {
DashboardRecentWidget.init(); DashboardRecentWidget.init();
} }
safeCreateIcons(); safeCreateIcons();
} else if (home) {
// Dashboard already exists, just show it
home.style.display = "";
// Activate default tab
document.querySelectorAll(".dashboard-tab").forEach(t => t.classList.remove("active"));
document.querySelectorAll(".dashboard-panel").forEach(p => p.classList.remove("active"));
const defaultTab = document.querySelector('.dashboard-tab[data-tab="stats"]');
const defaultPanel = document.getElementById("dashboard-panel-stats");
if (defaultTab) defaultTab.classList.add("active");
if (defaultPanel) defaultPanel.classList.add("active");
} }
// Show the dashboard widgets // Load all widgets (they handle missing elements gracefully)
if (typeof DashboardStatsWidget !== "undefined") { if (typeof DashboardStatsWidget !== "undefined") {
DashboardStatsWidget.load(); DashboardStatsWidget.load();
} }
@ -7763,12 +7798,22 @@
_showDashboard() { _showDashboard() {
const area = document.getElementById("content-area"); const area = document.getElementById("content-area");
area.innerHTML = ""; // Save dashboard DOM before clearing (it may have been removed from DOM by renderFile)
const dashboard = document.getElementById("dashboard-home"); let dashboard = document.getElementById("dashboard-home");
if (dashboard) { if (!dashboard) {
dashboard.style.display = ""; // Dashboard was destroyed — rebuild via showWelcome
area.appendChild(dashboard); area.innerHTML = "";
showWelcome();
return;
} }
area.innerHTML = "";
dashboard.style.display = "";
area.appendChild(dashboard);
// Refresh widgets after restoring
if (typeof DashboardStatsWidget !== "undefined") DashboardStatsWidget.load();
if (typeof DashboardConflictsWidget !== "undefined") DashboardConflictsWidget.load();
if (typeof DashboardRecentWidget !== "undefined") DashboardRecentWidget.load(selectedContextVault);
if (typeof DashboardBookmarkWidget !== "undefined") DashboardBookmarkWidget.load(selectedContextVault);
if (history.pushState) { if (history.pushState) {
history.pushState(null, "", "#"); history.pushState(null, "", "#");
} }
@ -7789,7 +7834,7 @@
this._tabs.forEach((tab, idx) => { this._tabs.forEach((tab, idx) => {
const el = document.createElement("div"); const el = document.createElement("div");
el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : ""); el.className = "tab-item" + (tab.id === this._activeTabId ? " active" : "") + (tab.preview ? " preview" : "");
el.draggable = true; el.draggable = true;
el.dataset.tabId = tab.id; el.dataset.tabId = tab.id;
el.dataset.index = idx; el.dataset.index = idx;

View File

@ -5744,3 +5744,37 @@ body.popup-mode .content-area {
cursor: pointer; cursor: pointer;
font-size: 0.85rem; font-size: 0.85rem;
} }
/* ── Dashboard Tab Navigation ── */
.dashboard-tabs {
display: flex;
gap: 2px;
margin-bottom: 16px;
border-bottom: 2px solid var(--border);
}
.dashboard-tab {
padding: 8px 16px;
border: none;
background: none;
color: var(--text-muted);
cursor: pointer;
font-size: 0.85rem;
font-weight: 500;
display: flex;
align-items: center;
gap: 6px;
border-bottom: 2px solid transparent;
margin-bottom: -2px;
transition: color 0.15s, border-color 0.15s;
}
.dashboard-tab:hover { color: var(--text-primary); }
.dashboard-tab.active {
color: var(--accent);
border-bottom-color: var(--accent);
}
.dashboard-panel {
display: none;
}
.dashboard-panel.active {
display: block;
}