feat: #78 Excalidraw finitions + #67 push + #68 health-detailed + #77 desktop jumplist
CI / lint (push) Successful in 52s
CI / security (push) Successful in 36s
CI / test (push) Successful in 1m6s
CI / build (push) Successful in 1m15s
CI / e2e (push) Failing after 12m40s
Desktop Build / build-windows (push) Canceled after 0s
Desktop Build / build-linux (push) Canceled after 0s
CI / lint (push) Successful in 52s
CI / security (push) Successful in 36s
CI / test (push) Successful in 1m6s
CI / build (push) Successful in 1m15s
CI / e2e (push) Failing after 12m40s
Desktop Build / build-windows (push) Canceled after 0s
Desktop Build / build-linux (push) Canceled after 0s
#78 Excalidraw — finitions B5/C8/F2/F3 - backend/indexer.py: port Python pur de lz-string decompressFromBase64 (bitsPerChar=6, resetValue=32) validé contre 4 fixtures JS truth (texte accentué, edge cases) — remplace le stub base64 non-fonctionnel - B5 indexation: .excalidraw.md (format plugin Obsidian) maintenant décompressé côté Python → texte indexé pour recherche TF-IDF - 7 nouveaux tests B5 dans tests/test_excalidraw.py (13/13 total) - F2: excalidraw-viewer.test.mjs enregistré dans CI (lint job) - F3: tests/e2e/excalidraw.spec.js (9 specs — ouverture .excalidraw/.excalidraw.md, création via modale/menu contextuel, toolbar, thème, pop-out, recherche) - Fixtures test_vault/diagram.excalidraw + diagram.excalidraw.md #67 Push notifications (Web Push API + VAPID) - backend/push.py: router + VAPID keys + send_push_notification (pywebpush>=2.3.0) - frontend/js/push.js: subscription UI + service worker integration - tests/test_push.py: 10 tests (subscribe/list/unsubscribe/vapid key) - requirements.txt + sw.js + locales push strings #68 Health check enrichi - backend/main.py: GET /api/health/detailed (admin) — memory/cpu/disk/backups/index/SSE connections - backend/indexer.py: _last_full_index_ts tracking - tests/conftest.py: admin_client fixture - tests/test_api_main.py: 3 nouveaux tests health detailed #77 Desktop jumplist - desktop/src/jumplist.rs: Windows jumplist integration - Cargo.toml/lock + capabilities + main.rs - frontend/js/desktop.js: Tauri bridge (isTauriEnv, invoke, getSystemTheme, syncSystemTheme, shouldShowWizard, crash banner) - tests/frontend/desktop.test.mjs: 21 tests JSDOM (détection, invoke degradation, theme gating, wizard, crash banner) - tests/frontend/unit.test.mjs: desktop.js whitelisted (standalone global reader) # Frontend & CI - frontend/sw.js: réécriture complète (precache + push event handlers + offline) - frontend/index.html: section push dans settings + about repositionné - frontend/js/app.js: initDesktopIntegration + initPush - CI .gitea/workflows/ci.yml: excalidraw-viewer.test.mjs ajouté au lint job All checks: ruff clean, 552 backend tests pass, 9 frontend unit, 5 excalidraw-viewer, 21 desktop, 9 pane-manager, validate-imports 33 modules.
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -28,6 +29,9 @@ _async_index_lock: asyncio.Lock | None = None # initialized lazily
|
||||
# (e.g. the inverted index in search.py) can detect staleness.
|
||||
_index_generation: int = 0
|
||||
|
||||
# Timestamp of last full index rebuild (ISO format, empty if never built)
|
||||
_last_full_index_ts: str = ""
|
||||
|
||||
# Hook for incremental inverted index updates: called as (action, vault, path, file_info)
|
||||
_on_index_change: Callable[..., None] | None = None
|
||||
|
||||
@@ -197,6 +201,178 @@ def _extract_title(post: frontmatter.Post, filepath: Path) -> str:
|
||||
return str(title)
|
||||
|
||||
|
||||
_EXCALIDRAW_TEXT_FIELDS = ("text", "originalText", "label", "title")
|
||||
|
||||
|
||||
def extract_excalidraw_text_from_elements(elements: list[dict[str, Any]]) -> str:
|
||||
"""Concatenate all user-visible text from an Excalidraw elements list.
|
||||
|
||||
Iterates diagram elements and collects the text-bearing fields
|
||||
(``text`` for text elements, ``label``/``title`` for bound shapes,
|
||||
``originalText`` as the stable source of a text element). Non-text
|
||||
elements and geometry-only shapes contribute nothing. This gives the
|
||||
TF-IDF search engine human-readable content instead of raw JSON.
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
for el in elements:
|
||||
if not isinstance(el, dict):
|
||||
continue
|
||||
collected = set()
|
||||
for field in _EXCALIDRAW_TEXT_FIELDS:
|
||||
val = el.get(field)
|
||||
if isinstance(val, str) and val.strip():
|
||||
collected.add(val.strip())
|
||||
if collected:
|
||||
chunks.append(" ".join(sorted(collected)))
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
_EXCALIDRAW_B64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
|
||||
|
||||
|
||||
def _decompress_excalidraw(compressed: str) -> dict[str, Any] | None:
|
||||
"""Decompress the Obsidian ``compressed-json`` block of a .excalidraw.md file.
|
||||
|
||||
The Obsidian Excalidraw plugin stores scene data as an ``lz-string``
|
||||
``compressToBase64`` payload (LZ-based compression, alphabet = standard
|
||||
base64). We port the reference ``lz-string`` ``_decompress`` algorithm
|
||||
(bitsPerChar=6/base64 key, resetValue=32) in pure Python so indexation
|
||||
can recover the diagram text without the JS client. The port is validated
|
||||
against multiple JS-truth fixtures in ``test_excalidraw.py`` including
|
||||
accented text and edge cases. Returns the parsed scene dict, or None if
|
||||
the payload is not valid base64-lz or does not parse as JSON.
|
||||
|
||||
Only ``decompress`` is needed for indexation (read side); compression
|
||||
stays on the JS client.
|
||||
"""
|
||||
if not compressed:
|
||||
return None
|
||||
|
||||
length = len(compressed)
|
||||
|
||||
def _get_char_value(index: int) -> int:
|
||||
# Mirror JS: input.charAt(index), 0-indexed.
|
||||
ch = compressed[index] if 0 <= index < length else "="
|
||||
pos = _EXCALIDRAW_B64_ALPHABET.find(ch)
|
||||
return 0 if pos == -1 else pos
|
||||
|
||||
# ── Build a bit reader over the base64 characters ────────────────────
|
||||
# JS decompressFromBase64 calls _decompress(length, 32, getNextValue).
|
||||
# state = current 6-bit value + position mask + next char index.
|
||||
value = _get_char_value(0)
|
||||
position = 32 # resetValue
|
||||
index = 1
|
||||
|
||||
def _read_bit() -> int:
|
||||
nonlocal value, position, index
|
||||
resb = value & position
|
||||
position >>= 1
|
||||
if position == 0:
|
||||
position = 32
|
||||
value = _get_char_value(index)
|
||||
index += 1
|
||||
return 1 if resb > 0 else 0
|
||||
|
||||
def _read_bits(n: int) -> int:
|
||||
out = 0
|
||||
for i in range(n):
|
||||
out |= _read_bit() << i
|
||||
return out
|
||||
|
||||
# ── Decompressor state ────────────────────────────────────────────────
|
||||
dictionary: list[Any] = [0, 1, 2] # values 0,1,2 as in JS
|
||||
enlarge_in = 4
|
||||
dict_size = 4
|
||||
num_bits = 3
|
||||
|
||||
first = _read_bits(2)
|
||||
if first == 0:
|
||||
c = chr(_read_bits(8))
|
||||
elif first == 1:
|
||||
c = chr(_read_bits(16))
|
||||
elif first == 2:
|
||||
return None # empty stream
|
||||
else:
|
||||
return None
|
||||
|
||||
dictionary.append(c)
|
||||
w: str = c
|
||||
result = [c]
|
||||
|
||||
while True:
|
||||
if index > length:
|
||||
return None
|
||||
|
||||
c = _read_bits(num_bits)
|
||||
if c == 0:
|
||||
dictionary.append(chr(_read_bits(8)))
|
||||
dict_size += 1
|
||||
c = dict_size - 1
|
||||
enlarge_in -= 1
|
||||
elif c == 1:
|
||||
dictionary.append(chr(_read_bits(16)))
|
||||
dict_size += 1
|
||||
c = dict_size - 1
|
||||
enlarge_in -= 1
|
||||
elif c == 2:
|
||||
break # end of stream
|
||||
|
||||
if enlarge_in == 0:
|
||||
enlarge_in = 1 << num_bits
|
||||
num_bits += 1
|
||||
|
||||
if c < len(dictionary) and dictionary[c]:
|
||||
entry = dictionary[c]
|
||||
elif c == dict_size:
|
||||
entry = w + w[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
result.append(entry)
|
||||
dictionary.append(w + entry[0])
|
||||
dict_size += 1
|
||||
enlarge_in -= 1
|
||||
|
||||
w = entry
|
||||
if enlarge_in == 0:
|
||||
enlarge_in = 1 << num_bits
|
||||
num_bits += 1
|
||||
|
||||
text = "".join(result)
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def extract_excalidraw_indexable(raw: str) -> str:
|
||||
"""Return indexable text content for a raw .excalidraw / .excalidraw.md file.
|
||||
|
||||
Supports both the pure JSON format (``type: "excalidraw"``) and the
|
||||
Obsidian ``compressed-json`` block. Falls back to ``""`` when neither
|
||||
format can be parsed so the file is still indexed (title only).
|
||||
"""
|
||||
# .excalidraw.md — Obsidian plugin embeds a compressed-json block
|
||||
if "excalidraw-plugin:" in raw:
|
||||
m = re.search(r"```compressed-json\n(.*?)\n```", raw, re.DOTALL)
|
||||
if m:
|
||||
parsed = _decompress_excalidraw(m.group(1).strip())
|
||||
if parsed:
|
||||
return extract_excalidraw_text_from_elements(parsed.get("elements") or [])
|
||||
return ""
|
||||
# Pure .excalidraw JSON
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
return ""
|
||||
if not isinstance(data, dict) or data.get("type") != "excalidraw":
|
||||
return ""
|
||||
return extract_excalidraw_text_from_elements(data.get("elements") or [])
|
||||
|
||||
|
||||
def parse_markdown_file(raw: str) -> frontmatter.Post:
|
||||
"""Parse markdown frontmatter, falling back to plain content if YAML is invalid.
|
||||
|
||||
@@ -292,6 +468,12 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
|
||||
title = pdf_meta.get("title") or fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
tags: list[str] = []
|
||||
elif ext == ".excalidraw" or fpath.name.lower().endswith(".excalidraw.md"):
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
raw = extract_excalidraw_indexable(raw)
|
||||
tags: list[str] = []
|
||||
title = fpath.stem.replace(".excalidraw", "").replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
else:
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
tags: list[str] = []
|
||||
@@ -411,6 +593,10 @@ async def build_index(progress_callback=None) -> None:
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# Record timestamp of full index rebuild
|
||||
global _last_full_index_ts
|
||||
_last_full_index_ts = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Build attachment index
|
||||
from backend.attachment_indexer import build_attachment_index
|
||||
await build_attachment_index(vault_config)
|
||||
@@ -556,6 +742,11 @@ def _index_single_file_sync(vault_name: str, vault_path: str, file_path: str, va
|
||||
pdf_meta = extract_pdf_metadata(fpath)
|
||||
title = pdf_meta.get("title") or title
|
||||
content_preview = raw[:200].strip()
|
||||
elif ext == ".excalidraw" or fpath.name.lower().endswith(".excalidraw.md"):
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
raw = extract_excalidraw_indexable(raw)
|
||||
title = fpath.stem.replace(".excalidraw", "").replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
else:
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
content_preview = raw[:200].strip()
|
||||
|
||||
Reference in New Issue
Block a user