feat: add Meilisearch backend integration with Docker Compose setup and Excalidraw support

This commit is contained in:
2025-10-14 10:41:15 -04:00
parent b359e8ab8e
commit 32a9998b40
71 changed files with 11544 additions and 435 deletions
+27
View File
@@ -0,0 +1,27 @@
import dotenv from 'dotenv';
// Load .env first and override existing env vars in DEV to ensure a single source of truth
// In Docker, environment variables will be injected at runtime and still be read here.
dotenv.config({ override: true });
// Normalize host (prefer explicit MEILI_HOST)
export const MEILI_HOST = process.env.MEILI_HOST || 'http://127.0.0.1:7700';
// Single API key resolution
export const MEILI_API_KEY = process.env.MEILI_API_KEY || process.env.MEILI_MASTER_KEY || undefined;
// Vault path resolution (default to ./vault)
export const VAULT_PATH = process.env.VAULT_PATH || './vault';
// Server port
export const PORT = Number(process.env.PORT || 4000);
export function debugPrintConfig(prefix = 'Config') {
console.log(`[${prefix}]`, {
MEILI_HOST,
MEILI_KEY_LENGTH: MEILI_API_KEY?.length,
MEILI_KEY_PREVIEW: MEILI_API_KEY ? `${MEILI_API_KEY.slice(0, 8)}...` : 'none',
VAULT_PATH,
PORT,
});
}
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env node
/**
* Obsidian Excalidraw format utilities
* Handles parsing and serialization of .excalidraw.md files in Obsidian format
*/
import LZString from 'lz-string';
/**
* Extract front matter from markdown content
* @param {string} md - Markdown content
* @returns {string|null} - Front matter content (including --- delimiters) or null
*/
export function extractFrontMatter(md) {
const match = md.match(/^---\s*\n([\s\S]*?)\n---/);
return match ? match[0] : null;
}
/**
* Parse Obsidian Excalidraw markdown format
* Extracts compressed-json block and decompresses using LZ-String
* @param {string} md - Markdown content
* @returns {object|null} - Excalidraw scene data or null if parsing fails
*/
export function parseObsidianExcalidrawMd(md) {
if (!md || typeof md !== 'string') return null;
// Try to extract compressed-json block
const compressedMatch = md.match(/```\s*compressed-json\s*\n([\s\S]*?)\n```/i);
if (compressedMatch && compressedMatch[1]) {
try {
// Remove whitespace from base64 data
const compressed = compressedMatch[1].replace(/\s+/g, '');
// Decompress using LZ-String
const decompressed = LZString.decompressFromBase64(compressed);
if (!decompressed) {
console.warn('[Excalidraw] LZ-String decompression returned null');
return null;
}
// Parse JSON
const data = JSON.parse(decompressed);
return data;
} catch (error) {
console.error('[Excalidraw] Failed to parse compressed-json:', error.message);
return null;
}
}
// Fallback: try to extract plain json block
const jsonMatch = md.match(/```\s*(?:excalidraw|json)\s*\n([\s\S]*?)\n```/i);
if (jsonMatch && jsonMatch[1]) {
try {
const data = JSON.parse(jsonMatch[1].trim());
return data;
} catch (error) {
console.error('[Excalidraw] Failed to parse json block:', error.message);
return null;
}
}
return null;
}
/**
* Parse flat JSON format (legacy ObsiViewer format)
* @param {string} text - JSON text
* @returns {object|null} - Excalidraw scene data or null if parsing fails
*/
export function parseFlatJson(text) {
if (!text || typeof text !== 'string') return null;
try {
const data = JSON.parse(text);
// Validate it has the expected structure
if (data && typeof data === 'object' && Array.isArray(data.elements)) {
return data;
}
return null;
} catch (error) {
return null;
}
}
/**
* Convert Excalidraw scene to Obsidian markdown format
* @param {object} data - Excalidraw scene data
* @param {string|null} existingFrontMatter - Existing front matter to preserve
* @returns {string} - Markdown content in Obsidian format
*/
export function toObsidianExcalidrawMd(data, existingFrontMatter = null) {
// Normalize scene data
const scene = {
elements: Array.isArray(data?.elements) ? data.elements : [],
appState: (data && typeof data.appState === 'object') ? data.appState : {},
files: (data && typeof data.files === 'object') ? data.files : {}
};
// Serialize to JSON
const json = JSON.stringify(scene);
// Compress using LZ-String
const compressed = LZString.compressToBase64(json);
// Build front matter: merge existing with required keys
const ensureFrontMatter = (fmRaw) => {
const wrap = (inner) => `---\n${inner}\n---`;
if (!fmRaw || typeof fmRaw !== 'string' || !/^---[\s\S]*?---\s*$/m.test(fmRaw)) {
return wrap(`excalidraw-plugin: parsed\ntags: [excalidraw]`);
}
// Strip leading/trailing ---
const inner = fmRaw.replace(/^---\s*\n?/, '').replace(/\n?---\s*$/, '');
const lines = inner.split(/\r?\n/);
let hasPlugin = false;
let tagsLineIdx = -1;
for (let i = 0; i < lines.length; i++) {
const l = lines[i].trim();
if (/^excalidraw-plugin\s*:/i.test(l)) hasPlugin = true;
if (/^tags\s*:/i.test(l)) tagsLineIdx = i;
}
if (!hasPlugin) {
lines.push('excalidraw-plugin: parsed');
}
if (tagsLineIdx === -1) {
lines.push('tags: [excalidraw]');
} else {
// Ensure 'excalidraw' tag present; naive merge without YAML parse
const orig = lines[tagsLineIdx];
if (!/excalidraw\b/.test(orig)) {
const mArr = orig.match(/^tags\s*:\s*\[(.*)\]\s*$/i);
if (mArr) {
const inside = mArr[1].trim();
const updatedInside = inside ? `${inside}, excalidraw` : 'excalidraw';
lines[tagsLineIdx] = `tags: [${updatedInside}]`;
} else if (/^tags\s*:\s*$/i.test(orig)) {
lines[tagsLineIdx] = 'tags: [excalidraw]';
} else {
// Fallback: append a separate tags line
lines.push('tags: [excalidraw]');
}
}
}
return wrap(lines.join('\n'));
};
const frontMatter = ensureFrontMatter(existingFrontMatter?.trim() || null);
// Banner text (Obsidian standard)
const banner = `==⚠ Switch to EXCALIDRAW VIEW in the MORE OPTIONS menu of this document. ⚠==
You can decompress Drawing data with the command palette: 'Decompress current Excalidraw file'. For more info check in plugin settings under 'Saving'`;
// Construct full markdown
return `${frontMatter}
${banner}
# Excalidraw Data
## Text Elements
%%
## Drawing
\`\`\`compressed-json
${compressed}
\`\`\`
%%`;
}
/**
* Parse Excalidraw content from any supported format
* Tries Obsidian MD format first, then falls back to flat JSON
* @param {string} text - File content
* @returns {object|null} - Excalidraw scene data or null
*/
export function parseExcalidrawAny(text) {
// Try Obsidian format first
let data = parseObsidianExcalidrawMd(text);
if (data) return data;
// Fallback to flat JSON
data = parseFlatJson(text);
if (data) return data;
return null;
}
/**
* Validate Excalidraw scene structure
* @param {any} data - Data to validate
* @returns {boolean} - True if valid
*/
export function isValidExcalidrawScene(data) {
if (!data || typeof data !== 'object') return false;
if (!Array.isArray(data.elements)) return false;
return true;
}
+204
View File
@@ -0,0 +1,204 @@
#!/usr/bin/env node
/**
* Unit tests for Excalidraw Obsidian format utilities
* Run with: node server/excalidraw-obsidian.test.mjs
*/
import assert from 'assert';
import {
parseObsidianExcalidrawMd,
parseFlatJson,
toObsidianExcalidrawMd,
extractFrontMatter,
parseExcalidrawAny,
isValidExcalidrawScene
} from './excalidraw-obsidian.mjs';
let passed = 0;
let failed = 0;
function test(name, fn) {
try {
fn();
console.log(`✅ ${name}`);
passed++;
} catch (error) {
console.error(`❌ ${name}`);
console.error(` ${error.message}`);
failed++;
}
}
// Sample data
const sampleScene = {
elements: [
{ id: '1', type: 'rectangle', x: 100, y: 100, width: 200, height: 100 }
],
appState: { viewBackgroundColor: '#ffffff' },
files: {}
};
const sampleFrontMatter = `---
excalidraw-plugin: parsed
tags: [excalidraw]
---`;
console.log('🧪 Running Excalidraw Obsidian Format Tests\n');
// Test: extractFrontMatter
test('extractFrontMatter - extracts front matter', () => {
const md = `---
excalidraw-plugin: parsed
---
Content here`;
const result = extractFrontMatter(md);
assert.ok(result);
assert.ok(result.includes('excalidraw-plugin'));
});
test('extractFrontMatter - returns null when no front matter', () => {
const md = 'Just some content';
const result = extractFrontMatter(md);
assert.strictEqual(result, null);
});
// Test: toObsidianExcalidrawMd
test('toObsidianExcalidrawMd - creates valid markdown', () => {
const md = toObsidianExcalidrawMd(sampleScene);
assert.ok(md.includes('---'));
assert.ok(md.includes('excalidraw-plugin'));
assert.ok(md.includes('```compressed-json'));
assert.ok(md.includes('# Excalidraw Data'));
});
test('toObsidianExcalidrawMd - preserves existing front matter', () => {
const customFrontMatter = `---
excalidraw-plugin: parsed
tags: [custom, test]
---`;
const md = toObsidianExcalidrawMd(sampleScene, customFrontMatter);
assert.ok(md.includes('tags: [custom, test]'));
});
// Test: parseObsidianExcalidrawMd round-trip
test('parseObsidianExcalidrawMd - round-trip conversion', () => {
const md = toObsidianExcalidrawMd(sampleScene);
const parsed = parseObsidianExcalidrawMd(md);
assert.ok(parsed);
assert.ok(Array.isArray(parsed.elements));
assert.strictEqual(parsed.elements.length, 1);
assert.strictEqual(parsed.elements[0].type, 'rectangle');
});
// Test: parseFlatJson
test('parseFlatJson - parses valid JSON', () => {
const json = JSON.stringify(sampleScene);
const parsed = parseFlatJson(json);
assert.ok(parsed);
assert.ok(Array.isArray(parsed.elements));
assert.strictEqual(parsed.elements.length, 1);
});
test('parseFlatJson - returns null for invalid JSON', () => {
const parsed = parseFlatJson('not valid json');
assert.strictEqual(parsed, null);
});
test('parseFlatJson - returns null for JSON without elements', () => {
const parsed = parseFlatJson('{"foo": "bar"}');
assert.strictEqual(parsed, null);
});
// Test: parseExcalidrawAny
test('parseExcalidrawAny - parses Obsidian format', () => {
const md = toObsidianExcalidrawMd(sampleScene);
const parsed = parseExcalidrawAny(md);
assert.ok(parsed);
assert.ok(Array.isArray(parsed.elements));
});
test('parseExcalidrawAny - parses flat JSON', () => {
const json = JSON.stringify(sampleScene);
const parsed = parseExcalidrawAny(json);
assert.ok(parsed);
assert.ok(Array.isArray(parsed.elements));
});
test('parseExcalidrawAny - returns null for invalid content', () => {
const parsed = parseExcalidrawAny('invalid content');
assert.strictEqual(parsed, null);
});
// Test: isValidExcalidrawScene
test('isValidExcalidrawScene - validates correct structure', () => {
assert.strictEqual(isValidExcalidrawScene(sampleScene), true);
});
test('isValidExcalidrawScene - rejects invalid structure', () => {
assert.strictEqual(isValidExcalidrawScene({}), false);
assert.strictEqual(isValidExcalidrawScene({ elements: 'not an array' }), false);
assert.strictEqual(isValidExcalidrawScene(null), false);
});
// Test: Empty scene handling
test('toObsidianExcalidrawMd - handles empty scene', () => {
const emptyScene = { elements: [], appState: {}, files: {} };
const md = toObsidianExcalidrawMd(emptyScene);
const parsed = parseObsidianExcalidrawMd(md);
assert.ok(parsed);
assert.ok(Array.isArray(parsed.elements));
assert.strictEqual(parsed.elements.length, 0);
});
// Test: Large scene handling
test('toObsidianExcalidrawMd - handles large scene', () => {
const largeScene = {
elements: Array.from({ length: 100 }, (_, i) => ({
id: `${i}`,
type: 'rectangle',
x: i * 10,
y: i * 10,
width: 50,
height: 50
})),
appState: {},
files: {}
};
const md = toObsidianExcalidrawMd(largeScene);
const parsed = parseObsidianExcalidrawMd(md);
assert.ok(parsed);
assert.strictEqual(parsed.elements.length, 100);
});
// Test: Special characters in scene
test('toObsidianExcalidrawMd - handles special characters', () => {
const sceneWithSpecialChars = {
elements: [
{ id: '1', type: 'text', text: 'Hello "World" & <test>' }
],
appState: {},
files: {}
};
const md = toObsidianExcalidrawMd(sceneWithSpecialChars);
const parsed = parseObsidianExcalidrawMd(md);
assert.ok(parsed);
assert.strictEqual(parsed.elements[0].text, 'Hello "World" & <test>');
});
// Summary
console.log('\n━'.repeat(30));
console.log(`✅ Passed: ${passed}`);
console.log(`❌ Failed: ${failed}`);
console.log('━'.repeat(30));
process.exit(failed > 0 ? 1 : 0);
+439 -7
View File
@@ -4,17 +4,30 @@ import express from 'express';
import path from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
import zlib from 'zlib';
import chokidar from 'chokidar';
import { meiliClient, vaultIndexName, ensureIndexSettings } from './meilisearch.client.mjs';
import { fullReindex, upsertFile, deleteFile } from './meilisearch-indexer.mjs';
import { mapObsidianQueryToMeili, buildSearchParams } from './search.mapping.mjs';
import { PORT as CFG_PORT, VAULT_PATH as CFG_VAULT_PATH, debugPrintConfig } from './config.mjs';
import { z } from 'zod';
import {
parseExcalidrawAny,
toObsidianExcalidrawMd,
extractFrontMatter,
isValidExcalidrawScene
} from './excalidraw-obsidian.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const app = express();
const PORT = process.env.PORT || 4000;
const PORT = CFG_PORT;
const rootDir = path.resolve(__dirname, '..');
const distDir = path.join(rootDir, 'dist');
const vaultDir = path.join(rootDir, 'vault');
// Centralized vault directory
const vaultDir = path.isAbsolute(CFG_VAULT_PATH) ? CFG_VAULT_PATH : path.join(rootDir, CFG_VAULT_PATH);
const vaultEventClients = new Set();
@@ -28,6 +41,7 @@ const registerVaultEventClient = (res) => {
}, 20000);
const client = { res, heartbeat };
// moved scanVaultDrawings to top-level
vaultEventClients.add(client);
return client;
};
@@ -154,6 +168,44 @@ const loadVaultNotes = (vaultPath) => {
return notes;
};
// Scan vault for .excalidraw.md files and return FileMetadata-like entries
const scanVaultDrawings = (vaultPath) => {
const items = [];
const walk = (currentDir) => {
let entries = [];
try {
entries = fs.readdirSync(currentDir, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
const entryPath = path.join(currentDir, entry.name);
if (entry.isDirectory()) {
walk(entryPath);
continue;
}
if (!entry.isFile()) continue;
const lower = entry.name.toLowerCase();
if (!lower.endsWith('.excalidraw.md')) continue;
try {
const stats = fs.statSync(entryPath);
const relPath = path.relative(vaultPath, entryPath).replace(/\\/g, '/');
const id = slugifyPath(relPath.replace(/\.excalidraw(?:\.md)?$/i, ''));
const title = path.basename(relPath).replace(/\.excalidraw(?:\.md)?$/i, '');
items.push({
id,
title,
path: relPath,
createdAt: new Date(stats.birthtimeMs ? stats.birthtimeMs : stats.ctimeMs).toISOString(),
updatedAt: new Date(stats.mtimeMs).toISOString(),
});
} catch {}
}
};
walk(vaultPath);
return items;
};
const buildFileMetadata = (notes) =>
notes.map((note) => ({
id: note.id,
@@ -198,6 +250,26 @@ watchedVaultEvents.forEach((eventName) => {
});
});
// Integrate Meilisearch with Chokidar for incremental updates
vaultWatcher.on('add', (filePath) => {
if (filePath.toLowerCase().endsWith('.md')) {
upsertFile(filePath).catch(err => console.error('[Meili] Upsert on add failed:', err));
}
});
vaultWatcher.on('change', (filePath) => {
if (filePath.toLowerCase().endsWith('.md')) {
upsertFile(filePath).catch(err => console.error('[Meili] Upsert on change failed:', err));
}
});
vaultWatcher.on('unlink', (filePath) => {
if (filePath.toLowerCase().endsWith('.md')) {
const relativePath = path.relative(vaultDir, filePath).replace(/\\/g, '/');
deleteFile(relativePath).catch(err => console.error('[Meili] Delete failed:', err));
}
});
vaultWatcher.on('ready', () => {
broadcastVaultEvent({
event: 'ready',
@@ -356,13 +428,84 @@ app.get('/api/vault', (req, res) => {
}
});
app.get('/api/files/metadata', (req, res) => {
// Fast file list from Meilisearch (id, title, path, createdAt, updatedAt)
app.get('/api/files/list', async (req, res) => {
try {
const notes = loadVaultNotes(vaultDir);
res.json(buildFileMetadata(notes));
const client = meiliClient();
const indexUid = vaultIndexName(vaultDir);
const index = await ensureIndexSettings(client, indexUid);
const result = await index.search('', {
limit: 10000,
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
});
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
id: hit.id,
title: hit.title,
path: hit.path,
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
})) : [];
res.json(items);
} catch (error) {
console.error('Failed to load file metadata:', error);
res.status(500).json({ error: 'Unable to load file metadata.' });
console.error('Failed to list files via Meilisearch, falling back to FS:', error);
try {
const notes = loadVaultNotes(vaultDir);
res.json(buildFileMetadata(notes));
} catch (err2) {
console.error('FS fallback failed:', err2);
res.status(500).json({ error: 'Unable to list files.' });
}
}
});
app.get('/api/files/metadata', async (req, res) => {
try {
// Prefer Meilisearch for fast metadata
const client = meiliClient();
const indexUid = vaultIndexName(vaultDir);
const index = await ensureIndexSettings(client, indexUid);
const result = await index.search('', {
limit: 10000,
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
});
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
id: hit.id,
title: hit.title,
path: hit.path,
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
})) : [];
// Merge .excalidraw files discovered via FS
const drawings = scanVaultDrawings(vaultDir);
const byPath = new Map(items.map(it => [String(it.path).toLowerCase(), it]));
for (const d of drawings) {
const key = String(d.path).toLowerCase();
if (!byPath.has(key)) {
byPath.set(key, d);
}
}
res.json(Array.from(byPath.values()));
} catch (error) {
console.error('Failed to load file metadata via Meilisearch, falling back to FS:', error);
try {
const notes = loadVaultNotes(vaultDir);
const base = buildFileMetadata(notes);
const drawings = scanVaultDrawings(vaultDir);
const byPath = new Map(base.map(it => [String(it.path).toLowerCase(), it]));
for (const d of drawings) {
const key = String(d.path).toLowerCase();
if (!byPath.has(key)) byPath.set(key, d);
}
res.json(Array.from(byPath.values()));
} catch (err2) {
console.error('FS fallback failed:', err2);
res.status(500).json({ error: 'Unable to load file metadata.' });
}
}
});
@@ -500,6 +643,239 @@ app.post('/api/logs', (req, res) => {
res.status(202).json({ status: 'queued' });
});
// --- Files API (supports .excalidraw.md (Markdown-wrapped JSON), .excalidraw, .json and binary sidecars) ---
// Helpers
const sanitizeRelPath = (rel) => String(rel || '').replace(/\\/g, '/').replace(/^\/+/, '');
const resolveVaultPath = (rel) => {
const clean = sanitizeRelPath(rel);
const abs = path.resolve(vaultDir, clean);
if (!abs.startsWith(path.resolve(vaultDir))) {
throw Object.assign(new Error('Invalid path'), { status: 400 });
}
return abs;
};
const excalidrawSceneSchema = z.object({
elements: z.array(z.any()),
appState: z.record(z.any()).optional(),
files: z.record(z.any()).optional(),
}).passthrough();
// Helper to determine content type
function guessContentType(filePath) {
const lower = filePath.toLowerCase();
if (lower.endsWith('.md')) return 'text/markdown; charset=utf-8';
if (lower.endsWith('.json')) return 'application/json; charset=utf-8';
return 'application/octet-stream';
}
// GET file content (supports .excalidraw.md, .excalidraw, .json, .md)
app.get('/api/files', (req, res) => {
try {
const pathParam = req.query.path;
if (!pathParam || typeof pathParam !== 'string') {
return res.status(400).json({ error: 'Missing path query parameter' });
}
const rel = decodeURIComponent(pathParam);
const abs = resolveVaultPath(rel);
if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) {
return res.status(404).json({ error: 'File not found' });
}
const base = path.basename(abs).toLowerCase();
const ext = path.extname(abs).toLowerCase();
const isExcalidrawMd = base.endsWith('.excalidraw.md');
const isExcalidraw = ext === '.excalidraw' || ext === '.json' || isExcalidrawMd;
if (!isExcalidraw && ext !== '.md') {
return res.status(415).json({ error: 'Unsupported file type' });
}
const content = fs.readFileSync(abs, 'utf-8');
// For Excalidraw files, parse and return JSON
if (isExcalidraw) {
const data = parseExcalidrawAny(content);
if (!data || !isValidExcalidrawScene(data)) {
return res.status(400).json({ error: 'Invalid Excalidraw format' });
}
// Normalize scene structure
const normalized = {
elements: Array.isArray(data.elements) ? data.elements : [],
appState: (data && typeof data.appState === 'object') ? data.appState : {},
files: (data && typeof data.files === 'object') ? data.files : {}
};
const rev = calculateSimpleHash(content);
res.setHeader('ETag', rev);
res.setHeader('Content-Type', 'application/json; charset=utf-8');
return res.send(JSON.stringify(normalized));
}
// For regular markdown, return as-is
const rev = calculateSimpleHash(content);
res.setHeader('ETag', rev);
res.setHeader('Content-Type', guessContentType(abs));
return res.send(content);
} catch (error) {
const code = typeof error?.status === 'number' ? error.status : 500;
console.error('GET /api/files error:', error);
res.status(code).json({ error: 'Internal server error' });
}
});
// PUT file content with If-Match check and size limit (10MB)
app.put('/api/files', express.json({ limit: '10mb' }), express.text({ limit: '10mb', type: 'text/markdown' }), (req, res) => {
try {
const pathParam = req.query.path;
if (!pathParam || typeof pathParam !== 'string') {
return res.status(400).json({ error: 'Missing path query parameter' });
}
const rel = decodeURIComponent(pathParam);
const abs = resolveVaultPath(rel);
const dir = path.dirname(abs);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
const contentType = (req.headers['content-type'] || '').split(';')[0];
const base = path.basename(abs).toLowerCase();
const isExcalidrawMd = base.endsWith('.excalidraw.md');
let finalContent;
let existingFrontMatter = null;
console.log('[PUT /api/files] path=%s contentType=%s isExcalidrawMd=%s', rel, contentType, isExcalidrawMd);
// Extract existing front matter if file exists
if (fs.existsSync(abs) && isExcalidrawMd) {
try {
const existing = fs.readFileSync(abs, 'utf-8');
existingFrontMatter = extractFrontMatter(existing);
} catch {}
}
// Handle JSON payload (Excalidraw scene)
if (contentType === 'application/json') {
const body = req.body;
const parsed = excalidrawSceneSchema.safeParse(body);
if (!parsed.success) {
console.warn('[PUT /api/files] invalid scene schema', parsed.error?.issues?.slice(0,3));
return res.status(400).json({
error: 'Invalid Excalidraw scene',
issues: parsed.error.issues?.slice(0, 5)
});
}
// Convert to Obsidian format if target is .excalidraw.md
if (isExcalidrawMd) {
finalContent = toObsidianExcalidrawMd(parsed.data, existingFrontMatter);
} else {
// Plain JSON for .excalidraw or .json files
finalContent = JSON.stringify(parsed.data, null, 2);
}
}
// Handle text/markdown payload (already formatted)
else if (contentType === 'text/markdown') {
finalContent = typeof req.body === 'string' ? req.body : String(req.body);
}
else {
console.warn('[PUT /api/files] unsupported content-type', contentType);
return res.status(400).json({ error: 'Unsupported content type' });
}
// Check size limit
if (Buffer.byteLength(finalContent, 'utf-8') > 10 * 1024 * 1024) {
console.warn('[PUT /api/files] payload too large path=%s size=%d', rel, Buffer.byteLength(finalContent, 'utf-8'));
return res.status(413).json({ error: 'Payload too large' });
}
// Check for conflicts with If-Match
const hasExisting = fs.existsSync(abs);
const ifMatch = req.headers['if-match'];
if (hasExisting && ifMatch) {
const current = fs.readFileSync(abs, 'utf-8');
const currentRev = calculateSimpleHash(current);
if (ifMatch !== currentRev) {
console.warn('[PUT /api/files] conflict path=%s ifMatch=%s current=%s', rel, ifMatch, currentRev);
return res.status(409).json({ error: 'Conflict detected' });
}
}
// Atomic write with backup
const temp = abs + '.tmp';
const backup = abs + '.bak';
try {
if (hasExisting) fs.copyFileSync(abs, backup);
fs.writeFileSync(temp, finalContent, 'utf-8');
fs.renameSync(temp, abs);
console.log('[PUT /api/files] wrote file path=%s bytes=%d', rel, Buffer.byteLength(finalContent, 'utf-8'));
} catch (e) {
if (fs.existsSync(temp)) try { fs.unlinkSync(temp); } catch {}
if (hasExisting && fs.existsSync(backup)) try { fs.copyFileSync(backup, abs); } catch {}
console.error('[PUT /api/files] write error path=%s', rel, e);
throw e;
}
const rev = calculateSimpleHash(finalContent);
res.setHeader('ETag', rev);
res.json({ rev });
} catch (error) {
const code = typeof error?.status === 'number' ? error.status : 500;
console.error('PUT /api/files error:', error);
res.status(code).json({ error: 'Internal server error' });
}
});
// PUT binary sidecar (e.g., PNG/SVG)
app.put('/api/files/blob', (req, res) => {
try {
const pathParam = req.query.path;
if (!pathParam || typeof pathParam !== 'string') {
return res.status(400).json({ error: 'Missing path query parameter' });
}
const rel = decodeURIComponent(pathParam);
const abs = resolveVaultPath(rel);
const dir = path.dirname(abs);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
// Collect raw body
const chunks = [];
req.on('data', (chunk) => {
chunks.push(chunk);
const size = chunks.reduce((a, b) => a + b.length, 0);
if (size > 10 * 1024 * 1024) { // 10MB limit
req.destroy();
}
});
req.on('end', () => {
const buf = Buffer.concat(chunks);
// Basic allowlist
const ext = path.extname(abs).toLowerCase();
if (!['.png', '.svg'].includes(ext)) {
return res.status(415).json({ error: 'unsupported_media_type' });
}
fs.writeFileSync(abs, buf);
res.json({ ok: true });
});
req.on('error', (err) => {
console.error('Blob upload error:', err);
res.status(500).json({ error: 'internal_error' });
});
} catch (error) {
const code = typeof error?.status === 'number' ? error.status : 500;
console.error('PUT /api/files/blob error:', error);
res.status(code).json({ error: 'internal_error' });
}
});
function ensureBookmarksStorage() {
const obsidianDir = path.join(vaultDir, '.obsidian');
if (!fs.existsSync(obsidianDir)) {
@@ -688,6 +1064,62 @@ function calculateSimpleHash(content) {
return Math.abs(hash).toString(36) + '-' + content.length;
}
// Meilisearch API endpoints
app.get('/api/search', async (req, res) => {
try {
const { q = '', limit = '20', offset = '0', sort, highlight = 'true' } = req.query;
// Parse Obsidian-style query to Meilisearch format
const parsedQuery = mapObsidianQueryToMeili(String(q));
// Build search parameters
const searchParams = buildSearchParams(parsedQuery, {
limit: Number(limit),
offset: Number(offset),
sort,
highlight: highlight === 'true'
});
// Execute search
const client = meiliClient();
const indexUid = vaultIndexName(vaultDir);
const index = await ensureIndexSettings(client, indexUid);
const result = await index.search(searchParams.q, searchParams);
// Return results
res.json({
hits: result.hits,
estimatedTotalHits: result.estimatedTotalHits,
facetDistribution: result.facetDistribution,
processingTimeMs: result.processingTimeMs,
query: q
});
} catch (error) {
console.error('[Meili] Search failed:', error);
res.status(500).json({
error: 'search_failed',
message: error.message
});
}
});
app.post('/api/reindex', async (_req, res) => {
try {
console.log('[Meili] Manual reindex triggered');
const result = await fullReindex();
res.json({
ok: true,
...result
});
} catch (error) {
console.error('[Meili] Reindex failed:', error);
res.status(500).json({
error: 'reindex_failed',
message: error.message
});
}
});
// Servir l'index.html pour toutes les routes (SPA)
const sendIndex = (req, res) => {
const indexPath = path.join(distDir, 'index.html');
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env node
import fs from 'node:fs/promises';
import fssync from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import fg from 'fast-glob';
import matter from 'gray-matter';
import removeMd from 'remove-markdown';
import { meiliClient, vaultIndexName, ensureIndexSettings } from './meilisearch.client.mjs';
import { VAULT_PATH as CFG_VAULT_PATH, MEILI_HOST as CFG_MEILI_HOST, MEILI_API_KEY as CFG_MEILI_KEY } from './config.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const VAULT_PATH = path.isAbsolute(CFG_VAULT_PATH)
? CFG_VAULT_PATH
: path.resolve(__dirname, '..', CFG_VAULT_PATH);
console.log('[Meili Indexer] Environment check:', {
MEILI_MASTER_KEY: CFG_MEILI_KEY ? `${String(CFG_MEILI_KEY).substring(0, 8)}... (${String(CFG_MEILI_KEY).length} chars)` : 'NOT SET',
MEILI_API_KEY: CFG_MEILI_KEY ? `${String(CFG_MEILI_KEY).substring(0, 8)}...` : 'NOT SET',
MEILI_HOST: CFG_MEILI_HOST,
VAULT_PATH: VAULT_PATH
});
/**
* Convert timestamp to year/month for faceting
*/
function toYMD(timestampMs) {
const dt = new Date(timestampMs);
return {
year: dt.getFullYear(),
month: dt.getMonth() + 1
};
}
/**
* Extract all parent directory paths for a file
* Example: "Projects/Angular/App.md" -> ["Projects", "Projects/Angular"]
*/
function parentDirs(relativePath) {
const parts = relativePath.split(/[\\/]/);
const acc = [];
for (let i = 0; i < parts.length - 1; i++) {
acc.push(parts.slice(0, i + 1).join('/'));
}
return acc;
}
/**
* Extract headings from markdown content
*/
function extractHeadings(content) {
const headingRegex = /^#+\s+(.+)$/gm;
const headings = [];
let match;
while ((match = headingRegex.exec(content)) !== null) {
headings.push(match[1].trim());
}
return headings.slice(0, 200); // Limit to 200 headings
}
/**
* Build a searchable document from a markdown file
*/
export async function buildDocumentFromFile(absPath) {
const rel = path.relative(VAULT_PATH, absPath).replaceAll('\\', '/');
const file = path.basename(rel);
const raw = await fs.readFile(absPath, 'utf8');
// Parse frontmatter with gray-matter
const { data: fm, content } = matter(raw);
// Remove markdown formatting and limit content size
const text = removeMd(content).slice(0, 200_000); // 200KB safety limit
// Extract metadata
const title = fm.title ?? path.parse(file).name;
const tags = Array.isArray(fm.tags)
? fm.tags.map(String)
: (fm.tags ? [String(fm.tags)] : []);
const headings = extractHeadings(content);
// Get file stats
const stat = await fs.stat(absPath);
const { year, month } = toYMD(stat.mtimeMs);
// Meilisearch requires alphanumeric IDs (a-z A-Z 0-9 - _)
// Replace dots, slashes, and other chars with underscores
const safeId = rel.replace(/[^a-zA-Z0-9_-]/g, '_');
return {
id: safeId,
path: rel,
file,
title,
tags,
properties: fm ?? {},
content: text,
headings,
createdAt: stat.birthtimeMs || stat.ctimeMs,
updatedAt: stat.mtimeMs,
year,
month,
parentDirs: parentDirs(rel),
excerpt: text.slice(0, 500)
};
}
/**
* Perform a full reindex of all markdown files in the vault
*/
export async function fullReindex() {
console.log('[Meili] Starting full reindex...');
const startTime = Date.now();
const client = meiliClient();
const indexUid = vaultIndexName(VAULT_PATH);
const index = await ensureIndexSettings(client, indexUid);
// Find all markdown files
const entries = await fg(['**/*.md'], {
cwd: VAULT_PATH,
dot: false,
onlyFiles: true,
absolute: true
});
console.log(`[Meili] Found ${entries.length} markdown files`);
// Process in batches to avoid memory issues
const batchSize = 750;
let totalIndexed = 0;
for (let i = 0; i < entries.length; i += batchSize) {
const chunk = entries.slice(i, i + batchSize);
const docs = await Promise.all(
chunk.map(async (file) => {
try {
return await buildDocumentFromFile(file);
} catch (err) {
console.error(`[Meili] Failed to process ${file}:`, err.message);
return null;
}
})
);
const validDocs = docs.filter(Boolean);
if (validDocs.length > 0) {
const task = await index.addDocuments(validDocs);
console.log(`[Meili] Batch ${Math.floor(i / batchSize) + 1}: Queued ${validDocs.length} documents (task ${task.taskUid})`);
totalIndexed += validDocs.length;
}
}
const elapsed = Date.now() - startTime;
console.log(`[Meili] Reindex complete: ${totalIndexed} documents indexed in ${elapsed}ms`);
return {
indexed: true,
count: totalIndexed,
elapsedMs: elapsed
};
}
/**
* Upsert a single file (add or update)
*/
export async function upsertFile(relOrAbs) {
const abs = path.isAbsolute(relOrAbs) ? relOrAbs : path.join(VAULT_PATH, relOrAbs);
if (!fssync.existsSync(abs)) {
console.warn(`[Meili] File not found for upsert: ${abs}`);
return;
}
try {
const client = meiliClient();
const indexUid = vaultIndexName(VAULT_PATH);
const index = await ensureIndexSettings(client, indexUid);
const doc = await buildDocumentFromFile(abs);
await index.addDocuments([doc]);
console.log(`[Meili] Upserted: ${doc.id}`);
} catch (err) {
console.error(`[Meili] Failed to upsert ${abs}:`, err.message);
}
}
/**
* Delete a file from the index
*/
export async function deleteFile(relPath) {
try {
const client = meiliClient();
const indexUid = vaultIndexName(VAULT_PATH);
const index = await ensureIndexSettings(client, indexUid);
await index.deleteDocuments([relPath]);
console.log(`[Meili] Deleted: ${relPath}`);
} catch (err) {
console.error(`[Meili] Failed to delete ${relPath}:`, err.message);
}
}
// CLI execution: node server/meilisearch-indexer.mjs
if (process.argv[1] === __filename) {
fullReindex()
.then((result) => {
console.log('[Meili] Reindex done:', result);
process.exit(0);
})
.catch((err) => {
console.error('[Meili] Reindex failed:', err);
process.exit(1);
});
}
+114
View File
@@ -0,0 +1,114 @@
import { MeiliSearch } from 'meilisearch';
import { MEILI_HOST, MEILI_API_KEY, VAULT_PATH as CFG_VAULT_PATH } from './config.mjs';
const MEILI_KEY = MEILI_API_KEY;
console.log('[Meili] Config:', {
host: MEILI_HOST,
keyLength: MEILI_KEY?.length,
keyPreview: MEILI_KEY ? `${MEILI_KEY.slice(0, 8)}...` : 'none'
});
if (!MEILI_KEY) {
console.warn('[Meili] No API key provided; running without authentication. Set MEILI_API_KEY or MEILI_MASTER_KEY when securing Meilisearch.');
}
/**
* Create and return a Meilisearch client instance
*/
export function meiliClient() {
const options = MEILI_KEY ? { host: MEILI_HOST, apiKey: MEILI_KEY } : { host: MEILI_HOST };
console.log('[Meili] Creating client with options:', {
host: options.host,
hasApiKey: !!options.apiKey,
apiKeyLength: options.apiKey?.length,
apiKeyHex: options.apiKey ? Buffer.from(options.apiKey).toString('hex') : 'none'
});
return new MeiliSearch(options);
}
/**
* Generate index name from vault path (supports multi-vault)
*/
export function vaultIndexName(vaultPath = CFG_VAULT_PATH ?? './vault') {
// Simple safe name generation; adjust to hash if paths can collide
const safe = vaultPath.replace(/[^a-z0-9]+/gi, '_').toLowerCase();
return `notes_${safe}`;
}
/**
* Ensure index exists and configure settings for optimal search
*/
export async function ensureIndexSettings(client, indexUid) {
// Create the index if it doesn't exist, then fetch it.
// Some Meilisearch operations are task-based; we wait for completion to avoid 404s.
let index;
try {
index = await client.getIndex(indexUid);
} catch (e) {
const task = await client.createIndex(indexUid, { primaryKey: 'id' });
if (task?.taskUid !== undefined) {
await client.waitForTask(task.taskUid, { timeOutMs: 30_000 });
}
index = await client.getIndex(indexUid);
}
// Configure searchable, filterable, sortable, and faceted attributes
const settingsTask = await index.updateSettings({
searchableAttributes: [
'title',
'content',
'file',
'path',
'tags',
'properties.*',
'headings'
],
displayedAttributes: [
'id',
'title',
'path',
'file',
'tags',
'properties',
'updatedAt',
'createdAt',
'excerpt'
],
filterableAttributes: [
'tags',
'file',
'path',
'parentDirs',
'properties.*',
'year',
'month'
],
sortableAttributes: [
'updatedAt',
'createdAt',
'title',
'file',
'path'
],
faceting: {
maxValuesPerFacet: 1000
},
typoTolerance: {
enabled: true,
minWordSizeForTypos: {
oneTypo: 3,
twoTypos: 6
}
},
pagination: {
maxTotalHits: 10000
}
});
if (settingsTask?.taskUid !== undefined) {
await client.waitForTask(settingsTask.taskUid, { timeOutMs: 30_000 });
}
console.log(`[Meili] Index "${indexUid}" configured successfully`);
return index;
}
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env node
/**
* Migration script for converting old flat JSON Excalidraw files to Obsidian format
* Usage: node server/migrate-excalidraw.mjs [--dry-run] [--vault-path=./vault]
*/
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { parseFlatJson, toObsidianExcalidrawMd, isValidExcalidrawScene } from './excalidraw-obsidian.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Parse command line arguments
const args = process.argv.slice(2);
const dryRun = args.includes('--dry-run');
const vaultPathArg = args.find(arg => arg.startsWith('--vault-path='));
const vaultPath = vaultPathArg
? path.resolve(vaultPathArg.split('=')[1])
: path.resolve(__dirname, '..', 'vault');
console.log('🔄 Excalidraw Migration Tool');
console.log('━'.repeat(50));
console.log(`Vault path: ${vaultPath}`);
console.log(`Mode: ${dryRun ? 'DRY RUN (no changes)' : 'LIVE (will modify files)'}`);
console.log('━'.repeat(50));
console.log();
if (!fs.existsSync(vaultPath)) {
console.error(`❌ Vault directory not found: ${vaultPath}`);
process.exit(1);
}
let filesScanned = 0;
let filesConverted = 0;
let filesSkipped = 0;
let filesErrored = 0;
/**
* Recursively scan directory for Excalidraw files
*/
function scanDirectory(dir) {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
// Skip hidden directories
if (entry.name.startsWith('.')) continue;
scanDirectory(fullPath);
continue;
}
if (!entry.isFile()) continue;
const lower = entry.name.toLowerCase();
// Look for .excalidraw or .json files (but not .excalidraw.md)
if (lower.endsWith('.excalidraw.md')) {
// Already in Obsidian format, skip
filesSkipped++;
continue;
}
if (!lower.endsWith('.excalidraw') && !lower.endsWith('.json')) {
continue;
}
filesScanned++;
processFile(fullPath);
}
}
/**
* Process a single file
*/
function processFile(filePath) {
const relativePath = path.relative(vaultPath, filePath);
try {
const content = fs.readFileSync(filePath, 'utf-8');
// Try to parse as flat JSON
const scene = parseFlatJson(content);
if (!scene || !isValidExcalidrawScene(scene)) {
console.log(`⏭️ Skipped (not valid Excalidraw): ${relativePath}`);
filesSkipped++;
return;
}
// Convert to Obsidian format
const obsidianMd = toObsidianExcalidrawMd(scene);
// Determine new file path
const dir = path.dirname(filePath);
const baseName = path.basename(filePath, path.extname(filePath));
const newPath = path.join(dir, `${baseName}.excalidraw.md`);
if (dryRun) {
console.log(`✅ Would convert: ${relativePath} → ${path.basename(newPath)}`);
filesConverted++;
return;
}
// Create backup of original
const backupPath = filePath + '.bak';
fs.copyFileSync(filePath, backupPath);
// Write new file
fs.writeFileSync(newPath, obsidianMd, 'utf-8');
// Remove original if new file is different
if (newPath !== filePath) {
fs.unlinkSync(filePath);
}
console.log(`✅ Converted: ${relativePath} → ${path.basename(newPath)}`);
filesConverted++;
} catch (error) {
console.error(`❌ Error processing ${relativePath}:`, error.message);
filesErrored++;
}
}
// Run migration
try {
scanDirectory(vaultPath);
console.log();
console.log('━'.repeat(50));
console.log('📊 Migration Summary');
console.log('━'.repeat(50));
console.log(`Files scanned: ${filesScanned}`);
console.log(`Files converted: ${filesConverted}`);
console.log(`Files skipped: ${filesSkipped}`);
console.log(`Files errored: ${filesErrored}`);
console.log('━'.repeat(50));
if (dryRun) {
console.log();
console.log('💡 This was a dry run. Run without --dry-run to apply changes.');
} else if (filesConverted > 0) {
console.log();
console.log('✅ Migration complete! Backup files (.bak) were created.');
}
process.exit(filesErrored > 0 ? 1 : 0);
} catch (error) {
console.error('❌ Migration failed:', error);
process.exit(1);
}
+151
View File
@@ -0,0 +1,151 @@
/**
* Map Obsidian-style search queries to Meilisearch format
* Supports: tag:, path:, file: operators with free text search
*/
export function mapObsidianQueryToMeili(qRaw) {
const tokens = String(qRaw).trim().split(/\s+/);
const filters = [];
const meiliQ = [];
let restrict = null;
let rangeStart = null;
let rangeEnd = null;
const parseDate = (s) => {
if (!s) return null;
// Accept YYYY-MM-DD
const m = String(s).match(/^(\d{4})-(\d{2})-(\d{2})$/);
if (!m) return null;
const d = new Date(`${m[1]}-${m[2]}-${m[3]}T00:00:00Z`);
if (Number.isNaN(d.getTime())) return null;
return d;
};
for (const token of tokens) {
// tag: operator - filter by exact tag match
if (token.startsWith('tag:')) {
const value = token.slice(4).replace(/^["']|["']$/g, '');
if (value) {
// Remove leading # if present
const cleanTag = value.startsWith('#') ? value.substring(1) : value;
filters.push(`tags = "${cleanTag}"`);
}
}
// path: operator - filter by parent directory path
// We use parentDirs array to simulate startsWith behavior
else if (token.startsWith('path:')) {
const value = token.slice(5).replace(/^["']|["']$/g, '');
if (value) {
// Normalize path separators
const normalizedPath = value.replace(/\\/g, '/');
filters.push(`parentDirs = "${normalizedPath}"`);
}
}
// file: operator - restrict search to file field
else if (token.startsWith('file:')) {
const value = token.slice(5).replace(/^["']|["']$/g, '');
if (value) {
// Restrict search to file field only
restrict = ['file'];
meiliQ.push(value);
}
}
// year: operator - facet filter
else if (token.startsWith('year:')) {
const value = token.slice(5).replace(/^["']|["']$/g, '');
const n = Number(value);
if (!Number.isNaN(n)) {
filters.push(`year = ${n}`);
}
}
// month: operator - facet filter (1-12)
else if (token.startsWith('month:')) {
const value = token.slice(6).replace(/^["']|["']$/g, '');
const n = Number(value);
if (!Number.isNaN(n)) {
filters.push(`month = ${n}`);
}
}
// date:YYYY-MM-DD (single day)
else if (token.startsWith('date:')) {
const value = token.slice(5);
const d = parseDate(value);
if (d) {
const start = new Date(d); start.setUTCHours(0,0,0,0);
const end = new Date(d); end.setUTCHours(23,59,59,999);
const startMs = start.getTime();
const endMs = end.getTime();
filters.push(`((createdAt >= ${startMs} AND createdAt <= ${endMs}) OR (updatedAt >= ${startMs} AND updatedAt <= ${endMs}))`);
}
}
// from:/to: YYYY-MM-DD
else if (token.startsWith('from:')) {
const d = parseDate(token.slice(5));
if (d) {
const s = new Date(d); s.setUTCHours(0,0,0,0);
rangeStart = s.getTime();
}
}
else if (token.startsWith('to:')) {
const d = parseDate(token.slice(3));
if (d) {
const e = new Date(d); e.setUTCHours(23,59,59,999);
rangeEnd = e.getTime();
}
}
// Regular text token
else if (token) {
meiliQ.push(token);
}
}
// If we captured a from/to range, add a combined date filter
if (rangeStart !== null || rangeEnd !== null) {
const startMs = rangeStart ?? 0;
const endMs = rangeEnd ?? 8640000000000000; // large future
filters.push(`((createdAt >= ${startMs} AND createdAt <= ${endMs}) OR (updatedAt >= ${startMs} AND updatedAt <= ${endMs}))`);
}
return {
meiliQ: meiliQ.join(' ').trim(),
filters,
restrict
};
}
/**
* Build Meilisearch search parameters from parsed query
*/
export function buildSearchParams(parsedQuery, options = {}) {
const {
limit = 20,
offset = 0,
sort,
highlight = true
} = options;
const params = {
q: parsedQuery.meiliQ,
limit: Number(limit),
offset: Number(offset),
filter: parsedQuery.filters.length ? parsedQuery.filters.join(' AND ') : undefined,
facets: ['tags', 'parentDirs', 'year', 'month'],
attributesToHighlight: highlight ? ['title', 'content'] : [],
highlightPreTag: '<mark>',
highlightPostTag: '</mark>',
attributesToCrop: ['content'],
cropLength: 80,
cropMarker: '…',
attributesToSearchOn: parsedQuery.restrict?.length ? parsedQuery.restrict : undefined,
sort: sort ? [String(sort)] : undefined,
showMatchesPosition: false
};
// Remove undefined values
Object.keys(params).forEach(key => {
if (params[key] === undefined) {
delete params[key];
}
});
return params;
}