feat: add Meilisearch backend integration with Docker Compose setup and Excalidraw support
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
// Load .env first and override existing env vars in DEV to ensure a single source of truth
|
||||
// In Docker, environment variables will be injected at runtime and still be read here.
|
||||
dotenv.config({ override: true });
|
||||
|
||||
// Normalize host (prefer explicit MEILI_HOST)
|
||||
export const MEILI_HOST = process.env.MEILI_HOST || 'http://127.0.0.1:7700';
|
||||
|
||||
// Single API key resolution
|
||||
export const MEILI_API_KEY = process.env.MEILI_API_KEY || process.env.MEILI_MASTER_KEY || undefined;
|
||||
|
||||
// Vault path resolution (default to ./vault)
|
||||
export const VAULT_PATH = process.env.VAULT_PATH || './vault';
|
||||
|
||||
// Server port
|
||||
export const PORT = Number(process.env.PORT || 4000);
|
||||
|
||||
export function debugPrintConfig(prefix = 'Config') {
|
||||
console.log(`[${prefix}]`, {
|
||||
MEILI_HOST,
|
||||
MEILI_KEY_LENGTH: MEILI_API_KEY?.length,
|
||||
MEILI_KEY_PREVIEW: MEILI_API_KEY ? `${MEILI_API_KEY.slice(0, 8)}...` : 'none',
|
||||
VAULT_PATH,
|
||||
PORT,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Obsidian Excalidraw format utilities
|
||||
* Handles parsing and serialization of .excalidraw.md files in Obsidian format
|
||||
*/
|
||||
|
||||
import LZString from 'lz-string';
|
||||
|
||||
/**
|
||||
* Extract front matter from markdown content
|
||||
* @param {string} md - Markdown content
|
||||
* @returns {string|null} - Front matter content (including --- delimiters) or null
|
||||
*/
|
||||
export function extractFrontMatter(md) {
|
||||
const match = md.match(/^---\s*\n([\s\S]*?)\n---/);
|
||||
return match ? match[0] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Obsidian Excalidraw markdown format
|
||||
* Extracts compressed-json block and decompresses using LZ-String
|
||||
* @param {string} md - Markdown content
|
||||
* @returns {object|null} - Excalidraw scene data or null if parsing fails
|
||||
*/
|
||||
export function parseObsidianExcalidrawMd(md) {
|
||||
if (!md || typeof md !== 'string') return null;
|
||||
|
||||
// Try to extract compressed-json block
|
||||
const compressedMatch = md.match(/```\s*compressed-json\s*\n([\s\S]*?)\n```/i);
|
||||
|
||||
if (compressedMatch && compressedMatch[1]) {
|
||||
try {
|
||||
// Remove whitespace from base64 data
|
||||
const compressed = compressedMatch[1].replace(/\s+/g, '');
|
||||
|
||||
// Decompress using LZ-String
|
||||
const decompressed = LZString.decompressFromBase64(compressed);
|
||||
|
||||
if (!decompressed) {
|
||||
console.warn('[Excalidraw] LZ-String decompression returned null');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Parse JSON
|
||||
const data = JSON.parse(decompressed);
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error('[Excalidraw] Failed to parse compressed-json:', error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: try to extract plain json block
|
||||
const jsonMatch = md.match(/```\s*(?:excalidraw|json)\s*\n([\s\S]*?)\n```/i);
|
||||
|
||||
if (jsonMatch && jsonMatch[1]) {
|
||||
try {
|
||||
const data = JSON.parse(jsonMatch[1].trim());
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error('[Excalidraw] Failed to parse json block:', error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse flat JSON format (legacy ObsiViewer format)
|
||||
* @param {string} text - JSON text
|
||||
* @returns {object|null} - Excalidraw scene data or null if parsing fails
|
||||
*/
|
||||
export function parseFlatJson(text) {
|
||||
if (!text || typeof text !== 'string') return null;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(text);
|
||||
|
||||
// Validate it has the expected structure
|
||||
if (data && typeof data === 'object' && Array.isArray(data.elements)) {
|
||||
return data;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Excalidraw scene to Obsidian markdown format
|
||||
* @param {object} data - Excalidraw scene data
|
||||
* @param {string|null} existingFrontMatter - Existing front matter to preserve
|
||||
* @returns {string} - Markdown content in Obsidian format
|
||||
*/
|
||||
export function toObsidianExcalidrawMd(data, existingFrontMatter = null) {
|
||||
// Normalize scene data
|
||||
const scene = {
|
||||
elements: Array.isArray(data?.elements) ? data.elements : [],
|
||||
appState: (data && typeof data.appState === 'object') ? data.appState : {},
|
||||
files: (data && typeof data.files === 'object') ? data.files : {}
|
||||
};
|
||||
|
||||
// Serialize to JSON
|
||||
const json = JSON.stringify(scene);
|
||||
|
||||
// Compress using LZ-String
|
||||
const compressed = LZString.compressToBase64(json);
|
||||
|
||||
// Build front matter: merge existing with required keys
|
||||
const ensureFrontMatter = (fmRaw) => {
|
||||
const wrap = (inner) => `---\n${inner}\n---`;
|
||||
if (!fmRaw || typeof fmRaw !== 'string' || !/^---[\s\S]*?---\s*$/m.test(fmRaw)) {
|
||||
return wrap(`excalidraw-plugin: parsed\ntags: [excalidraw]`);
|
||||
}
|
||||
// Strip leading/trailing ---
|
||||
const inner = fmRaw.replace(/^---\s*\n?/, '').replace(/\n?---\s*$/, '');
|
||||
const lines = inner.split(/\r?\n/);
|
||||
let hasPlugin = false;
|
||||
let tagsLineIdx = -1;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const l = lines[i].trim();
|
||||
if (/^excalidraw-plugin\s*:/i.test(l)) hasPlugin = true;
|
||||
if (/^tags\s*:/i.test(l)) tagsLineIdx = i;
|
||||
}
|
||||
if (!hasPlugin) {
|
||||
lines.push('excalidraw-plugin: parsed');
|
||||
}
|
||||
if (tagsLineIdx === -1) {
|
||||
lines.push('tags: [excalidraw]');
|
||||
} else {
|
||||
// Ensure 'excalidraw' tag present; naive merge without YAML parse
|
||||
const orig = lines[tagsLineIdx];
|
||||
if (!/excalidraw\b/.test(orig)) {
|
||||
const mArr = orig.match(/^tags\s*:\s*\[(.*)\]\s*$/i);
|
||||
if (mArr) {
|
||||
const inside = mArr[1].trim();
|
||||
const updatedInside = inside ? `${inside}, excalidraw` : 'excalidraw';
|
||||
lines[tagsLineIdx] = `tags: [${updatedInside}]`;
|
||||
} else if (/^tags\s*:\s*$/i.test(orig)) {
|
||||
lines[tagsLineIdx] = 'tags: [excalidraw]';
|
||||
} else {
|
||||
// Fallback: append a separate tags line
|
||||
lines.push('tags: [excalidraw]');
|
||||
}
|
||||
}
|
||||
}
|
||||
return wrap(lines.join('\n'));
|
||||
};
|
||||
|
||||
const frontMatter = ensureFrontMatter(existingFrontMatter?.trim() || null);
|
||||
|
||||
// Banner text (Obsidian standard)
|
||||
const banner = `==⚠ Switch to EXCALIDRAW VIEW in the MORE OPTIONS menu of this document. ⚠==
|
||||
You can decompress Drawing data with the command palette: 'Decompress current Excalidraw file'. For more info check in plugin settings under 'Saving'`;
|
||||
|
||||
// Construct full markdown
|
||||
return `${frontMatter}
|
||||
${banner}
|
||||
|
||||
# Excalidraw Data
|
||||
|
||||
## Text Elements
|
||||
%%
|
||||
## Drawing
|
||||
\`\`\`compressed-json
|
||||
${compressed}
|
||||
\`\`\`
|
||||
%%`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Excalidraw content from any supported format
|
||||
* Tries Obsidian MD format first, then falls back to flat JSON
|
||||
* @param {string} text - File content
|
||||
* @returns {object|null} - Excalidraw scene data or null
|
||||
*/
|
||||
export function parseExcalidrawAny(text) {
|
||||
// Try Obsidian format first
|
||||
let data = parseObsidianExcalidrawMd(text);
|
||||
if (data) return data;
|
||||
|
||||
// Fallback to flat JSON
|
||||
data = parseFlatJson(text);
|
||||
if (data) return data;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate Excalidraw scene structure
|
||||
* @param {any} data - Data to validate
|
||||
* @returns {boolean} - True if valid
|
||||
*/
|
||||
export function isValidExcalidrawScene(data) {
|
||||
if (!data || typeof data !== 'object') return false;
|
||||
if (!Array.isArray(data.elements)) return false;
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Unit tests for Excalidraw Obsidian format utilities
|
||||
* Run with: node server/excalidraw-obsidian.test.mjs
|
||||
*/
|
||||
|
||||
import assert from 'assert';
|
||||
import {
|
||||
parseObsidianExcalidrawMd,
|
||||
parseFlatJson,
|
||||
toObsidianExcalidrawMd,
|
||||
extractFrontMatter,
|
||||
parseExcalidrawAny,
|
||||
isValidExcalidrawScene
|
||||
} from './excalidraw-obsidian.mjs';
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(`✅ ${name}`);
|
||||
passed++;
|
||||
} catch (error) {
|
||||
console.error(`❌ ${name}`);
|
||||
console.error(` ${error.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Sample data
|
||||
const sampleScene = {
|
||||
elements: [
|
||||
{ id: '1', type: 'rectangle', x: 100, y: 100, width: 200, height: 100 }
|
||||
],
|
||||
appState: { viewBackgroundColor: '#ffffff' },
|
||||
files: {}
|
||||
};
|
||||
|
||||
const sampleFrontMatter = `---
|
||||
excalidraw-plugin: parsed
|
||||
tags: [excalidraw]
|
||||
---`;
|
||||
|
||||
console.log('🧪 Running Excalidraw Obsidian Format Tests\n');
|
||||
|
||||
// Test: extractFrontMatter
|
||||
test('extractFrontMatter - extracts front matter', () => {
|
||||
const md = `---
|
||||
excalidraw-plugin: parsed
|
||||
---
|
||||
Content here`;
|
||||
const result = extractFrontMatter(md);
|
||||
assert.ok(result);
|
||||
assert.ok(result.includes('excalidraw-plugin'));
|
||||
});
|
||||
|
||||
test('extractFrontMatter - returns null when no front matter', () => {
|
||||
const md = 'Just some content';
|
||||
const result = extractFrontMatter(md);
|
||||
assert.strictEqual(result, null);
|
||||
});
|
||||
|
||||
// Test: toObsidianExcalidrawMd
|
||||
test('toObsidianExcalidrawMd - creates valid markdown', () => {
|
||||
const md = toObsidianExcalidrawMd(sampleScene);
|
||||
assert.ok(md.includes('---'));
|
||||
assert.ok(md.includes('excalidraw-plugin'));
|
||||
assert.ok(md.includes('```compressed-json'));
|
||||
assert.ok(md.includes('# Excalidraw Data'));
|
||||
});
|
||||
|
||||
test('toObsidianExcalidrawMd - preserves existing front matter', () => {
|
||||
const customFrontMatter = `---
|
||||
excalidraw-plugin: parsed
|
||||
tags: [custom, test]
|
||||
---`;
|
||||
const md = toObsidianExcalidrawMd(sampleScene, customFrontMatter);
|
||||
assert.ok(md.includes('tags: [custom, test]'));
|
||||
});
|
||||
|
||||
// Test: parseObsidianExcalidrawMd round-trip
|
||||
test('parseObsidianExcalidrawMd - round-trip conversion', () => {
|
||||
const md = toObsidianExcalidrawMd(sampleScene);
|
||||
const parsed = parseObsidianExcalidrawMd(md);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.ok(Array.isArray(parsed.elements));
|
||||
assert.strictEqual(parsed.elements.length, 1);
|
||||
assert.strictEqual(parsed.elements[0].type, 'rectangle');
|
||||
});
|
||||
|
||||
// Test: parseFlatJson
|
||||
test('parseFlatJson - parses valid JSON', () => {
|
||||
const json = JSON.stringify(sampleScene);
|
||||
const parsed = parseFlatJson(json);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.ok(Array.isArray(parsed.elements));
|
||||
assert.strictEqual(parsed.elements.length, 1);
|
||||
});
|
||||
|
||||
test('parseFlatJson - returns null for invalid JSON', () => {
|
||||
const parsed = parseFlatJson('not valid json');
|
||||
assert.strictEqual(parsed, null);
|
||||
});
|
||||
|
||||
test('parseFlatJson - returns null for JSON without elements', () => {
|
||||
const parsed = parseFlatJson('{"foo": "bar"}');
|
||||
assert.strictEqual(parsed, null);
|
||||
});
|
||||
|
||||
// Test: parseExcalidrawAny
|
||||
test('parseExcalidrawAny - parses Obsidian format', () => {
|
||||
const md = toObsidianExcalidrawMd(sampleScene);
|
||||
const parsed = parseExcalidrawAny(md);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.ok(Array.isArray(parsed.elements));
|
||||
});
|
||||
|
||||
test('parseExcalidrawAny - parses flat JSON', () => {
|
||||
const json = JSON.stringify(sampleScene);
|
||||
const parsed = parseExcalidrawAny(json);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.ok(Array.isArray(parsed.elements));
|
||||
});
|
||||
|
||||
test('parseExcalidrawAny - returns null for invalid content', () => {
|
||||
const parsed = parseExcalidrawAny('invalid content');
|
||||
assert.strictEqual(parsed, null);
|
||||
});
|
||||
|
||||
// Test: isValidExcalidrawScene
|
||||
test('isValidExcalidrawScene - validates correct structure', () => {
|
||||
assert.strictEqual(isValidExcalidrawScene(sampleScene), true);
|
||||
});
|
||||
|
||||
test('isValidExcalidrawScene - rejects invalid structure', () => {
|
||||
assert.strictEqual(isValidExcalidrawScene({}), false);
|
||||
assert.strictEqual(isValidExcalidrawScene({ elements: 'not an array' }), false);
|
||||
assert.strictEqual(isValidExcalidrawScene(null), false);
|
||||
});
|
||||
|
||||
// Test: Empty scene handling
|
||||
test('toObsidianExcalidrawMd - handles empty scene', () => {
|
||||
const emptyScene = { elements: [], appState: {}, files: {} };
|
||||
const md = toObsidianExcalidrawMd(emptyScene);
|
||||
const parsed = parseObsidianExcalidrawMd(md);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.ok(Array.isArray(parsed.elements));
|
||||
assert.strictEqual(parsed.elements.length, 0);
|
||||
});
|
||||
|
||||
// Test: Large scene handling
|
||||
test('toObsidianExcalidrawMd - handles large scene', () => {
|
||||
const largeScene = {
|
||||
elements: Array.from({ length: 100 }, (_, i) => ({
|
||||
id: `${i}`,
|
||||
type: 'rectangle',
|
||||
x: i * 10,
|
||||
y: i * 10,
|
||||
width: 50,
|
||||
height: 50
|
||||
})),
|
||||
appState: {},
|
||||
files: {}
|
||||
};
|
||||
|
||||
const md = toObsidianExcalidrawMd(largeScene);
|
||||
const parsed = parseObsidianExcalidrawMd(md);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.strictEqual(parsed.elements.length, 100);
|
||||
});
|
||||
|
||||
// Test: Special characters in scene
|
||||
test('toObsidianExcalidrawMd - handles special characters', () => {
|
||||
const sceneWithSpecialChars = {
|
||||
elements: [
|
||||
{ id: '1', type: 'text', text: 'Hello "World" & <test>' }
|
||||
],
|
||||
appState: {},
|
||||
files: {}
|
||||
};
|
||||
|
||||
const md = toObsidianExcalidrawMd(sceneWithSpecialChars);
|
||||
const parsed = parseObsidianExcalidrawMd(md);
|
||||
|
||||
assert.ok(parsed);
|
||||
assert.strictEqual(parsed.elements[0].text, 'Hello "World" & <test>');
|
||||
});
|
||||
|
||||
// Summary
|
||||
console.log('\n━'.repeat(30));
|
||||
console.log(`✅ Passed: ${passed}`);
|
||||
console.log(`❌ Failed: ${failed}`);
|
||||
console.log('━'.repeat(30));
|
||||
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
+439
-7
@@ -4,17 +4,30 @@ import express from 'express';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import fs from 'fs';
|
||||
import zlib from 'zlib';
|
||||
import chokidar from 'chokidar';
|
||||
import { meiliClient, vaultIndexName, ensureIndexSettings } from './meilisearch.client.mjs';
|
||||
import { fullReindex, upsertFile, deleteFile } from './meilisearch-indexer.mjs';
|
||||
import { mapObsidianQueryToMeili, buildSearchParams } from './search.mapping.mjs';
|
||||
import { PORT as CFG_PORT, VAULT_PATH as CFG_VAULT_PATH, debugPrintConfig } from './config.mjs';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
parseExcalidrawAny,
|
||||
toObsidianExcalidrawMd,
|
||||
extractFrontMatter,
|
||||
isValidExcalidrawScene
|
||||
} from './excalidraw-obsidian.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 4000;
|
||||
const PORT = CFG_PORT;
|
||||
|
||||
const rootDir = path.resolve(__dirname, '..');
|
||||
const distDir = path.join(rootDir, 'dist');
|
||||
const vaultDir = path.join(rootDir, 'vault');
|
||||
// Centralized vault directory
|
||||
const vaultDir = path.isAbsolute(CFG_VAULT_PATH) ? CFG_VAULT_PATH : path.join(rootDir, CFG_VAULT_PATH);
|
||||
|
||||
const vaultEventClients = new Set();
|
||||
|
||||
@@ -28,6 +41,7 @@ const registerVaultEventClient = (res) => {
|
||||
}, 20000);
|
||||
|
||||
const client = { res, heartbeat };
|
||||
// moved scanVaultDrawings to top-level
|
||||
vaultEventClients.add(client);
|
||||
return client;
|
||||
};
|
||||
@@ -154,6 +168,44 @@ const loadVaultNotes = (vaultPath) => {
|
||||
return notes;
|
||||
};
|
||||
|
||||
// Scan vault for .excalidraw.md files and return FileMetadata-like entries
|
||||
const scanVaultDrawings = (vaultPath) => {
|
||||
const items = [];
|
||||
const walk = (currentDir) => {
|
||||
let entries = [];
|
||||
try {
|
||||
entries = fs.readdirSync(currentDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(currentDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(entryPath);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) continue;
|
||||
const lower = entry.name.toLowerCase();
|
||||
if (!lower.endsWith('.excalidraw.md')) continue;
|
||||
try {
|
||||
const stats = fs.statSync(entryPath);
|
||||
const relPath = path.relative(vaultPath, entryPath).replace(/\\/g, '/');
|
||||
const id = slugifyPath(relPath.replace(/\.excalidraw(?:\.md)?$/i, ''));
|
||||
const title = path.basename(relPath).replace(/\.excalidraw(?:\.md)?$/i, '');
|
||||
items.push({
|
||||
id,
|
||||
title,
|
||||
path: relPath,
|
||||
createdAt: new Date(stats.birthtimeMs ? stats.birthtimeMs : stats.ctimeMs).toISOString(),
|
||||
updatedAt: new Date(stats.mtimeMs).toISOString(),
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
};
|
||||
walk(vaultPath);
|
||||
return items;
|
||||
};
|
||||
|
||||
const buildFileMetadata = (notes) =>
|
||||
notes.map((note) => ({
|
||||
id: note.id,
|
||||
@@ -198,6 +250,26 @@ watchedVaultEvents.forEach((eventName) => {
|
||||
});
|
||||
});
|
||||
|
||||
// Integrate Meilisearch with Chokidar for incremental updates
|
||||
vaultWatcher.on('add', (filePath) => {
|
||||
if (filePath.toLowerCase().endsWith('.md')) {
|
||||
upsertFile(filePath).catch(err => console.error('[Meili] Upsert on add failed:', err));
|
||||
}
|
||||
});
|
||||
|
||||
vaultWatcher.on('change', (filePath) => {
|
||||
if (filePath.toLowerCase().endsWith('.md')) {
|
||||
upsertFile(filePath).catch(err => console.error('[Meili] Upsert on change failed:', err));
|
||||
}
|
||||
});
|
||||
|
||||
vaultWatcher.on('unlink', (filePath) => {
|
||||
if (filePath.toLowerCase().endsWith('.md')) {
|
||||
const relativePath = path.relative(vaultDir, filePath).replace(/\\/g, '/');
|
||||
deleteFile(relativePath).catch(err => console.error('[Meili] Delete failed:', err));
|
||||
}
|
||||
});
|
||||
|
||||
vaultWatcher.on('ready', () => {
|
||||
broadcastVaultEvent({
|
||||
event: 'ready',
|
||||
@@ -356,13 +428,84 @@ app.get('/api/vault', (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/files/metadata', (req, res) => {
|
||||
// Fast file list from Meilisearch (id, title, path, createdAt, updatedAt)
|
||||
app.get('/api/files/list', async (req, res) => {
|
||||
try {
|
||||
const notes = loadVaultNotes(vaultDir);
|
||||
res.json(buildFileMetadata(notes));
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
const result = await index.search('', {
|
||||
limit: 10000,
|
||||
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
|
||||
});
|
||||
|
||||
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
|
||||
id: hit.id,
|
||||
title: hit.title,
|
||||
path: hit.path,
|
||||
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
||||
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
||||
})) : [];
|
||||
|
||||
res.json(items);
|
||||
} catch (error) {
|
||||
console.error('Failed to load file metadata:', error);
|
||||
res.status(500).json({ error: 'Unable to load file metadata.' });
|
||||
console.error('Failed to list files via Meilisearch, falling back to FS:', error);
|
||||
try {
|
||||
const notes = loadVaultNotes(vaultDir);
|
||||
res.json(buildFileMetadata(notes));
|
||||
} catch (err2) {
|
||||
console.error('FS fallback failed:', err2);
|
||||
res.status(500).json({ error: 'Unable to list files.' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/files/metadata', async (req, res) => {
|
||||
try {
|
||||
// Prefer Meilisearch for fast metadata
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
const result = await index.search('', {
|
||||
limit: 10000,
|
||||
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
|
||||
});
|
||||
|
||||
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
|
||||
id: hit.id,
|
||||
title: hit.title,
|
||||
path: hit.path,
|
||||
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
||||
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
||||
})) : [];
|
||||
|
||||
// Merge .excalidraw files discovered via FS
|
||||
const drawings = scanVaultDrawings(vaultDir);
|
||||
const byPath = new Map(items.map(it => [String(it.path).toLowerCase(), it]));
|
||||
for (const d of drawings) {
|
||||
const key = String(d.path).toLowerCase();
|
||||
if (!byPath.has(key)) {
|
||||
byPath.set(key, d);
|
||||
}
|
||||
}
|
||||
|
||||
res.json(Array.from(byPath.values()));
|
||||
} catch (error) {
|
||||
console.error('Failed to load file metadata via Meilisearch, falling back to FS:', error);
|
||||
try {
|
||||
const notes = loadVaultNotes(vaultDir);
|
||||
const base = buildFileMetadata(notes);
|
||||
const drawings = scanVaultDrawings(vaultDir);
|
||||
const byPath = new Map(base.map(it => [String(it.path).toLowerCase(), it]));
|
||||
for (const d of drawings) {
|
||||
const key = String(d.path).toLowerCase();
|
||||
if (!byPath.has(key)) byPath.set(key, d);
|
||||
}
|
||||
res.json(Array.from(byPath.values()));
|
||||
} catch (err2) {
|
||||
console.error('FS fallback failed:', err2);
|
||||
res.status(500).json({ error: 'Unable to load file metadata.' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -500,6 +643,239 @@ app.post('/api/logs', (req, res) => {
|
||||
res.status(202).json({ status: 'queued' });
|
||||
});
|
||||
|
||||
// --- Files API (supports .excalidraw.md (Markdown-wrapped JSON), .excalidraw, .json and binary sidecars) ---
|
||||
// Helpers
|
||||
const sanitizeRelPath = (rel) => String(rel || '').replace(/\\/g, '/').replace(/^\/+/, '');
|
||||
const resolveVaultPath = (rel) => {
|
||||
const clean = sanitizeRelPath(rel);
|
||||
const abs = path.resolve(vaultDir, clean);
|
||||
if (!abs.startsWith(path.resolve(vaultDir))) {
|
||||
throw Object.assign(new Error('Invalid path'), { status: 400 });
|
||||
}
|
||||
return abs;
|
||||
};
|
||||
|
||||
const excalidrawSceneSchema = z.object({
|
||||
elements: z.array(z.any()),
|
||||
appState: z.record(z.any()).optional(),
|
||||
files: z.record(z.any()).optional(),
|
||||
}).passthrough();
|
||||
|
||||
// Helper to determine content type
|
||||
function guessContentType(filePath) {
|
||||
const lower = filePath.toLowerCase();
|
||||
if (lower.endsWith('.md')) return 'text/markdown; charset=utf-8';
|
||||
if (lower.endsWith('.json')) return 'application/json; charset=utf-8';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
// GET file content (supports .excalidraw.md, .excalidraw, .json, .md)
|
||||
app.get('/api/files', (req, res) => {
|
||||
try {
|
||||
const pathParam = req.query.path;
|
||||
if (!pathParam || typeof pathParam !== 'string') {
|
||||
return res.status(400).json({ error: 'Missing path query parameter' });
|
||||
}
|
||||
|
||||
const rel = decodeURIComponent(pathParam);
|
||||
const abs = resolveVaultPath(rel);
|
||||
|
||||
if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) {
|
||||
return res.status(404).json({ error: 'File not found' });
|
||||
}
|
||||
|
||||
const base = path.basename(abs).toLowerCase();
|
||||
const ext = path.extname(abs).toLowerCase();
|
||||
const isExcalidrawMd = base.endsWith('.excalidraw.md');
|
||||
const isExcalidraw = ext === '.excalidraw' || ext === '.json' || isExcalidrawMd;
|
||||
|
||||
if (!isExcalidraw && ext !== '.md') {
|
||||
return res.status(415).json({ error: 'Unsupported file type' });
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(abs, 'utf-8');
|
||||
|
||||
// For Excalidraw files, parse and return JSON
|
||||
if (isExcalidraw) {
|
||||
const data = parseExcalidrawAny(content);
|
||||
|
||||
if (!data || !isValidExcalidrawScene(data)) {
|
||||
return res.status(400).json({ error: 'Invalid Excalidraw format' });
|
||||
}
|
||||
|
||||
// Normalize scene structure
|
||||
const normalized = {
|
||||
elements: Array.isArray(data.elements) ? data.elements : [],
|
||||
appState: (data && typeof data.appState === 'object') ? data.appState : {},
|
||||
files: (data && typeof data.files === 'object') ? data.files : {}
|
||||
};
|
||||
|
||||
const rev = calculateSimpleHash(content);
|
||||
res.setHeader('ETag', rev);
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
return res.send(JSON.stringify(normalized));
|
||||
}
|
||||
|
||||
// For regular markdown, return as-is
|
||||
const rev = calculateSimpleHash(content);
|
||||
res.setHeader('ETag', rev);
|
||||
res.setHeader('Content-Type', guessContentType(abs));
|
||||
return res.send(content);
|
||||
|
||||
} catch (error) {
|
||||
const code = typeof error?.status === 'number' ? error.status : 500;
|
||||
console.error('GET /api/files error:', error);
|
||||
res.status(code).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// PUT file content with If-Match check and size limit (10MB)
|
||||
app.put('/api/files', express.json({ limit: '10mb' }), express.text({ limit: '10mb', type: 'text/markdown' }), (req, res) => {
|
||||
try {
|
||||
const pathParam = req.query.path;
|
||||
if (!pathParam || typeof pathParam !== 'string') {
|
||||
return res.status(400).json({ error: 'Missing path query parameter' });
|
||||
}
|
||||
|
||||
const rel = decodeURIComponent(pathParam);
|
||||
const abs = resolveVaultPath(rel);
|
||||
const dir = path.dirname(abs);
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
const contentType = (req.headers['content-type'] || '').split(';')[0];
|
||||
const base = path.basename(abs).toLowerCase();
|
||||
const isExcalidrawMd = base.endsWith('.excalidraw.md');
|
||||
|
||||
let finalContent;
|
||||
let existingFrontMatter = null;
|
||||
console.log('[PUT /api/files] path=%s contentType=%s isExcalidrawMd=%s', rel, contentType, isExcalidrawMd);
|
||||
|
||||
// Extract existing front matter if file exists
|
||||
if (fs.existsSync(abs) && isExcalidrawMd) {
|
||||
try {
|
||||
const existing = fs.readFileSync(abs, 'utf-8');
|
||||
existingFrontMatter = extractFrontMatter(existing);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Handle JSON payload (Excalidraw scene)
|
||||
if (contentType === 'application/json') {
|
||||
const body = req.body;
|
||||
const parsed = excalidrawSceneSchema.safeParse(body);
|
||||
|
||||
if (!parsed.success) {
|
||||
console.warn('[PUT /api/files] invalid scene schema', parsed.error?.issues?.slice(0,3));
|
||||
return res.status(400).json({
|
||||
error: 'Invalid Excalidraw scene',
|
||||
issues: parsed.error.issues?.slice(0, 5)
|
||||
});
|
||||
}
|
||||
|
||||
// Convert to Obsidian format if target is .excalidraw.md
|
||||
if (isExcalidrawMd) {
|
||||
finalContent = toObsidianExcalidrawMd(parsed.data, existingFrontMatter);
|
||||
} else {
|
||||
// Plain JSON for .excalidraw or .json files
|
||||
finalContent = JSON.stringify(parsed.data, null, 2);
|
||||
}
|
||||
}
|
||||
// Handle text/markdown payload (already formatted)
|
||||
else if (contentType === 'text/markdown') {
|
||||
finalContent = typeof req.body === 'string' ? req.body : String(req.body);
|
||||
}
|
||||
else {
|
||||
console.warn('[PUT /api/files] unsupported content-type', contentType);
|
||||
return res.status(400).json({ error: 'Unsupported content type' });
|
||||
}
|
||||
|
||||
// Check size limit
|
||||
if (Buffer.byteLength(finalContent, 'utf-8') > 10 * 1024 * 1024) {
|
||||
console.warn('[PUT /api/files] payload too large path=%s size=%d', rel, Buffer.byteLength(finalContent, 'utf-8'));
|
||||
return res.status(413).json({ error: 'Payload too large' });
|
||||
}
|
||||
|
||||
// Check for conflicts with If-Match
|
||||
const hasExisting = fs.existsSync(abs);
|
||||
const ifMatch = req.headers['if-match'];
|
||||
if (hasExisting && ifMatch) {
|
||||
const current = fs.readFileSync(abs, 'utf-8');
|
||||
const currentRev = calculateSimpleHash(current);
|
||||
if (ifMatch !== currentRev) {
|
||||
console.warn('[PUT /api/files] conflict path=%s ifMatch=%s current=%s', rel, ifMatch, currentRev);
|
||||
return res.status(409).json({ error: 'Conflict detected' });
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic write with backup
|
||||
const temp = abs + '.tmp';
|
||||
const backup = abs + '.bak';
|
||||
|
||||
try {
|
||||
if (hasExisting) fs.copyFileSync(abs, backup);
|
||||
fs.writeFileSync(temp, finalContent, 'utf-8');
|
||||
fs.renameSync(temp, abs);
|
||||
console.log('[PUT /api/files] wrote file path=%s bytes=%d', rel, Buffer.byteLength(finalContent, 'utf-8'));
|
||||
} catch (e) {
|
||||
if (fs.existsSync(temp)) try { fs.unlinkSync(temp); } catch {}
|
||||
if (hasExisting && fs.existsSync(backup)) try { fs.copyFileSync(backup, abs); } catch {}
|
||||
console.error('[PUT /api/files] write error path=%s', rel, e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
const rev = calculateSimpleHash(finalContent);
|
||||
res.setHeader('ETag', rev);
|
||||
res.json({ rev });
|
||||
|
||||
} catch (error) {
|
||||
const code = typeof error?.status === 'number' ? error.status : 500;
|
||||
console.error('PUT /api/files error:', error);
|
||||
res.status(code).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// PUT binary sidecar (e.g., PNG/SVG)
|
||||
app.put('/api/files/blob', (req, res) => {
|
||||
try {
|
||||
const pathParam = req.query.path;
|
||||
if (!pathParam || typeof pathParam !== 'string') {
|
||||
return res.status(400).json({ error: 'Missing path query parameter' });
|
||||
}
|
||||
|
||||
const rel = decodeURIComponent(pathParam);
|
||||
const abs = resolveVaultPath(rel);
|
||||
const dir = path.dirname(abs);
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Collect raw body
|
||||
const chunks = [];
|
||||
req.on('data', (chunk) => {
|
||||
chunks.push(chunk);
|
||||
const size = chunks.reduce((a, b) => a + b.length, 0);
|
||||
if (size > 10 * 1024 * 1024) { // 10MB limit
|
||||
req.destroy();
|
||||
}
|
||||
});
|
||||
req.on('end', () => {
|
||||
const buf = Buffer.concat(chunks);
|
||||
// Basic allowlist
|
||||
const ext = path.extname(abs).toLowerCase();
|
||||
if (!['.png', '.svg'].includes(ext)) {
|
||||
return res.status(415).json({ error: 'unsupported_media_type' });
|
||||
}
|
||||
fs.writeFileSync(abs, buf);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
req.on('error', (err) => {
|
||||
console.error('Blob upload error:', err);
|
||||
res.status(500).json({ error: 'internal_error' });
|
||||
});
|
||||
} catch (error) {
|
||||
const code = typeof error?.status === 'number' ? error.status : 500;
|
||||
console.error('PUT /api/files/blob error:', error);
|
||||
res.status(code).json({ error: 'internal_error' });
|
||||
}
|
||||
});
|
||||
|
||||
function ensureBookmarksStorage() {
|
||||
const obsidianDir = path.join(vaultDir, '.obsidian');
|
||||
if (!fs.existsSync(obsidianDir)) {
|
||||
@@ -688,6 +1064,62 @@ function calculateSimpleHash(content) {
|
||||
return Math.abs(hash).toString(36) + '-' + content.length;
|
||||
}
|
||||
|
||||
// Meilisearch API endpoints
|
||||
app.get('/api/search', async (req, res) => {
|
||||
try {
|
||||
const { q = '', limit = '20', offset = '0', sort, highlight = 'true' } = req.query;
|
||||
|
||||
// Parse Obsidian-style query to Meilisearch format
|
||||
const parsedQuery = mapObsidianQueryToMeili(String(q));
|
||||
|
||||
// Build search parameters
|
||||
const searchParams = buildSearchParams(parsedQuery, {
|
||||
limit: Number(limit),
|
||||
offset: Number(offset),
|
||||
sort,
|
||||
highlight: highlight === 'true'
|
||||
});
|
||||
|
||||
// Execute search
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
const result = await index.search(searchParams.q, searchParams);
|
||||
|
||||
// Return results
|
||||
res.json({
|
||||
hits: result.hits,
|
||||
estimatedTotalHits: result.estimatedTotalHits,
|
||||
facetDistribution: result.facetDistribution,
|
||||
processingTimeMs: result.processingTimeMs,
|
||||
query: q
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Meili] Search failed:', error);
|
||||
res.status(500).json({
|
||||
error: 'search_failed',
|
||||
message: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/reindex', async (_req, res) => {
|
||||
try {
|
||||
console.log('[Meili] Manual reindex triggered');
|
||||
const result = await fullReindex();
|
||||
res.json({
|
||||
ok: true,
|
||||
...result
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Meili] Reindex failed:', error);
|
||||
res.status(500).json({
|
||||
error: 'reindex_failed',
|
||||
message: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Servir l'index.html pour toutes les routes (SPA)
|
||||
const sendIndex = (req, res) => {
|
||||
const indexPath = path.join(distDir, 'index.html');
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import fssync from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import fg from 'fast-glob';
|
||||
import matter from 'gray-matter';
|
||||
import removeMd from 'remove-markdown';
|
||||
import { meiliClient, vaultIndexName, ensureIndexSettings } from './meilisearch.client.mjs';
|
||||
import { VAULT_PATH as CFG_VAULT_PATH, MEILI_HOST as CFG_MEILI_HOST, MEILI_API_KEY as CFG_MEILI_KEY } from './config.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const VAULT_PATH = path.isAbsolute(CFG_VAULT_PATH)
|
||||
? CFG_VAULT_PATH
|
||||
: path.resolve(__dirname, '..', CFG_VAULT_PATH);
|
||||
|
||||
console.log('[Meili Indexer] Environment check:', {
|
||||
MEILI_MASTER_KEY: CFG_MEILI_KEY ? `${String(CFG_MEILI_KEY).substring(0, 8)}... (${String(CFG_MEILI_KEY).length} chars)` : 'NOT SET',
|
||||
MEILI_API_KEY: CFG_MEILI_KEY ? `${String(CFG_MEILI_KEY).substring(0, 8)}...` : 'NOT SET',
|
||||
MEILI_HOST: CFG_MEILI_HOST,
|
||||
VAULT_PATH: VAULT_PATH
|
||||
});
|
||||
|
||||
/**
|
||||
* Convert timestamp to year/month for faceting
|
||||
*/
|
||||
function toYMD(timestampMs) {
|
||||
const dt = new Date(timestampMs);
|
||||
return {
|
||||
year: dt.getFullYear(),
|
||||
month: dt.getMonth() + 1
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all parent directory paths for a file
|
||||
* Example: "Projects/Angular/App.md" -> ["Projects", "Projects/Angular"]
|
||||
*/
|
||||
function parentDirs(relativePath) {
|
||||
const parts = relativePath.split(/[\\/]/);
|
||||
const acc = [];
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
acc.push(parts.slice(0, i + 1).join('/'));
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract headings from markdown content
|
||||
*/
|
||||
function extractHeadings(content) {
|
||||
const headingRegex = /^#+\s+(.+)$/gm;
|
||||
const headings = [];
|
||||
let match;
|
||||
while ((match = headingRegex.exec(content)) !== null) {
|
||||
headings.push(match[1].trim());
|
||||
}
|
||||
return headings.slice(0, 200); // Limit to 200 headings
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a searchable document from a markdown file
|
||||
*/
|
||||
export async function buildDocumentFromFile(absPath) {
|
||||
const rel = path.relative(VAULT_PATH, absPath).replaceAll('\\', '/');
|
||||
const file = path.basename(rel);
|
||||
const raw = await fs.readFile(absPath, 'utf8');
|
||||
|
||||
// Parse frontmatter with gray-matter
|
||||
const { data: fm, content } = matter(raw);
|
||||
|
||||
// Remove markdown formatting and limit content size
|
||||
const text = removeMd(content).slice(0, 200_000); // 200KB safety limit
|
||||
|
||||
// Extract metadata
|
||||
const title = fm.title ?? path.parse(file).name;
|
||||
const tags = Array.isArray(fm.tags)
|
||||
? fm.tags.map(String)
|
||||
: (fm.tags ? [String(fm.tags)] : []);
|
||||
|
||||
const headings = extractHeadings(content);
|
||||
|
||||
// Get file stats
|
||||
const stat = await fs.stat(absPath);
|
||||
const { year, month } = toYMD(stat.mtimeMs);
|
||||
|
||||
// Meilisearch requires alphanumeric IDs (a-z A-Z 0-9 - _)
|
||||
// Replace dots, slashes, and other chars with underscores
|
||||
const safeId = rel.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
|
||||
return {
|
||||
id: safeId,
|
||||
path: rel,
|
||||
file,
|
||||
title,
|
||||
tags,
|
||||
properties: fm ?? {},
|
||||
content: text,
|
||||
headings,
|
||||
createdAt: stat.birthtimeMs || stat.ctimeMs,
|
||||
updatedAt: stat.mtimeMs,
|
||||
year,
|
||||
month,
|
||||
parentDirs: parentDirs(rel),
|
||||
excerpt: text.slice(0, 500)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a full reindex of all markdown files in the vault
|
||||
*/
|
||||
export async function fullReindex() {
|
||||
console.log('[Meili] Starting full reindex...');
|
||||
const startTime = Date.now();
|
||||
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(VAULT_PATH);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
|
||||
// Find all markdown files
|
||||
const entries = await fg(['**/*.md'], {
|
||||
cwd: VAULT_PATH,
|
||||
dot: false,
|
||||
onlyFiles: true,
|
||||
absolute: true
|
||||
});
|
||||
|
||||
console.log(`[Meili] Found ${entries.length} markdown files`);
|
||||
|
||||
// Process in batches to avoid memory issues
|
||||
const batchSize = 750;
|
||||
let totalIndexed = 0;
|
||||
|
||||
for (let i = 0; i < entries.length; i += batchSize) {
|
||||
const chunk = entries.slice(i, i + batchSize);
|
||||
const docs = await Promise.all(
|
||||
chunk.map(async (file) => {
|
||||
try {
|
||||
return await buildDocumentFromFile(file);
|
||||
} catch (err) {
|
||||
console.error(`[Meili] Failed to process ${file}:`, err.message);
|
||||
return null;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
const validDocs = docs.filter(Boolean);
|
||||
if (validDocs.length > 0) {
|
||||
const task = await index.addDocuments(validDocs);
|
||||
console.log(`[Meili] Batch ${Math.floor(i / batchSize) + 1}: Queued ${validDocs.length} documents (task ${task.taskUid})`);
|
||||
totalIndexed += validDocs.length;
|
||||
}
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - startTime;
|
||||
console.log(`[Meili] Reindex complete: ${totalIndexed} documents indexed in ${elapsed}ms`);
|
||||
|
||||
return {
|
||||
indexed: true,
|
||||
count: totalIndexed,
|
||||
elapsedMs: elapsed
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert a single file (add or update)
|
||||
*/
|
||||
export async function upsertFile(relOrAbs) {
|
||||
const abs = path.isAbsolute(relOrAbs) ? relOrAbs : path.join(VAULT_PATH, relOrAbs);
|
||||
|
||||
if (!fssync.existsSync(abs)) {
|
||||
console.warn(`[Meili] File not found for upsert: ${abs}`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(VAULT_PATH);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
const doc = await buildDocumentFromFile(abs);
|
||||
await index.addDocuments([doc]);
|
||||
console.log(`[Meili] Upserted: ${doc.id}`);
|
||||
} catch (err) {
|
||||
console.error(`[Meili] Failed to upsert ${abs}:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from the index
|
||||
*/
|
||||
export async function deleteFile(relPath) {
|
||||
try {
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(VAULT_PATH);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
await index.deleteDocuments([relPath]);
|
||||
console.log(`[Meili] Deleted: ${relPath}`);
|
||||
} catch (err) {
|
||||
console.error(`[Meili] Failed to delete ${relPath}:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// CLI execution: node server/meilisearch-indexer.mjs
|
||||
if (process.argv[1] === __filename) {
|
||||
fullReindex()
|
||||
.then((result) => {
|
||||
console.log('[Meili] Reindex done:', result);
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('[Meili] Reindex failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { MeiliSearch } from 'meilisearch';
|
||||
import { MEILI_HOST, MEILI_API_KEY, VAULT_PATH as CFG_VAULT_PATH } from './config.mjs';
|
||||
|
||||
const MEILI_KEY = MEILI_API_KEY;
|
||||
|
||||
console.log('[Meili] Config:', {
|
||||
host: MEILI_HOST,
|
||||
keyLength: MEILI_KEY?.length,
|
||||
keyPreview: MEILI_KEY ? `${MEILI_KEY.slice(0, 8)}...` : 'none'
|
||||
});
|
||||
|
||||
if (!MEILI_KEY) {
|
||||
console.warn('[Meili] No API key provided; running without authentication. Set MEILI_API_KEY or MEILI_MASTER_KEY when securing Meilisearch.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and return a Meilisearch client instance
|
||||
*/
|
||||
export function meiliClient() {
|
||||
const options = MEILI_KEY ? { host: MEILI_HOST, apiKey: MEILI_KEY } : { host: MEILI_HOST };
|
||||
console.log('[Meili] Creating client with options:', {
|
||||
host: options.host,
|
||||
hasApiKey: !!options.apiKey,
|
||||
apiKeyLength: options.apiKey?.length,
|
||||
apiKeyHex: options.apiKey ? Buffer.from(options.apiKey).toString('hex') : 'none'
|
||||
});
|
||||
return new MeiliSearch(options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate index name from vault path (supports multi-vault)
|
||||
*/
|
||||
export function vaultIndexName(vaultPath = CFG_VAULT_PATH ?? './vault') {
|
||||
// Simple safe name generation; adjust to hash if paths can collide
|
||||
const safe = vaultPath.replace(/[^a-z0-9]+/gi, '_').toLowerCase();
|
||||
return `notes_${safe}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure index exists and configure settings for optimal search
|
||||
*/
|
||||
export async function ensureIndexSettings(client, indexUid) {
|
||||
// Create the index if it doesn't exist, then fetch it.
|
||||
// Some Meilisearch operations are task-based; we wait for completion to avoid 404s.
|
||||
let index;
|
||||
try {
|
||||
index = await client.getIndex(indexUid);
|
||||
} catch (e) {
|
||||
const task = await client.createIndex(indexUid, { primaryKey: 'id' });
|
||||
if (task?.taskUid !== undefined) {
|
||||
await client.waitForTask(task.taskUid, { timeOutMs: 30_000 });
|
||||
}
|
||||
index = await client.getIndex(indexUid);
|
||||
}
|
||||
|
||||
// Configure searchable, filterable, sortable, and faceted attributes
|
||||
const settingsTask = await index.updateSettings({
|
||||
searchableAttributes: [
|
||||
'title',
|
||||
'content',
|
||||
'file',
|
||||
'path',
|
||||
'tags',
|
||||
'properties.*',
|
||||
'headings'
|
||||
],
|
||||
displayedAttributes: [
|
||||
'id',
|
||||
'title',
|
||||
'path',
|
||||
'file',
|
||||
'tags',
|
||||
'properties',
|
||||
'updatedAt',
|
||||
'createdAt',
|
||||
'excerpt'
|
||||
],
|
||||
filterableAttributes: [
|
||||
'tags',
|
||||
'file',
|
||||
'path',
|
||||
'parentDirs',
|
||||
'properties.*',
|
||||
'year',
|
||||
'month'
|
||||
],
|
||||
sortableAttributes: [
|
||||
'updatedAt',
|
||||
'createdAt',
|
||||
'title',
|
||||
'file',
|
||||
'path'
|
||||
],
|
||||
faceting: {
|
||||
maxValuesPerFacet: 1000
|
||||
},
|
||||
typoTolerance: {
|
||||
enabled: true,
|
||||
minWordSizeForTypos: {
|
||||
oneTypo: 3,
|
||||
twoTypos: 6
|
||||
}
|
||||
},
|
||||
pagination: {
|
||||
maxTotalHits: 10000
|
||||
}
|
||||
});
|
||||
if (settingsTask?.taskUid !== undefined) {
|
||||
await client.waitForTask(settingsTask.taskUid, { timeOutMs: 30_000 });
|
||||
}
|
||||
|
||||
console.log(`[Meili] Index "${indexUid}" configured successfully`);
|
||||
return index;
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Migration script for converting old flat JSON Excalidraw files to Obsidian format
|
||||
* Usage: node server/migrate-excalidraw.mjs [--dry-run] [--vault-path=./vault]
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { parseFlatJson, toObsidianExcalidrawMd, isValidExcalidrawScene } from './excalidraw-obsidian.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Parse command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const vaultPathArg = args.find(arg => arg.startsWith('--vault-path='));
|
||||
const vaultPath = vaultPathArg
|
||||
? path.resolve(vaultPathArg.split('=')[1])
|
||||
: path.resolve(__dirname, '..', 'vault');
|
||||
|
||||
console.log('🔄 Excalidraw Migration Tool');
|
||||
console.log('━'.repeat(50));
|
||||
console.log(`Vault path: ${vaultPath}`);
|
||||
console.log(`Mode: ${dryRun ? 'DRY RUN (no changes)' : 'LIVE (will modify files)'}`);
|
||||
console.log('━'.repeat(50));
|
||||
console.log();
|
||||
|
||||
if (!fs.existsSync(vaultPath)) {
|
||||
console.error(`❌ Vault directory not found: ${vaultPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let filesScanned = 0;
|
||||
let filesConverted = 0;
|
||||
let filesSkipped = 0;
|
||||
let filesErrored = 0;
|
||||
|
||||
/**
|
||||
* Recursively scan directory for Excalidraw files
|
||||
*/
|
||||
function scanDirectory(dir) {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
// Skip hidden directories
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
scanDirectory(fullPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile()) continue;
|
||||
|
||||
const lower = entry.name.toLowerCase();
|
||||
|
||||
// Look for .excalidraw or .json files (but not .excalidraw.md)
|
||||
if (lower.endsWith('.excalidraw.md')) {
|
||||
// Already in Obsidian format, skip
|
||||
filesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!lower.endsWith('.excalidraw') && !lower.endsWith('.json')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
filesScanned++;
|
||||
processFile(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single file
|
||||
*/
|
||||
function processFile(filePath) {
|
||||
const relativePath = path.relative(vaultPath, filePath);
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
|
||||
// Try to parse as flat JSON
|
||||
const scene = parseFlatJson(content);
|
||||
|
||||
if (!scene || !isValidExcalidrawScene(scene)) {
|
||||
console.log(`⏭️ Skipped (not valid Excalidraw): ${relativePath}`);
|
||||
filesSkipped++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert to Obsidian format
|
||||
const obsidianMd = toObsidianExcalidrawMd(scene);
|
||||
|
||||
// Determine new file path
|
||||
const dir = path.dirname(filePath);
|
||||
const baseName = path.basename(filePath, path.extname(filePath));
|
||||
const newPath = path.join(dir, `${baseName}.excalidraw.md`);
|
||||
|
||||
if (dryRun) {
|
||||
console.log(`✅ Would convert: ${relativePath} → ${path.basename(newPath)}`);
|
||||
filesConverted++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Create backup of original
|
||||
const backupPath = filePath + '.bak';
|
||||
fs.copyFileSync(filePath, backupPath);
|
||||
|
||||
// Write new file
|
||||
fs.writeFileSync(newPath, obsidianMd, 'utf-8');
|
||||
|
||||
// Remove original if new file is different
|
||||
if (newPath !== filePath) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
|
||||
console.log(`✅ Converted: ${relativePath} → ${path.basename(newPath)}`);
|
||||
filesConverted++;
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing ${relativePath}:`, error.message);
|
||||
filesErrored++;
|
||||
}
|
||||
}
|
||||
|
||||
// Run migration
|
||||
try {
|
||||
scanDirectory(vaultPath);
|
||||
|
||||
console.log();
|
||||
console.log('━'.repeat(50));
|
||||
console.log('📊 Migration Summary');
|
||||
console.log('━'.repeat(50));
|
||||
console.log(`Files scanned: ${filesScanned}`);
|
||||
console.log(`Files converted: ${filesConverted}`);
|
||||
console.log(`Files skipped: ${filesSkipped}`);
|
||||
console.log(`Files errored: ${filesErrored}`);
|
||||
console.log('━'.repeat(50));
|
||||
|
||||
if (dryRun) {
|
||||
console.log();
|
||||
console.log('💡 This was a dry run. Run without --dry-run to apply changes.');
|
||||
} else if (filesConverted > 0) {
|
||||
console.log();
|
||||
console.log('✅ Migration complete! Backup files (.bak) were created.');
|
||||
}
|
||||
|
||||
process.exit(filesErrored > 0 ? 1 : 0);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Migration failed:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* Map Obsidian-style search queries to Meilisearch format
|
||||
* Supports: tag:, path:, file: operators with free text search
|
||||
*/
|
||||
export function mapObsidianQueryToMeili(qRaw) {
|
||||
const tokens = String(qRaw).trim().split(/\s+/);
|
||||
const filters = [];
|
||||
const meiliQ = [];
|
||||
let restrict = null;
|
||||
let rangeStart = null;
|
||||
let rangeEnd = null;
|
||||
|
||||
const parseDate = (s) => {
|
||||
if (!s) return null;
|
||||
// Accept YYYY-MM-DD
|
||||
const m = String(s).match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
if (!m) return null;
|
||||
const d = new Date(`${m[1]}-${m[2]}-${m[3]}T00:00:00Z`);
|
||||
if (Number.isNaN(d.getTime())) return null;
|
||||
return d;
|
||||
};
|
||||
|
||||
for (const token of tokens) {
|
||||
// tag: operator - filter by exact tag match
|
||||
if (token.startsWith('tag:')) {
|
||||
const value = token.slice(4).replace(/^["']|["']$/g, '');
|
||||
if (value) {
|
||||
// Remove leading # if present
|
||||
const cleanTag = value.startsWith('#') ? value.substring(1) : value;
|
||||
filters.push(`tags = "${cleanTag}"`);
|
||||
}
|
||||
}
|
||||
// path: operator - filter by parent directory path
|
||||
// We use parentDirs array to simulate startsWith behavior
|
||||
else if (token.startsWith('path:')) {
|
||||
const value = token.slice(5).replace(/^["']|["']$/g, '');
|
||||
if (value) {
|
||||
// Normalize path separators
|
||||
const normalizedPath = value.replace(/\\/g, '/');
|
||||
filters.push(`parentDirs = "${normalizedPath}"`);
|
||||
}
|
||||
}
|
||||
// file: operator - restrict search to file field
|
||||
else if (token.startsWith('file:')) {
|
||||
const value = token.slice(5).replace(/^["']|["']$/g, '');
|
||||
if (value) {
|
||||
// Restrict search to file field only
|
||||
restrict = ['file'];
|
||||
meiliQ.push(value);
|
||||
}
|
||||
}
|
||||
// year: operator - facet filter
|
||||
else if (token.startsWith('year:')) {
|
||||
const value = token.slice(5).replace(/^["']|["']$/g, '');
|
||||
const n = Number(value);
|
||||
if (!Number.isNaN(n)) {
|
||||
filters.push(`year = ${n}`);
|
||||
}
|
||||
}
|
||||
// month: operator - facet filter (1-12)
|
||||
else if (token.startsWith('month:')) {
|
||||
const value = token.slice(6).replace(/^["']|["']$/g, '');
|
||||
const n = Number(value);
|
||||
if (!Number.isNaN(n)) {
|
||||
filters.push(`month = ${n}`);
|
||||
}
|
||||
}
|
||||
// date:YYYY-MM-DD (single day)
|
||||
else if (token.startsWith('date:')) {
|
||||
const value = token.slice(5);
|
||||
const d = parseDate(value);
|
||||
if (d) {
|
||||
const start = new Date(d); start.setUTCHours(0,0,0,0);
|
||||
const end = new Date(d); end.setUTCHours(23,59,59,999);
|
||||
const startMs = start.getTime();
|
||||
const endMs = end.getTime();
|
||||
filters.push(`((createdAt >= ${startMs} AND createdAt <= ${endMs}) OR (updatedAt >= ${startMs} AND updatedAt <= ${endMs}))`);
|
||||
}
|
||||
}
|
||||
// from:/to: YYYY-MM-DD
|
||||
else if (token.startsWith('from:')) {
|
||||
const d = parseDate(token.slice(5));
|
||||
if (d) {
|
||||
const s = new Date(d); s.setUTCHours(0,0,0,0);
|
||||
rangeStart = s.getTime();
|
||||
}
|
||||
}
|
||||
else if (token.startsWith('to:')) {
|
||||
const d = parseDate(token.slice(3));
|
||||
if (d) {
|
||||
const e = new Date(d); e.setUTCHours(23,59,59,999);
|
||||
rangeEnd = e.getTime();
|
||||
}
|
||||
}
|
||||
// Regular text token
|
||||
else if (token) {
|
||||
meiliQ.push(token);
|
||||
}
|
||||
}
|
||||
|
||||
// If we captured a from/to range, add a combined date filter
|
||||
if (rangeStart !== null || rangeEnd !== null) {
|
||||
const startMs = rangeStart ?? 0;
|
||||
const endMs = rangeEnd ?? 8640000000000000; // large future
|
||||
filters.push(`((createdAt >= ${startMs} AND createdAt <= ${endMs}) OR (updatedAt >= ${startMs} AND updatedAt <= ${endMs}))`);
|
||||
}
|
||||
|
||||
return {
|
||||
meiliQ: meiliQ.join(' ').trim(),
|
||||
filters,
|
||||
restrict
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Meilisearch search parameters from parsed query
|
||||
*/
|
||||
export function buildSearchParams(parsedQuery, options = {}) {
|
||||
const {
|
||||
limit = 20,
|
||||
offset = 0,
|
||||
sort,
|
||||
highlight = true
|
||||
} = options;
|
||||
|
||||
const params = {
|
||||
q: parsedQuery.meiliQ,
|
||||
limit: Number(limit),
|
||||
offset: Number(offset),
|
||||
filter: parsedQuery.filters.length ? parsedQuery.filters.join(' AND ') : undefined,
|
||||
facets: ['tags', 'parentDirs', 'year', 'month'],
|
||||
attributesToHighlight: highlight ? ['title', 'content'] : [],
|
||||
highlightPreTag: '<mark>',
|
||||
highlightPostTag: '</mark>',
|
||||
attributesToCrop: ['content'],
|
||||
cropLength: 80,
|
||||
cropMarker: '…',
|
||||
attributesToSearchOn: parsedQuery.restrict?.length ? parsedQuery.restrict : undefined,
|
||||
sort: sort ? [String(sort)] : undefined,
|
||||
showMatchesPosition: false
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(params).forEach(key => {
|
||||
if (params[key] === undefined) {
|
||||
delete params[key];
|
||||
}
|
||||
});
|
||||
|
||||
return params;
|
||||
}
|
||||
Reference in New Issue
Block a user