feat: add quick links filtering and frontmatter enrichment on file load

This commit is contained in:
2025-10-19 20:47:18 -04:00
parent 8cccf83f9a
commit 0f7610bed1
51 changed files with 4078 additions and 647 deletions
+248
View File
@@ -0,0 +1,248 @@
#!/usr/bin/env node
/**
* Front-matter enrichment utility for ObsiViewer
* Ensures all markdown files have complete, standardized YAML front-matter
*/
import { promises as fs } from 'fs';
import path from 'path';
import matter from 'gray-matter';
import { Document, parseDocument } from 'yaml';
const TZ_OFFSET = '-04:00'; // America/Toronto
/**
* Mutex map to prevent concurrent writes to the same file
*/
const fileLocks = new Map();
/**
* Acquire a lock for a file path
*/
async function acquireLock(filePath) {
while (fileLocks.has(filePath)) {
await new Promise(resolve => setTimeout(resolve, 10));
}
fileLocks.set(filePath, true);
}
/**
* Release a lock for a file path
*/
function releaseLock(filePath) {
fileLocks.delete(filePath);
}
/**
* Format a date to ISO 8601 with Toronto timezone offset
* @param {Date} date - Date to format
* @returns {string} - ISO 8601 formatted date with timezone
*/
function formatDateISO(date) {
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0');
const day = String(date.getDate()).padStart(2, '0');
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
return `${year}-${month}-${day}T${hours}:${minutes}:${seconds}${TZ_OFFSET}`;
}
/**
* Get current timestamp in ISO 8601 format with timezone
*/
function nowISO() {
return formatDateISO(new Date());
}
/**
* Get file creation date (birthtime or fallback to ctime/mtime)
*/
async function getCreationDate(absPath) {
try {
const stats = await fs.stat(absPath);
// Use birthtime if available (Windows, macOS), otherwise ctime
const creationTime = stats.birthtime && stats.birthtime.getTime() > 0
? stats.birthtime
: (stats.ctime || stats.mtime);
return formatDateISO(creationTime);
} catch (error) {
console.warn(`[ensureFrontmatter] Could not get creation date for ${absPath}:`, error.message);
return nowISO();
}
}
/**
* Enrich front-matter of a markdown file with required properties
*
* @param {string} absPath - Absolute path to the markdown file
* @returns {Promise<{modified: boolean, content: string}>}
*/
export async function enrichFrontmatterOnOpen(absPath) {
await acquireLock(absPath);
try {
// Read file
const raw = await fs.readFile(absPath, 'utf-8');
// Parse front-matter and content (disable date parsing to keep strings)
const parsed = matter(raw, {
language: 'yaml',
delimiters: '---',
engines: {
yaml: {
parse: (str) => {
// Use yaml library directly to preserve types
const doc = parseDocument(str);
return doc.toJSON();
}
}
}
});
// Get file basename without extension
const basename = path.basename(absPath, '.md');
// Get creation date
const creationDate = await getCreationDate(absPath);
const modificationDate = nowISO();
// Define required properties in order
const requiredProps = [
['titre', basename],
['auteur', 'Bruno Charest'],
['creation_date', creationDate],
['modification_date', modificationDate],
['catégorie', ''],
['tags', []],
['aliases', []],
['status', 'en-cours'],
['publish', false],
['favoris', false],
['template', false],
['task', false],
['archive', false],
['draft', false],
['private', false],
];
// Parse existing front-matter data
const existingData = parsed.data || {};
// Track if we inserted any new properties
let inserted = false;
// Build complete data object with defaults for missing keys
const completeData = {};
// First, add all required properties in order
for (const [key, defaultValue] of requiredProps) {
if (existingData.hasOwnProperty(key)) {
// Preserve existing value
completeData[key] = existingData[key];
} else {
// Add default value
completeData[key] = defaultValue;
inserted = true;
}
}
// Update modification_date only if we inserted something
if (inserted) {
completeData['modification_date'] = modificationDate;
}
// Then add any custom properties that exist
for (const key of Object.keys(existingData)) {
if (!completeData.hasOwnProperty(key)) {
completeData[key] = existingData[key];
}
}
// Build new ordered document from complete data
const orderedDoc = new Document();
for (const key of Object.keys(completeData)) {
orderedDoc.set(key, completeData[key]);
}
// Serialize YAML without blank lines
let yamlContent = orderedDoc.toString().trim();
// Remove any blank lines within the YAML
yamlContent = yamlContent.split('\n').filter(line => line.trim() !== '').join('\n');
// Reconstruct the file
const frontmatter = `---\n${yamlContent}\n---`;
const bodyContent = parsed.content;
// Ensure proper spacing after front-matter
const newContent = bodyContent.startsWith('\n')
? `${frontmatter}${bodyContent}`
: `${frontmatter}\n${bodyContent}`;
// Check if content actually changed (compare normalized content)
const modified = raw.trim() !== newContent.trim();
if (modified) {
// Atomic write: temp file + rename
const tempPath = `${absPath}.tmp`;
const backupPath = `${absPath}.bak`;
try {
// Create backup
await fs.copyFile(absPath, backupPath);
// Write to temp
await fs.writeFile(tempPath, newContent, 'utf-8');
// Atomic rename
await fs.rename(tempPath, absPath);
console.log(`[ensureFrontmatter] Enriched: ${path.basename(absPath)}`);
} catch (writeError) {
// Cleanup on error
try {
await fs.unlink(tempPath).catch(() => {});
await fs.copyFile(backupPath, absPath).catch(() => {});
} catch {}
throw writeError;
}
}
return { modified, content: newContent };
} finally {
releaseLock(absPath);
}
}
/**
* Extract front-matter properties from a markdown file
*
* @param {string} absPath - Absolute path to the markdown file
* @returns {Promise<object>} - Front-matter data
*/
export async function extractFrontmatter(absPath) {
try {
const raw = await fs.readFile(absPath, 'utf-8');
const parsed = matter(raw, {
language: 'yaml',
delimiters: '---',
engines: {
yaml: {
parse: (str) => {
// Use yaml library directly to preserve types
const doc = parseDocument(str);
return doc.toJSON();
}
}
}
});
return parsed.data || {};
} catch (error) {
console.warn(`[ensureFrontmatter] Could not extract front-matter from ${absPath}:`, error.message);
return {};
}
}
+324
View File
@@ -0,0 +1,324 @@
#!/usr/bin/env node
/**
* Unit tests for front-matter enrichment
*/
import { promises as fs } from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { enrichFrontmatterOnOpen, extractFrontmatter } from './ensureFrontmatter.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const TEST_DIR = path.join(__dirname, '..', 'test-vault-frontmatter');
// Test utilities
async function setupTestDir() {
await fs.mkdir(TEST_DIR, { recursive: true });
}
async function cleanupTestDir() {
try {
await fs.rm(TEST_DIR, { recursive: true, force: true });
} catch (err) {
console.warn('Cleanup warning:', err.message);
}
}
async function createTestFile(filename, content) {
const filePath = path.join(TEST_DIR, filename);
await fs.writeFile(filePath, content, 'utf-8');
return filePath;
}
// Test runner
const tests = [];
function test(name, fn) {
tests.push({ name, fn });
}
// Tests
test('Should add front-matter to file without any', async () => {
const filePath = await createTestFile('no-frontmatter.md', '# Hello World\n\nThis is content.');
const result = await enrichFrontmatterOnOpen(filePath);
if (!result.modified) {
throw new Error('Expected file to be modified');
}
const content = await fs.readFile(filePath, 'utf-8');
if (!content.startsWith('---\n')) {
throw new Error('Expected front-matter to be added');
}
if (!content.includes('titre: no-frontmatter')) {
throw new Error('Expected titre to be set to filename');
}
if (!content.includes('auteur: Bruno Charest')) {
throw new Error('Expected auteur to be set');
}
if (!content.includes('favoris: false')) {
throw new Error('Expected favoris to be false');
}
if (!content.includes('template: false')) {
throw new Error('Expected template to be false');
}
if (!content.includes('task: false')) {
throw new Error('Expected task to be false');
}
console.log('✓ Front-matter added successfully');
});
test('Should be idempotent (no changes on second run)', async () => {
const filePath = await createTestFile('idempotent.md', '# Test\n\nContent here.');
// First enrichment
const result1 = await enrichFrontmatterOnOpen(filePath);
if (!result1.modified) {
throw new Error('Expected first enrichment to modify file');
}
const content1 = await fs.readFile(filePath, 'utf-8');
// Wait a bit to ensure timestamp would differ if modified
await new Promise(resolve => setTimeout(resolve, 100));
// Second enrichment
const result2 = await enrichFrontmatterOnOpen(filePath);
if (result2.modified) {
throw new Error('Expected second enrichment to NOT modify file (idempotent)');
}
const content2 = await fs.readFile(filePath, 'utf-8');
if (content1 !== content2) {
throw new Error('Expected content to be identical after second enrichment');
}
console.log('✓ Idempotence verified');
});
test('Should preserve existing properties', async () => {
const initialContent = `---
titre: Custom Title
auteur: Bruno Charest
custom_field: custom value
favoris: true
---
# Content`;
const filePath = await createTestFile('preserve.md', initialContent);
const result = await enrichFrontmatterOnOpen(filePath);
const content = await fs.readFile(filePath, 'utf-8');
if (!content.includes('custom_field: custom value')) {
throw new Error('Expected custom field to be preserved');
}
if (!content.includes('favoris: true')) {
throw new Error('Expected favoris: true to be preserved');
}
if (!content.includes('titre: Custom Title')) {
throw new Error('Expected custom title to be preserved');
}
console.log('✓ Existing properties preserved');
});
test('Should maintain correct key order', async () => {
const filePath = await createTestFile('order.md', '# Test\n\nContent.');
await enrichFrontmatterOnOpen(filePath);
const content = await fs.readFile(filePath, 'utf-8');
// Extract front-matter
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
if (!fmMatch) {
throw new Error('No front-matter found');
}
const lines = fmMatch[1].split('\n').filter(l => l.trim());
// Check order of required keys
const expectedOrder = [
'titre:',
'auteur:',
'creation_date:',
'modification_date:',
'catégorie:',
'tags:',
'aliases:',
'status:',
'publish:',
'favoris:',
'template:',
'task:',
'archive:',
'draft:',
'private:'
];
let lastIndex = -1;
for (const key of expectedOrder) {
const currentIndex = lines.findIndex(l => l.startsWith(key));
if (currentIndex === -1) {
throw new Error(`Expected key ${key} not found`);
}
if (currentIndex < lastIndex) {
throw new Error(`Key order incorrect: ${key} appears before previous key`);
}
lastIndex = currentIndex;
}
console.log('✓ Key order is correct');
});
test('Should have no blank lines in front-matter', async () => {
const filePath = await createTestFile('no-blanks.md', '# Test\n\nContent.');
await enrichFrontmatterOnOpen(filePath);
const content = await fs.readFile(filePath, 'utf-8');
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
if (!fmMatch) {
throw new Error('No front-matter found');
}
const fmContent = fmMatch[1];
if (fmContent.includes('\n\n')) {
throw new Error('Found blank lines in front-matter');
}
console.log('✓ No blank lines in front-matter');
});
test('Should use correct boolean types', async () => {
const filePath = await createTestFile('booleans.md', '# Test\n\nContent.');
await enrichFrontmatterOnOpen(filePath);
const fm = await extractFrontmatter(filePath);
if (fm.favoris !== false) {
throw new Error(`Expected favoris to be boolean false, got ${typeof fm.favoris}: ${fm.favoris}`);
}
if (fm.template !== false) {
throw new Error(`Expected template to be boolean false, got ${typeof fm.template}: ${fm.template}`);
}
if (fm.task !== false) {
throw new Error(`Expected task to be boolean false, got ${typeof fm.task}: ${fm.task}`);
}
if (fm.publish !== false) {
throw new Error(`Expected publish to be boolean false, got ${typeof fm.publish}: ${fm.publish}`);
}
console.log('✓ Boolean types are correct');
});
test('Should use correct array types for tags and aliases', async () => {
const filePath = await createTestFile('arrays.md', '# Test\n\nContent.');
await enrichFrontmatterOnOpen(filePath);
const fm = await extractFrontmatter(filePath);
if (!Array.isArray(fm.tags)) {
throw new Error(`Expected tags to be array, got ${typeof fm.tags}`);
}
if (!Array.isArray(fm.aliases)) {
throw new Error(`Expected aliases to be array, got ${typeof fm.aliases}`);
}
if (fm.tags.length !== 0) {
throw new Error(`Expected tags to be empty array, got length ${fm.tags.length}`);
}
if (fm.aliases.length !== 0) {
throw new Error(`Expected aliases to be empty array, got length ${fm.aliases.length}`);
}
console.log('✓ Array types are correct');
});
test('Should format dates in ISO 8601 with timezone', async () => {
const filePath = await createTestFile('dates.md', '# Test\n\nContent.');
await enrichFrontmatterOnOpen(filePath);
const fm = await extractFrontmatter(filePath);
// Check creation_date format
if (!fm.creation_date || typeof fm.creation_date !== 'string') {
throw new Error('Expected creation_date to be a string');
}
if (!fm.creation_date.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$/)) {
throw new Error(`creation_date format incorrect: ${fm.creation_date}`);
}
// Check modification_date format
if (!fm.modification_date || typeof fm.modification_date !== 'string') {
throw new Error('Expected modification_date to be a string');
}
if (!fm.modification_date.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$/)) {
throw new Error(`modification_date format incorrect: ${fm.modification_date}`);
}
console.log('✓ Date formats are correct');
});
// Run all tests
async function runTests() {
console.log('\n🧪 Running front-matter enrichment tests...\n');
await setupTestDir();
let passed = 0;
let failed = 0;
for (const { name, fn } of tests) {
try {
await fn();
passed++;
} catch (err) {
console.error(`✗ ${name}`);
console.error(` ${err.message}`);
failed++;
}
}
await cleanupTestDir();
console.log(`\n📊 Results: ${passed} passed, ${failed} failed\n`);
if (failed > 0) {
process.exit(1);
}
}
// Execute if run directly
if (process.argv[1] === __filename) {
runTests().catch(err => {
console.error('Test runner error:', err);
process.exit(1);
});
}
+88 -18
View File
@@ -18,6 +18,7 @@ import {
isValidExcalidrawScene
} from './excalidraw-obsidian.mjs';
import { rewriteTagsFrontmatter, extractTagsFromFrontmatter } from './markdown-frontmatter.mjs';
import { enrichFrontmatterOnOpen } from './ensureFrontmatter.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -112,10 +113,10 @@ const extractTags = (content) => {
return Array.from(tags);
};
const loadVaultNotes = (vaultPath) => {
const loadVaultNotes = async (vaultPath) => {
const notes = [];
const walk = (currentDir) => {
const walk = async (currentDir) => {
if (!fs.existsSync(currentDir)) {
return;
}
@@ -125,7 +126,7 @@ const loadVaultNotes = (vaultPath) => {
const entryPath = path.join(currentDir, entry.name);
if (entry.isDirectory()) {
walk(entryPath);
await walk(entryPath);
continue;
}
@@ -134,7 +135,11 @@ const loadVaultNotes = (vaultPath) => {
}
try {
const content = fs.readFileSync(entryPath, 'utf-8');
// Enrichir automatiquement le frontmatter lors du chargement
const absPath = entryPath;
const enrichResult = await enrichFrontmatterOnOpen(absPath);
const content = enrichResult.content;
const stats = fs.statSync(entryPath);
const relativePathWithExt = path.relative(vaultPath, entryPath).replace(/\\/g, '/');
const relativePath = relativePathWithExt.replace(/\.md$/i, '');
@@ -160,12 +165,12 @@ const loadVaultNotes = (vaultPath) => {
updatedAt: updatedDate.toISOString()
});
} catch (err) {
console.error(`Failed to read note at ${entryPath}:`, err);
console.error(`Failed to read/enrich note at ${entryPath}:`, err);
}
}
};
walk(vaultPath);
await walk(vaultPath);
return notes;
};
@@ -252,8 +257,19 @@ watchedVaultEvents.forEach((eventName) => {
});
// Integrate Meilisearch with Chokidar for incremental updates
vaultWatcher.on('add', (filePath) => {
vaultWatcher.on('add', async (filePath) => {
if (filePath.toLowerCase().endsWith('.md')) {
// Enrichir le frontmatter pour les nouveaux fichiers
try {
const enrichResult = await enrichFrontmatterOnOpen(filePath);
if (enrichResult.modified) {
console.log('[Watcher] Enriched frontmatter for new file:', path.basename(filePath));
}
} catch (enrichError) {
console.warn('[Watcher] Failed to enrich frontmatter for new file:', enrichError);
}
// Puis indexer dans Meilisearch
upsertFile(filePath).catch(err => console.error('[Meili] Upsert on add failed:', err));
}
});
@@ -419,9 +435,9 @@ app.get('/api/vault/events', (req, res) => {
});
// API endpoint pour les données de la voûte (contenu réel)
app.get('/api/vault', (req, res) => {
app.get('/api/vault', async (req, res) => {
try {
const notes = loadVaultNotes(vaultDir);
const notes = await loadVaultNotes(vaultDir);
res.json({ notes });
} catch (error) {
console.error('Failed to load vault notes:', error);
@@ -452,7 +468,7 @@ app.get('/api/files/list', async (req, res) => {
} catch (error) {
console.error('Failed to list files via Meilisearch, falling back to FS:', error);
try {
const notes = loadVaultNotes(vaultDir);
const notes = await loadVaultNotes(vaultDir);
res.json(buildFileMetadata(notes));
} catch (err2) {
console.error('FS fallback failed:', err2);
@@ -494,7 +510,7 @@ app.get('/api/files/metadata', async (req, res) => {
} catch (error) {
console.error('Failed to load file metadata via Meilisearch, falling back to FS:', error);
try {
const notes = loadVaultNotes(vaultDir);
const notes = await loadVaultNotes(vaultDir);
const base = buildFileMetadata(notes);
const drawings = scanVaultDrawings(vaultDir);
const byPath = new Map(base.map(it => [String(it.path).toLowerCase(), it]));
@@ -510,7 +526,7 @@ app.get('/api/files/metadata', async (req, res) => {
}
});
app.get('/api/files/by-date', (req, res) => {
app.get('/api/files/by-date', async (req, res) => {
const { date } = req.query;
const targetDate = normalizeDateInput(date);
@@ -519,7 +535,7 @@ app.get('/api/files/by-date', (req, res) => {
}
try {
const notes = loadVaultNotes(vaultDir);
const notes = await loadVaultNotes(vaultDir);
const startOfDay = new Date(targetDate);
startOfDay.setHours(0, 0, 0, 0);
const endOfDay = new Date(targetDate);
@@ -540,7 +556,7 @@ app.get('/api/files/by-date', (req, res) => {
}
});
app.get('/api/files/by-date-range', (req, res) => {
app.get('/api/files/by-date-range', async (req, res) => {
const { start, end } = req.query;
const startDate = normalizeDateInput(start);
const endDate = normalizeDateInput(end ?? start);
@@ -555,7 +571,7 @@ app.get('/api/files/by-date-range', (req, res) => {
normalizedEnd.setHours(23, 59, 59, 999);
try {
const notes = loadVaultNotes(vaultDir);
const notes = await loadVaultNotes(vaultDir);
const filtered = notes.filter((note) => {
const createdAt = normalizeDateInput(note.createdAt);
const updatedAt = normalizeDateInput(note.updatedAt);
@@ -671,7 +687,7 @@ function guessContentType(filePath) {
}
// GET file content (supports .excalidraw.md, .excalidraw, .json, .md)
app.get('/api/files', (req, res) => {
app.get('/api/files', async (req, res) => {
try {
const pathParam = req.query.path;
if (!pathParam || typeof pathParam !== 'string') {
@@ -694,6 +710,28 @@ app.get('/api/files', (req, res) => {
return res.status(415).json({ error: 'Unsupported file type' });
}
// For regular markdown files, enrich front-matter before reading
if (!isExcalidraw && ext === '.md') {
try {
const enrichResult = await enrichFrontmatterOnOpen(abs);
// If modified, trigger Meilisearch reindex
if (enrichResult.modified) {
upsertFile(abs).catch(err =>
console.warn('[GET /api/files] Failed to reindex after enrichment:', err)
);
}
const rev = calculateSimpleHash(enrichResult.content);
res.setHeader('ETag', rev);
res.setHeader('Content-Type', guessContentType(abs));
return res.send(enrichResult.content);
} catch (enrichError) {
console.error('[GET /api/files] Front-matter enrichment failed:', enrichError);
// Fallback to reading without enrichment
}
}
const content = fs.readFileSync(abs, 'utf-8');
// For Excalidraw files, parse and return JSON
@@ -717,7 +755,7 @@ app.get('/api/files', (req, res) => {
return res.send(JSON.stringify(normalized));
}
// For regular markdown, return as-is
// For regular markdown, return as-is (fallback)
const rev = calculateSimpleHash(content);
res.setHeader('ETag', rev);
res.setHeader('Content-Type', guessContentType(abs));
@@ -1117,7 +1155,7 @@ app.put(/^\/api\/notes\/(.+?)\/tags$/, express.json(), async (req, res) => {
}
// Find note by id or by path
const notes = loadVaultNotes(vaultDir);
const notes = await loadVaultNotes(vaultDir);
let note = notes.find(n => n.id === noteParam);
if (!note) {
@@ -1219,6 +1257,38 @@ app.post('/api/reindex', async (_req, res) => {
}
});
// Get counts for Quick Links (favorites, templates, tasks, drafts, archive)
app.get('/api/quick-links/counts', async (req, res) => {
try {
const client = meiliClient();
const indexUid = vaultIndexName(vaultDir);
const index = await ensureIndexSettings(client, indexUid);
// Get counts for each filter
const [favoritesResult, templatesResult, tasksResult, draftsResult, archiveResult] = await Promise.all([
index.search('', { filter: 'favoris = true', limit: 0 }),
index.search('', { filter: 'template = true', limit: 0 }),
index.search('', { filter: 'task = true', limit: 0 }),
index.search('', { filter: 'draft = true', limit: 0 }),
index.search('', { filter: 'archive = true', limit: 0 })
]);
res.json({
favorites: favoritesResult.estimatedTotalHits || 0,
templates: templatesResult.estimatedTotalHits || 0,
tasks: tasksResult.estimatedTotalHits || 0,
drafts: draftsResult.estimatedTotalHits || 0,
archive: archiveResult.estimatedTotalHits || 0
});
} catch (error) {
console.error('[Quick Links] Failed to get counts:', error);
res.status(500).json({
error: 'Failed to get counts',
message: error.message
});
}
});
// Servir l'index.html pour toutes les routes (SPA)
const sendIndex = (req, res) => {
const indexPath = path.join(distDir, 'index.html');
+5 -1
View File
@@ -105,7 +105,11 @@ export async function buildDocumentFromFile(absPath) {
year,
month,
parentDirs: parentDirs(rel),
excerpt: text.slice(0, 500)
excerpt: text.slice(0, 500),
// Extract boolean flags for quick filtering
favoris: fm.favoris === true,
template: fm.template === true,
task: fm.task === true
};
}
+4 -1
View File
@@ -82,7 +82,10 @@ export async function ensureIndexSettings(client, indexUid) {
'parentDirs',
'properties.*',
'year',
'month'
'month',
'favoris',
'template',
'task'
],
sortableAttributes: [
'updatedAt',