Projets/ObsiViewer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add non-markdown files to metadata endpoints

Browse Source 
- Added support for non-markdown files (images, PDFs, code, etc) in vault metadata endpoints
- Implemented filesystem scanning for non-markdown files with configurable extension whitelist
- Added deduplication by file path to prevent duplicates between Meilisearch and filesystem
- Enhanced folder filtering and kind filtering to properly handle non-markdown files
- Added validation to remove stale entries not present on disk
- Improved logging to track
This commit is contained in:
bruno
2025-10-31 23:20:36 -04:00
parent 545c07a4b3
commit b985a7cdbb
55 changed files with 8465 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/index-phase3-patch.mjs
+160 -18
View File
@@ -15,6 +15,67 @@ import express from 'express';
import fs from 'fs';
import path from 'path';
// Local helpers to include non-markdown files in metadata endpoints
function normalizeString(value = '') {
return String(value)
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.trim();
}
function slugifySegment(segment = '') {
const normalized = normalizeString(segment);
const slug = normalized
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '');
return slug || normalized.toLowerCase() || segment.toLowerCase();
}
function slugifyPath(relativePath = '') {
return String(relativePath)
.split('/')
.map((segment) => slugifySegment(segment))
.filter(Boolean)
.join('/');
}
function scanVaultNonNotes(vaultPath) {
const items = [];
const includeExt = new Set([
'.png','.jpg','.jpeg','.gif','.svg','.webp','.bmp','.ico', // images
'.pdf', // pdf
'.mp4','.mov','.avi','.mkv','.webm', // video
'.json','.js','.ts','.jsx','.tsx','.html','.css','.yml','.yaml','.toml','.ini','.cfg','.conf','.sh','.bash','.ps1','.bat','.csv','.txt','.py' // code/text
]);
const walk = (dir) => {
let entries = [];
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) { walk(entryPath); continue; }
if (!entry.isFile()) continue;
const lower = entry.name.toLowerCase();
if (lower.endsWith('.md') || lower.endsWith('.excalidraw.md')) continue; // markdown handled elsewhere
const ext = path.extname(lower);
if (!includeExt.has(ext)) continue;
try {
const stats = fs.statSync(entryPath);
const relPath = path.relative(vaultPath, entryPath).replace(/\\/g, '/');
const id = slugifyPath(relPath.replace(/\.[^.]+$/i, ''));
const title = path.basename(relPath);
items.push({
id,
title,
path: relPath,
createdAt: new Date(stats.birthtimeMs ? stats.birthtimeMs : stats.ctimeMs).toISOString(),
updatedAt: new Date(stats.mtimeMs).toISOString(),
});
} catch {}
}
};
walk(vaultPath);
return items;
}
// ============================================================================
// ENDPOINT X: /api/files/rename - Rename a markdown file within the same folder
// ============================================================================
@@ -430,7 +491,8 @@ export function setupMetadataEndpoint(app, metadataCache, performanceMonitor, va
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
});
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
// Markdown items from Meili
let items = Array.isArray(result.hits) ? result.hits.map(hit => ({
id: hit.id,
title: hit.title,
filePath: hit.path,
@@ -438,8 +500,30 @@ export function setupMetadataEndpoint(app, metadataCache, performanceMonitor, va
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
})) : [];
console.log(`[/api/vault/metadata] Loaded ${items.length} items from Meilisearch`);
return items;
// Merge non-markdown files from filesystem
const others = scanVaultNonNotes(vaultDir);
items = items.concat(others.map(n => ({
id: n.id,
title: n.title,
filePath: n.path,
createdAt: n.createdAt,
updatedAt: n.updatedAt
})));
// Deduplicate by filePath (case-insensitive)
const byPath = new Map(items.map(it => [String(it.filePath).toLowerCase(), it]));
let deduped = Array.from(byPath.values());
// Drop stale entries not present on filesystem
const before = deduped.length;
deduped = deduped.filter(it => {
try { return fs.existsSync(path.join(vaultDir, String(it.filePath))); } catch { return false; }
});
const removed = before - deduped.length;
if (removed > 0) {
console.warn(`[/api/vault/metadata] Removed ${removed} stale items not found on disk`);
}
console.log(`[/api/vault/metadata] Loaded ${items.length} items from Meilisearch (dedup -> ${deduped.length})`);
return deduped;
},
{
onRetry: ({ attempt, delay, err }) => {
@@ -458,16 +542,36 @@ export function setupMetadataEndpoint(app, metadataCache, performanceMonitor, va
return await retryWithBackoff(
async () => {
const notes = await loadVaultMetadataOnly(vaultDir);
const metadata = notes.map(n => ({
const mdItems = notes.map(n => ({
id: n.id,
title: n.title,
filePath: n.filePath,
createdAt: n.createdAt,
updatedAt: n.updatedAt
}));
const others = scanVaultNonNotes(vaultDir);
const merged = mdItems.concat(others.map(n => ({
id: n.id,
title: n.title,
filePath: n.path,
createdAt: n.createdAt,
updatedAt: n.updatedAt
})));
console.log(`[/api/vault/metadata] Loaded ${metadata.length} items from filesystem`);
return metadata;
// Deduplicate by filePath (case-insensitive)
const byPath = new Map(merged.map(it => [String(it.filePath).toLowerCase(), it]));
let deduped = Array.from(byPath.values());
// Drop stale entries not present on filesystem (should be none in FS branch, but safe)
const before = deduped.length;
deduped = deduped.filter(it => {
try { return fs.existsSync(path.join(vaultDir, String(it.filePath))); } catch { return false; }
});
const removed = before - deduped.length;
if (removed > 0) {
console.warn(`[/api/vault/metadata] [FS] Removed ${removed} stale items not found on disk`);
}
console.log(`[/api/vault/metadata] Loaded ${merged.length} items from filesystem (md + non-md, dedup -> ${deduped.length})`);
return deduped;
},
{
retries: 2,
@@ -532,13 +636,42 @@ export function setupPaginatedMetadataEndpoint(app, metadataCache, performanceMo
sort: ['updatedAt:desc']
});
return Array.isArray(result.hits) ? result.hits.map(hit => ({
// Markdown results from Meilisearch
const mdItems = Array.isArray(result.hits) ? result.hits.map(hit => ({
id: hit.id,
title: hit.title,
filePath: hit.path,
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
})) : [];
// Merge non-markdown files from filesystem
let others = scanVaultNonNotes(vaultDir);
if (search) {
const s = String(search).toLowerCase();
others = others.filter(it => (it.title || '').toLowerCase().includes(s) || (it.path || '').toLowerCase().includes(s));
}
let merged = mdItems.concat(others.map(n => ({
id: n.id,
title: n.title,
filePath: n.path,
createdAt: n.createdAt,
updatedAt: n.updatedAt
})));
// Dedup by filePath and drop stale
const byPath = new Map(merged.map(it => [String(it.filePath).toLowerCase(), it]));
let deduped = Array.from(byPath.values());
const before = deduped.length;
deduped = deduped.filter(it => {
try { return fs.existsSync(path.join(vaultDir, String(it.filePath))); } catch { return false; }
});
const removed = before - deduped.length;
if (removed > 0) {
console.warn(`[/api/vault/metadata/paginated] Removed ${removed} stale items not found on disk`);
}
// Ensure updatedAt desc
deduped.sort((a, b) => new Date(b.updatedAt || b.createdAt || 0) - new Date(a.updatedAt || a.createdAt || 0));
return deduped;
},
{
onRetry: ({ attempt, delay, err }) => {
@@ -552,30 +685,39 @@ export function setupPaginatedMetadataEndpoint(app, metadataCache, performanceMo
return await retryWithBackoff(
async () => {
const allMetadata = await loadVaultMetadataOnly(vaultDir);
const allMarkdown = await loadVaultMetadataOnly(vaultDir);
const others = scanVaultNonNotes(vaultDir);
let combined = allMarkdown.map(n => ({ id: n.id, title: n.title, filePath: n.filePath, createdAt: n.createdAt, updatedAt: n.updatedAt }))
.concat(others.map(n => ({ id: n.id, title: n.title, filePath: n.path, createdAt: n.createdAt, updatedAt: n.updatedAt })));
let filtered = allMetadata;
let filtered = combined;
if (search) {
const searchLower = search.toLowerCase();
filtered = allMetadata.filter(item =>
filtered = combined.filter(item =>
(item.title || '').toLowerCase().includes(searchLower) ||
(item.filePath || '').toLowerCase().includes(searchLower)
);
}
filtered.sort((a, b) => {
// Dedup by filePath and drop stale
const byPathFs = new Map(filtered.map(it => [String(it.filePath).toLowerCase(), it]));
let dedupedFs = Array.from(byPathFs.values());
const beforeFs = dedupedFs.length;
dedupedFs = dedupedFs.filter(it => {
try { return fs.existsSync(path.join(vaultDir, String(it.filePath))); } catch { return false; }
});
const removedFs = beforeFs - dedupedFs.length;
if (removedFs > 0) {
console.warn(`[/api/vault/metadata/paginated][FS] Removed ${removedFs} stale items not found on disk`);
}
dedupedFs.sort((a, b) => {
const dateA = new Date(a.updatedAt || a.createdAt || 0).getTime();
const dateB = new Date(b.updatedAt || b.createdAt || 0).getTime();
return dateB - dateA;
});
return filtered.map(n => ({
id: n.id,
title: n.title,
filePath: n.filePath,
createdAt: n.createdAt,
updatedAt: n.updatedAt
}));
return dedupedFs.map(n => ({ id: n.id, title: n.title, filePath: n.filePath, createdAt: n.createdAt, updatedAt: n.updatedAt }));
},
{
retries: 2,
server/index.mjs
+13 -13
View File
@@ -418,7 +418,7 @@ const scanVaultNonNotes = (vaultPath) => {
// video
'.mp4','.mov','.avi','.mkv','.webm',
// code/text configs
'.json','.js','.ts','.jsx','.tsx','.html','.css','.yml','.yaml','.toml','.ini','.cfg','.conf','.sh','.bash','.ps1','.bat','.csv','.txt'
'.json','.js','.ts','.jsx','.tsx','.html','.css','.yml','.yaml','.toml','.ini','.cfg','.conf','.sh','.bash','.ps1','.bat','.csv','.txt','.py'
]);
const walk = (dir) => {
@@ -767,20 +767,20 @@ app.get('/api/files/list', async (req, res) => {
});
// Phase 3: Fast metadata endpoint with cache read-through and monitoring
// setupMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, {
// meiliClient,
// vaultIndexName,
// ensureIndexSettings,
// loadVaultMetadataOnly
// });
setupMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, {
meiliClient,
vaultIndexName,
ensureIndexSettings,
loadVaultMetadataOnly
});
// Phase 3: Paginated metadata endpoint with cache read-through and monitoring
// setupPaginatedMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, {
// meiliClient,
// vaultIndexName,
// ensureIndexSettings,
// loadVaultMetadataOnly
// });
setupPaginatedMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, {
meiliClient,
vaultIndexName,
ensureIndexSettings,
loadVaultMetadataOnly
});
app.get('/api/files/metadata', async (req, res) => {
try {