292 lines
12 KiB
JavaScript
292 lines
12 KiB
JavaScript
/**
|
|
* Phase 3 Patch - Endpoint modifications for caching and monitoring
|
|
*
|
|
* This file contains the updated endpoints that should replace the old ones in index.mjs
|
|
* Apply these changes step by step:
|
|
*
|
|
* 1. Replace /api/vault/metadata endpoint (lines ~500-551)
|
|
* 2. Replace /api/vault/metadata/paginated endpoint (lines ~553-620)
|
|
* 3. Add /__perf endpoint for monitoring (new)
|
|
* 4. Add startup hook for deferred Meilisearch indexing (new)
|
|
*/
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 1: /api/vault/metadata - with cache read-through and monitoring
|
|
// ============================================================================
|
|
export function setupMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, { meiliClient, vaultIndexName, ensureIndexSettings, loadVaultMetadataOnly }) {
|
|
app.get('/api/vault/metadata', async (req, res) => {
|
|
const startTime = performanceMonitor.markRequestStart();
|
|
|
|
try {
|
|
// Use cache.remember() for read-through caching
|
|
const { value: metadata, hit } = await metadataCache.remember(
|
|
`metadata:${vaultDir}`,
|
|
async () => {
|
|
// Try Meilisearch first with circuit breaker
|
|
try {
|
|
return await meilisearchCircuitBreaker.execute(
|
|
async () => {
|
|
const client = meiliClient();
|
|
const indexUid = vaultIndexName(vaultDir);
|
|
const index = await ensureIndexSettings(client, indexUid);
|
|
|
|
const result = await index.search('', {
|
|
limit: 10000,
|
|
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
|
|
});
|
|
|
|
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
|
|
id: hit.id,
|
|
title: hit.title,
|
|
filePath: hit.path,
|
|
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
|
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
|
})) : [];
|
|
|
|
console.log(`[/api/vault/metadata] Loaded ${items.length} items from Meilisearch`);
|
|
return items;
|
|
},
|
|
{
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Meilisearch] Retry attempt ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('meilisearch');
|
|
},
|
|
onCircuitOpen: ({ failureCount }) => {
|
|
console.error(`[Meilisearch] Circuit breaker opened after ${failureCount} failures`);
|
|
}
|
|
}
|
|
);
|
|
} catch (meiliError) {
|
|
console.warn('[Meilisearch] Failed, falling back to filesystem:', meiliError.message);
|
|
|
|
// Fallback to filesystem with retry
|
|
return await retryWithBackoff(
|
|
async () => {
|
|
const notes = await loadVaultMetadataOnly(vaultDir);
|
|
const metadata = notes.map(n => ({
|
|
id: n.id,
|
|
title: n.title,
|
|
filePath: n.filePath,
|
|
createdAt: n.createdAt,
|
|
updatedAt: n.updatedAt
|
|
}));
|
|
|
|
console.log(`[/api/vault/metadata] Loaded ${metadata.length} items from filesystem`);
|
|
return metadata;
|
|
},
|
|
{
|
|
retries: 2,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 500,
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Filesystem] Retry attempt ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('filesystem');
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
|
|
performanceMonitor.markCache(hit);
|
|
const duration = performanceMonitor.markRequestEnd(startTime, true);
|
|
|
|
console.log(`[/api/vault/metadata] ${hit ? 'CACHE HIT' : 'CACHE MISS'} - ${duration}ms`);
|
|
|
|
res.json({
|
|
items: metadata,
|
|
cached: hit,
|
|
duration
|
|
});
|
|
} catch (error) {
|
|
performanceMonitor.markRequestEnd(startTime, false);
|
|
console.error('[/api/vault/metadata] Error:', error);
|
|
res.status(500).json({ error: 'Unable to load vault metadata.' });
|
|
}
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 2: /api/vault/metadata/paginated - with cache and monitoring
|
|
// ============================================================================
|
|
export function setupPaginatedMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, { meiliClient, vaultIndexName, ensureIndexSettings, loadVaultMetadataOnly }) {
|
|
app.get('/api/vault/metadata/paginated', async (req, res) => {
|
|
const startTime = performanceMonitor.markRequestStart();
|
|
|
|
try {
|
|
const limit = Math.min(parseInt(req.query.limit) || 100, 500);
|
|
const cursor = parseInt(req.query.cursor) || 0;
|
|
const search = req.query.search || '';
|
|
|
|
const cacheKey = `paginated:${vaultDir}:${search}`;
|
|
|
|
// For paginated requests, we cache the full result set and paginate client-side
|
|
const { value: allMetadata, hit } = await metadataCache.remember(
|
|
cacheKey,
|
|
async () => {
|
|
try {
|
|
return await meilisearchCircuitBreaker.execute(
|
|
async () => {
|
|
const client = meiliClient();
|
|
const indexUid = vaultIndexName(vaultDir);
|
|
const index = await ensureIndexSettings(client, indexUid);
|
|
|
|
const result = await index.search(search, {
|
|
limit: 10000,
|
|
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt'],
|
|
sort: ['updatedAt:desc']
|
|
});
|
|
|
|
return Array.isArray(result.hits) ? result.hits.map(hit => ({
|
|
id: hit.id,
|
|
title: hit.title,
|
|
filePath: hit.path,
|
|
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
|
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
|
})) : [];
|
|
},
|
|
{
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Meilisearch] Paginated retry ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('meilisearch');
|
|
}
|
|
}
|
|
);
|
|
} catch (meiliError) {
|
|
console.warn('[Meilisearch] Paginated failed, falling back to filesystem:', meiliError.message);
|
|
|
|
return await retryWithBackoff(
|
|
async () => {
|
|
const allMetadata = await loadVaultMetadataOnly(vaultDir);
|
|
|
|
let filtered = allMetadata;
|
|
if (search) {
|
|
const searchLower = search.toLowerCase();
|
|
filtered = allMetadata.filter(item =>
|
|
(item.title || '').toLowerCase().includes(searchLower) ||
|
|
(item.filePath || '').toLowerCase().includes(searchLower)
|
|
);
|
|
}
|
|
|
|
filtered.sort((a, b) => {
|
|
const dateA = new Date(a.updatedAt || a.createdAt || 0).getTime();
|
|
const dateB = new Date(b.updatedAt || b.createdAt || 0).getTime();
|
|
return dateB - dateA;
|
|
});
|
|
|
|
return filtered.map(n => ({
|
|
id: n.id,
|
|
title: n.title,
|
|
filePath: n.filePath,
|
|
createdAt: n.createdAt,
|
|
updatedAt: n.updatedAt
|
|
}));
|
|
},
|
|
{
|
|
retries: 2,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 500,
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Filesystem] Paginated retry ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('filesystem');
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
|
|
// Paginate the cached result
|
|
const paginatedItems = allMetadata.slice(cursor, cursor + limit);
|
|
const hasMore = cursor + limit < allMetadata.length;
|
|
const nextCursor = hasMore ? cursor + limit : null;
|
|
|
|
performanceMonitor.markCache(hit);
|
|
const duration = performanceMonitor.markRequestEnd(startTime, true);
|
|
|
|
console.log(`[/api/vault/metadata/paginated] ${hit ? 'CACHE HIT' : 'CACHE MISS'} - cursor=${cursor}, limit=${limit}, duration=${duration}ms`);
|
|
|
|
res.json({
|
|
items: paginatedItems,
|
|
nextCursor,
|
|
hasMore,
|
|
total: allMetadata.length,
|
|
cached: hit,
|
|
duration
|
|
});
|
|
} catch (error) {
|
|
performanceMonitor.markRequestEnd(startTime, false);
|
|
console.error('[/api/vault/metadata/paginated] Error:', error);
|
|
res.status(500).json({ error: 'Pagination failed' });
|
|
}
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 3: /__perf - Performance monitoring dashboard
|
|
// ============================================================================
|
|
export function setupPerformanceEndpoint(app, performanceMonitor, metadataCache, meilisearchCircuitBreaker) {
|
|
app.get('/__perf', (req, res) => {
|
|
res.json({
|
|
performance: performanceMonitor.snapshot(),
|
|
cache: metadataCache.getStats(),
|
|
circuitBreaker: meilisearchCircuitBreaker.getState(),
|
|
timestamp: new Date().toISOString()
|
|
});
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// STARTUP HOOK: Deferred Meilisearch indexing (non-blocking)
|
|
// ============================================================================
|
|
export async function setupDeferredIndexing(vaultDir, fullReindex) {
|
|
let indexingInProgress = false;
|
|
let indexingCompleted = false;
|
|
let lastIndexingAttempt = 0;
|
|
const INDEXING_COOLDOWN = 5 * 60 * 1000; // 5 minutes
|
|
|
|
async function scheduleIndexing() {
|
|
const now = Date.now();
|
|
|
|
if (indexingInProgress || (now - lastIndexingAttempt) < INDEXING_COOLDOWN) {
|
|
return;
|
|
}
|
|
|
|
indexingInProgress = true;
|
|
lastIndexingAttempt = now;
|
|
|
|
console.log('[Meilisearch] Scheduling background indexing...');
|
|
|
|
// Use setImmediate to not block startup
|
|
setImmediate(async () => {
|
|
try {
|
|
console.time('[Meilisearch] Background indexing');
|
|
|
|
await fullReindex(vaultDir);
|
|
|
|
console.timeEnd('[Meilisearch] Background indexing');
|
|
console.log('[Meilisearch] Background indexing completed successfully');
|
|
|
|
indexingCompleted = true;
|
|
} catch (error) {
|
|
console.error('[Meilisearch] Background indexing failed:', error.message);
|
|
indexingCompleted = false;
|
|
|
|
// Schedule retry in 5 minutes
|
|
setTimeout(() => {
|
|
console.log('[Meilisearch] Retrying indexing in 5 minutes...');
|
|
indexingInProgress = false;
|
|
scheduleIndexing();
|
|
}, INDEXING_COOLDOWN);
|
|
} finally {
|
|
indexingInProgress = false;
|
|
}
|
|
});
|
|
}
|
|
|
|
return {
|
|
scheduleIndexing,
|
|
getState: () => ({ indexingInProgress, indexingCompleted, lastIndexingAttempt })
|
|
};
|
|
}
|