docs: remove Phase 2 completion and executive summary files
This commit is contained in:
@@ -0,0 +1,291 @@
|
||||
/**
|
||||
* Phase 3 Patch - Endpoint modifications for caching and monitoring
|
||||
*
|
||||
* This file contains the updated endpoints that should replace the old ones in index.mjs
|
||||
* Apply these changes step by step:
|
||||
*
|
||||
* 1. Replace /api/vault/metadata endpoint (lines ~500-551)
|
||||
* 2. Replace /api/vault/metadata/paginated endpoint (lines ~553-620)
|
||||
* 3. Add /__perf endpoint for monitoring (new)
|
||||
* 4. Add startup hook for deferred Meilisearch indexing (new)
|
||||
*/
|
||||
|
||||
// ============================================================================
|
||||
// ENDPOINT 1: /api/vault/metadata - with cache read-through and monitoring
|
||||
// ============================================================================
|
||||
export function setupMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, { meiliClient, vaultIndexName, ensureIndexSettings, loadVaultMetadataOnly }) {
|
||||
app.get('/api/vault/metadata', async (req, res) => {
|
||||
const startTime = performanceMonitor.markRequestStart();
|
||||
|
||||
try {
|
||||
// Use cache.remember() for read-through caching
|
||||
const { value: metadata, hit } = await metadataCache.remember(
|
||||
`metadata:${vaultDir}`,
|
||||
async () => {
|
||||
// Try Meilisearch first with circuit breaker
|
||||
try {
|
||||
return await meilisearchCircuitBreaker.execute(
|
||||
async () => {
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
|
||||
const result = await index.search('', {
|
||||
limit: 10000,
|
||||
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
|
||||
});
|
||||
|
||||
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
|
||||
id: hit.id,
|
||||
title: hit.title,
|
||||
filePath: hit.path,
|
||||
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
||||
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
||||
})) : [];
|
||||
|
||||
console.log(`[/api/vault/metadata] Loaded ${items.length} items from Meilisearch`);
|
||||
return items;
|
||||
},
|
||||
{
|
||||
onRetry: ({ attempt, delay, err }) => {
|
||||
console.warn(`[Meilisearch] Retry attempt ${attempt}, delay ${delay}ms:`, err.message);
|
||||
performanceMonitor.markRetry('meilisearch');
|
||||
},
|
||||
onCircuitOpen: ({ failureCount }) => {
|
||||
console.error(`[Meilisearch] Circuit breaker opened after ${failureCount} failures`);
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (meiliError) {
|
||||
console.warn('[Meilisearch] Failed, falling back to filesystem:', meiliError.message);
|
||||
|
||||
// Fallback to filesystem with retry
|
||||
return await retryWithBackoff(
|
||||
async () => {
|
||||
const notes = await loadVaultMetadataOnly(vaultDir);
|
||||
const metadata = notes.map(n => ({
|
||||
id: n.id,
|
||||
title: n.title,
|
||||
filePath: n.filePath,
|
||||
createdAt: n.createdAt,
|
||||
updatedAt: n.updatedAt
|
||||
}));
|
||||
|
||||
console.log(`[/api/vault/metadata] Loaded ${metadata.length} items from filesystem`);
|
||||
return metadata;
|
||||
},
|
||||
{
|
||||
retries: 2,
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 500,
|
||||
onRetry: ({ attempt, delay, err }) => {
|
||||
console.warn(`[Filesystem] Retry attempt ${attempt}, delay ${delay}ms:`, err.message);
|
||||
performanceMonitor.markRetry('filesystem');
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
performanceMonitor.markCache(hit);
|
||||
const duration = performanceMonitor.markRequestEnd(startTime, true);
|
||||
|
||||
console.log(`[/api/vault/metadata] ${hit ? 'CACHE HIT' : 'CACHE MISS'} - ${duration}ms`);
|
||||
|
||||
res.json({
|
||||
items: metadata,
|
||||
cached: hit,
|
||||
duration
|
||||
});
|
||||
} catch (error) {
|
||||
performanceMonitor.markRequestEnd(startTime, false);
|
||||
console.error('[/api/vault/metadata] Error:', error);
|
||||
res.status(500).json({ error: 'Unable to load vault metadata.' });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ENDPOINT 2: /api/vault/metadata/paginated - with cache and monitoring
|
||||
// ============================================================================
|
||||
export function setupPaginatedMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, { meiliClient, vaultIndexName, ensureIndexSettings, loadVaultMetadataOnly }) {
|
||||
app.get('/api/vault/metadata/paginated', async (req, res) => {
|
||||
const startTime = performanceMonitor.markRequestStart();
|
||||
|
||||
try {
|
||||
const limit = Math.min(parseInt(req.query.limit) || 100, 500);
|
||||
const cursor = parseInt(req.query.cursor) || 0;
|
||||
const search = req.query.search || '';
|
||||
|
||||
const cacheKey = `paginated:${vaultDir}:${search}`;
|
||||
|
||||
// For paginated requests, we cache the full result set and paginate client-side
|
||||
const { value: allMetadata, hit } = await metadataCache.remember(
|
||||
cacheKey,
|
||||
async () => {
|
||||
try {
|
||||
return await meilisearchCircuitBreaker.execute(
|
||||
async () => {
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
|
||||
const result = await index.search(search, {
|
||||
limit: 10000,
|
||||
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt'],
|
||||
sort: ['updatedAt:desc']
|
||||
});
|
||||
|
||||
return Array.isArray(result.hits) ? result.hits.map(hit => ({
|
||||
id: hit.id,
|
||||
title: hit.title,
|
||||
filePath: hit.path,
|
||||
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
||||
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
||||
})) : [];
|
||||
},
|
||||
{
|
||||
onRetry: ({ attempt, delay, err }) => {
|
||||
console.warn(`[Meilisearch] Paginated retry ${attempt}, delay ${delay}ms:`, err.message);
|
||||
performanceMonitor.markRetry('meilisearch');
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (meiliError) {
|
||||
console.warn('[Meilisearch] Paginated failed, falling back to filesystem:', meiliError.message);
|
||||
|
||||
return await retryWithBackoff(
|
||||
async () => {
|
||||
const allMetadata = await loadVaultMetadataOnly(vaultDir);
|
||||
|
||||
let filtered = allMetadata;
|
||||
if (search) {
|
||||
const searchLower = search.toLowerCase();
|
||||
filtered = allMetadata.filter(item =>
|
||||
(item.title || '').toLowerCase().includes(searchLower) ||
|
||||
(item.filePath || '').toLowerCase().includes(searchLower)
|
||||
);
|
||||
}
|
||||
|
||||
filtered.sort((a, b) => {
|
||||
const dateA = new Date(a.updatedAt || a.createdAt || 0).getTime();
|
||||
const dateB = new Date(b.updatedAt || b.createdAt || 0).getTime();
|
||||
return dateB - dateA;
|
||||
});
|
||||
|
||||
return filtered.map(n => ({
|
||||
id: n.id,
|
||||
title: n.title,
|
||||
filePath: n.filePath,
|
||||
createdAt: n.createdAt,
|
||||
updatedAt: n.updatedAt
|
||||
}));
|
||||
},
|
||||
{
|
||||
retries: 2,
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 500,
|
||||
onRetry: ({ attempt, delay, err }) => {
|
||||
console.warn(`[Filesystem] Paginated retry ${attempt}, delay ${delay}ms:`, err.message);
|
||||
performanceMonitor.markRetry('filesystem');
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Paginate the cached result
|
||||
const paginatedItems = allMetadata.slice(cursor, cursor + limit);
|
||||
const hasMore = cursor + limit < allMetadata.length;
|
||||
const nextCursor = hasMore ? cursor + limit : null;
|
||||
|
||||
performanceMonitor.markCache(hit);
|
||||
const duration = performanceMonitor.markRequestEnd(startTime, true);
|
||||
|
||||
console.log(`[/api/vault/metadata/paginated] ${hit ? 'CACHE HIT' : 'CACHE MISS'} - cursor=${cursor}, limit=${limit}, duration=${duration}ms`);
|
||||
|
||||
res.json({
|
||||
items: paginatedItems,
|
||||
nextCursor,
|
||||
hasMore,
|
||||
total: allMetadata.length,
|
||||
cached: hit,
|
||||
duration
|
||||
});
|
||||
} catch (error) {
|
||||
performanceMonitor.markRequestEnd(startTime, false);
|
||||
console.error('[/api/vault/metadata/paginated] Error:', error);
|
||||
res.status(500).json({ error: 'Pagination failed' });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ENDPOINT 3: /__perf - Performance monitoring dashboard
|
||||
// ============================================================================
|
||||
export function setupPerformanceEndpoint(app, performanceMonitor, metadataCache, meilisearchCircuitBreaker) {
|
||||
app.get('/__perf', (req, res) => {
|
||||
res.json({
|
||||
performance: performanceMonitor.snapshot(),
|
||||
cache: metadataCache.getStats(),
|
||||
circuitBreaker: meilisearchCircuitBreaker.getState(),
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// STARTUP HOOK: Deferred Meilisearch indexing (non-blocking)
|
||||
// ============================================================================
|
||||
export async function setupDeferredIndexing(vaultDir, fullReindex) {
|
||||
let indexingInProgress = false;
|
||||
let indexingCompleted = false;
|
||||
let lastIndexingAttempt = 0;
|
||||
const INDEXING_COOLDOWN = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
async function scheduleIndexing() {
|
||||
const now = Date.now();
|
||||
|
||||
if (indexingInProgress || (now - lastIndexingAttempt) < INDEXING_COOLDOWN) {
|
||||
return;
|
||||
}
|
||||
|
||||
indexingInProgress = true;
|
||||
lastIndexingAttempt = now;
|
||||
|
||||
console.log('[Meilisearch] Scheduling background indexing...');
|
||||
|
||||
// Use setImmediate to not block startup
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
console.time('[Meilisearch] Background indexing');
|
||||
|
||||
await fullReindex(vaultDir);
|
||||
|
||||
console.timeEnd('[Meilisearch] Background indexing');
|
||||
console.log('[Meilisearch] Background indexing completed successfully');
|
||||
|
||||
indexingCompleted = true;
|
||||
} catch (error) {
|
||||
console.error('[Meilisearch] Background indexing failed:', error.message);
|
||||
indexingCompleted = false;
|
||||
|
||||
// Schedule retry in 5 minutes
|
||||
setTimeout(() => {
|
||||
console.log('[Meilisearch] Retrying indexing in 5 minutes...');
|
||||
indexingInProgress = false;
|
||||
scheduleIndexing();
|
||||
}, INDEXING_COOLDOWN);
|
||||
} finally {
|
||||
indexingInProgress = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
scheduleIndexing,
|
||||
getState: () => ({ indexingInProgress, indexingCompleted, lastIndexingAttempt })
|
||||
};
|
||||
}
|
||||
+186
-201
@@ -20,7 +20,16 @@ import {
|
||||
import { rewriteTagsFrontmatter, extractTagsFromFrontmatter } from './markdown-frontmatter.mjs';
|
||||
import { enrichFrontmatterOnOpen } from './ensureFrontmatter.mjs';
|
||||
import { loadVaultMetadataOnly } from './vault-metadata-loader.mjs';
|
||||
import { MetadataCache, PerformanceLogger } from './performance-config.mjs';
|
||||
import { MetadataCache as MetadataCacheOld, PerformanceLogger } from './performance-config.mjs';
|
||||
import { MetadataCache } from './perf/metadata-cache.js';
|
||||
import { PerformanceMonitor } from './perf/performance-monitor.js';
|
||||
import { retryWithBackoff, CircuitBreaker } from './utils/retry.js';
|
||||
import {
|
||||
setupMetadataEndpoint,
|
||||
setupPaginatedMetadataEndpoint,
|
||||
setupPerformanceEndpoint,
|
||||
setupDeferredIndexing
|
||||
} from './index-phase3-patch.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
@@ -34,19 +43,24 @@ const distDir = path.join(rootDir, 'dist');
|
||||
const vaultDir = path.isAbsolute(CFG_VAULT_PATH) ? CFG_VAULT_PATH : path.join(rootDir, CFG_VAULT_PATH);
|
||||
|
||||
const vaultEventClients = new Set();
|
||||
const metadataCache = new MetadataCache();
|
||||
|
||||
// Phase 3: Advanced caching and monitoring
|
||||
const metadataCache = new MetadataCache({ ttlMs: 5 * 60 * 1000, maxItems: 10_000 });
|
||||
const performanceMonitor = new PerformanceMonitor();
|
||||
const meilisearchCircuitBreaker = new CircuitBreaker({ failureThreshold: 5, resetTimeoutMs: 30_000 });
|
||||
|
||||
const registerVaultEventClient = (res) => {
|
||||
const heartbeat = setInterval(() => {
|
||||
try {
|
||||
res.write(':keepalive\n\n');
|
||||
} catch {
|
||||
// Write failures will be handled by the close handler.
|
||||
} catch (error) {
|
||||
// Client disconnected, clean up
|
||||
console.log('[SSE] Client heartbeat failed, cleaning up');
|
||||
unregisterVaultEventClient({ res, heartbeat });
|
||||
}
|
||||
}, 20000);
|
||||
}, 20000); // Send heartbeat every 20 seconds
|
||||
|
||||
const client = { res, heartbeat };
|
||||
// moved scanVaultDrawings to top-level
|
||||
vaultEventClients.add(client);
|
||||
return client;
|
||||
};
|
||||
@@ -261,8 +275,8 @@ watchedVaultEvents.forEach((eventName) => {
|
||||
// Integrate Meilisearch with Chokidar for incremental updates
|
||||
vaultWatcher.on('add', async (filePath) => {
|
||||
if (filePath.toLowerCase().endsWith('.md')) {
|
||||
// Invalidate metadata cache (Phase 1)
|
||||
metadataCache.invalidate();
|
||||
// Clear metadata cache (Phase 1)
|
||||
metadataCache.clear();
|
||||
|
||||
// Enrichir le frontmatter pour les nouveaux fichiers
|
||||
try {
|
||||
@@ -281,8 +295,8 @@ vaultWatcher.on('add', async (filePath) => {
|
||||
|
||||
vaultWatcher.on('change', (filePath) => {
|
||||
if (filePath.toLowerCase().endsWith('.md')) {
|
||||
// Invalidate metadata cache (Phase 1)
|
||||
metadataCache.invalidate();
|
||||
// Clear metadata cache (Phase 1)
|
||||
metadataCache.clear();
|
||||
|
||||
upsertFile(filePath).catch(err => console.error('[Meili] Upsert on change failed:', err));
|
||||
}
|
||||
@@ -290,8 +304,8 @@ vaultWatcher.on('change', (filePath) => {
|
||||
|
||||
vaultWatcher.on('unlink', (filePath) => {
|
||||
if (filePath.toLowerCase().endsWith('.md')) {
|
||||
// Invalidate metadata cache (Phase 1)
|
||||
metadataCache.invalidate();
|
||||
// Clear metadata cache (Phase 1)
|
||||
metadataCache.clear();
|
||||
|
||||
const relativePath = path.relative(vaultDir, filePath).replace(/\\/g, '/');
|
||||
deleteFile(relativePath).catch(err => console.error('[Meili] Delete failed:', err));
|
||||
@@ -488,145 +502,20 @@ app.get('/api/files/list', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Fast metadata endpoint - no content, no enrichment (Phase 1 optimization)
|
||||
// Used for initial UI load to minimize startup time
|
||||
app.get('/api/vault/metadata', async (req, res) => {
|
||||
try {
|
||||
// Check cache first
|
||||
const cached = metadataCache.get();
|
||||
if (cached) {
|
||||
console.log('[/api/vault/metadata] Returning cached metadata');
|
||||
return res.json(cached);
|
||||
}
|
||||
|
||||
// Try Meilisearch first (already indexed)
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
|
||||
const result = await index.search('', {
|
||||
limit: 10000,
|
||||
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
|
||||
});
|
||||
|
||||
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
|
||||
id: hit.id,
|
||||
title: hit.title,
|
||||
filePath: hit.path,
|
||||
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
||||
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
||||
})) : [];
|
||||
|
||||
metadataCache.set(items);
|
||||
console.log(`[/api/vault/metadata] Returned ${items.length} items from Meilisearch`);
|
||||
res.json(items);
|
||||
} catch (error) {
|
||||
console.error('Failed to load metadata via Meilisearch, falling back to FS:', error);
|
||||
try {
|
||||
// Fallback: fast filesystem scan without enrichment
|
||||
const notes = await loadVaultMetadataOnly(vaultDir);
|
||||
const metadata = notes.map(n => ({
|
||||
id: n.id,
|
||||
title: n.title,
|
||||
filePath: n.filePath,
|
||||
createdAt: n.createdAt,
|
||||
updatedAt: n.updatedAt
|
||||
}));
|
||||
|
||||
metadataCache.set(metadata);
|
||||
console.log(`[/api/vault/metadata] Returned ${metadata.length} items from filesystem`);
|
||||
res.json(metadata);
|
||||
} catch (err2) {
|
||||
console.error('FS fallback failed:', err2);
|
||||
res.status(500).json({ error: 'Unable to load vault metadata.' });
|
||||
}
|
||||
}
|
||||
// Phase 3: Fast metadata endpoint with cache read-through and monitoring
|
||||
setupMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, {
|
||||
meiliClient,
|
||||
vaultIndexName,
|
||||
ensureIndexSettings,
|
||||
loadVaultMetadataOnly
|
||||
});
|
||||
|
||||
// Paginated metadata endpoint - supports cursor-based pagination for large vaults
|
||||
// Used for virtual scrolling with 10,000+ files
|
||||
app.get('/api/vault/metadata/paginated', async (req, res) => {
|
||||
try {
|
||||
const limit = Math.min(parseInt(req.query.limit) || 100, 500);
|
||||
const cursor = parseInt(req.query.cursor) || 0;
|
||||
const search = req.query.search || '';
|
||||
|
||||
console.time(`[/api/vault/metadata/paginated] Load page cursor=${cursor}, limit=${limit}`);
|
||||
|
||||
// Try Meilisearch first (already indexed)
|
||||
const client = meiliClient();
|
||||
const indexUid = vaultIndexName(vaultDir);
|
||||
const index = await ensureIndexSettings(client, indexUid);
|
||||
|
||||
const result = await index.search(search, {
|
||||
limit: limit + 1, // +1 to detect if there are more results
|
||||
offset: cursor,
|
||||
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt'],
|
||||
sort: ['updatedAt:desc'] // Sort by modification date
|
||||
});
|
||||
|
||||
const hasMore = result.hits.length > limit;
|
||||
const items = result.hits.slice(0, limit);
|
||||
const nextCursor = hasMore ? cursor + limit : null;
|
||||
|
||||
// Convert to NoteMetadata format
|
||||
const metadata = items.map(hit => ({
|
||||
id: hit.id,
|
||||
title: hit.title,
|
||||
filePath: hit.path,
|
||||
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
||||
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
||||
}));
|
||||
|
||||
console.timeEnd(`[/api/vault/metadata/paginated] Load page cursor=${cursor}, limit=${limit}`);
|
||||
|
||||
res.json({
|
||||
items: metadata,
|
||||
nextCursor,
|
||||
hasMore,
|
||||
total: result.estimatedTotalHits || result.hits.length
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[/api/vault/metadata/paginated] Meilisearch error:', error);
|
||||
|
||||
// Fallback: simple pagination on filesystem
|
||||
try {
|
||||
const limit = Math.min(parseInt(req.query.limit) || 100, 500);
|
||||
const cursor = parseInt(req.query.cursor) || 0;
|
||||
const search = (req.query.search || '').toLowerCase();
|
||||
|
||||
const allMetadata = await loadVaultMetadataOnly(vaultDir);
|
||||
|
||||
// Filter by search term if provided
|
||||
let filtered = allMetadata;
|
||||
if (search) {
|
||||
filtered = allMetadata.filter(item =>
|
||||
(item.title || '').toLowerCase().includes(search) ||
|
||||
(item.filePath || '').toLowerCase().includes(search)
|
||||
);
|
||||
}
|
||||
|
||||
// Sort by updatedAt descending
|
||||
filtered.sort((a, b) => {
|
||||
const dateA = new Date(a.updatedAt || a.createdAt || 0).getTime();
|
||||
const dateB = new Date(b.updatedAt || b.createdAt || 0).getTime();
|
||||
return dateB - dateA;
|
||||
});
|
||||
|
||||
const paginatedItems = filtered.slice(cursor, cursor + limit);
|
||||
const hasMore = cursor + limit < filtered.length;
|
||||
|
||||
res.json({
|
||||
items: paginatedItems,
|
||||
nextCursor: hasMore ? cursor + limit : null,
|
||||
hasMore,
|
||||
total: filtered.length
|
||||
});
|
||||
} catch (fallbackError) {
|
||||
console.error('[/api/vault/metadata/paginated] Fallback error:', fallbackError);
|
||||
res.status(500).json({ error: 'Pagination failed' });
|
||||
}
|
||||
}
|
||||
// Phase 3: Paginated metadata endpoint with cache read-through and monitoring
|
||||
setupPaginatedMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, {
|
||||
meiliClient,
|
||||
vaultIndexName,
|
||||
ensureIndexSettings,
|
||||
loadVaultMetadataOnly
|
||||
});
|
||||
|
||||
app.get('/api/files/metadata', async (req, res) => {
|
||||
@@ -752,12 +641,20 @@ app.post('/api/log', (req, res) => {
|
||||
|
||||
const records = Array.isArray(payload) ? payload : [payload];
|
||||
|
||||
records.forEach((record) => {
|
||||
// Validate and process records
|
||||
const validRecords = records.filter((record) => {
|
||||
if (!record || typeof record !== 'object') {
|
||||
console.warn('[FrontendLog] Ignored invalid record', record);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (validRecords.length === 0) {
|
||||
return res.status(400).json({ error: 'No valid log records provided' });
|
||||
}
|
||||
|
||||
validRecords.forEach((record) => {
|
||||
const {
|
||||
event = 'UNKNOWN_EVENT',
|
||||
level = 'info',
|
||||
@@ -781,35 +678,55 @@ app.post('/api/log', (req, res) => {
|
||||
console.log(`[FrontendLog:${level}]`, event, summary, userAgent ?? '');
|
||||
});
|
||||
|
||||
return res.status(202).json({ ok: true });
|
||||
return res.status(202).json({ ok: true, processed: validRecords.length });
|
||||
} catch (error) {
|
||||
console.error('Failed to process frontend logs:', error);
|
||||
return res.status(500).json({ error: 'Failed to process logs' });
|
||||
return res.status(500).json({
|
||||
error: 'Failed to process logs',
|
||||
message: error.message || 'Internal server error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/api/logs', (req, res) => {
|
||||
const { source = 'frontend', level = 'info', message = '', data = null, timestamp = Date.now() } = req.body || {};
|
||||
try {
|
||||
const { source = 'frontend', level = 'info', message = '', data = null, timestamp = Date.now() } = req.body || {};
|
||||
|
||||
const prefix = `[ClientLog:${source}]`;
|
||||
const payload = data !== undefined ? { data } : undefined;
|
||||
// Validate inputs
|
||||
if (!message || typeof message !== 'string') {
|
||||
return res.status(400).json({ error: 'Invalid or missing message' });
|
||||
}
|
||||
|
||||
switch (level) {
|
||||
case 'error':
|
||||
console.error(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
case 'warn':
|
||||
console.warn(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
case 'debug':
|
||||
console.debug(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
default:
|
||||
console.log(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
if (!['error', 'warn', 'info', 'debug'].includes(level)) {
|
||||
return res.status(400).json({ error: 'Invalid log level' });
|
||||
}
|
||||
|
||||
const prefix = `[ClientLog:${source}]`;
|
||||
const payload = data !== undefined ? { data } : undefined;
|
||||
|
||||
switch (level) {
|
||||
case 'error':
|
||||
console.error(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
case 'warn':
|
||||
console.warn(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
case 'debug':
|
||||
console.debug(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
default:
|
||||
console.log(prefix, message, payload ?? '', new Date(timestamp).toISOString());
|
||||
break;
|
||||
}
|
||||
|
||||
res.status(202).json({ status: 'queued' });
|
||||
} catch (error) {
|
||||
console.error('Failed to process client logs:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to process logs',
|
||||
message: error.message || 'Internal server error'
|
||||
});
|
||||
}
|
||||
|
||||
res.status(202).json({ status: 'queued' });
|
||||
});
|
||||
|
||||
// --- Files API (supports .excalidraw.md (Markdown-wrapped JSON), .excalidraw, .json and binary sidecars) ---
|
||||
@@ -925,43 +842,59 @@ app.put('/api/files', express.json({ limit: '10mb' }), express.text({ limit: '10
|
||||
try {
|
||||
const pathParam = req.query.path;
|
||||
if (!pathParam || typeof pathParam !== 'string') {
|
||||
return res.status(400).json({ error: 'Missing path query parameter' });
|
||||
return res.status(400).json({ error: 'Missing or invalid path query parameter' });
|
||||
}
|
||||
|
||||
|
||||
const rel = decodeURIComponent(pathParam);
|
||||
const abs = resolveVaultPath(rel);
|
||||
|
||||
// Check if the path is a directory (reject directory operations)
|
||||
if (fs.existsSync(abs) && fs.statSync(abs).isDirectory()) {
|
||||
return res.status(400).json({ error: 'Cannot write to directory path. Specify a file path.' });
|
||||
}
|
||||
|
||||
const dir = path.dirname(abs);
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
if (!fs.existsSync(dir)) {
|
||||
// Create parent directories if they don't exist
|
||||
try {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
} catch (mkdirError) {
|
||||
console.error('[PUT /api/files] Failed to create directory:', mkdirError);
|
||||
return res.status(500).json({ error: 'Failed to create parent directories' });
|
||||
}
|
||||
}
|
||||
|
||||
const contentType = (req.headers['content-type'] || '').split(';')[0];
|
||||
const base = path.basename(abs).toLowerCase();
|
||||
const isExcalidrawMd = base.endsWith('.excalidraw.md');
|
||||
|
||||
|
||||
let finalContent;
|
||||
let existingFrontMatter = null;
|
||||
console.log('[PUT /api/files] path=%s contentType=%s isExcalidrawMd=%s', rel, contentType, isExcalidrawMd);
|
||||
|
||||
|
||||
// Extract existing front matter if file exists
|
||||
if (fs.existsSync(abs) && isExcalidrawMd) {
|
||||
try {
|
||||
const existing = fs.readFileSync(abs, 'utf-8');
|
||||
existingFrontMatter = extractFrontMatter(existing);
|
||||
} catch {}
|
||||
} catch (readError) {
|
||||
console.warn('[PUT /api/files] Failed to read existing file for frontmatter:', readError);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Handle JSON payload (Excalidraw scene)
|
||||
if (contentType === 'application/json') {
|
||||
const body = req.body;
|
||||
const parsed = excalidrawSceneSchema.safeParse(body);
|
||||
|
||||
|
||||
if (!parsed.success) {
|
||||
console.warn('[PUT /api/files] invalid scene schema', parsed.error?.issues?.slice(0,3));
|
||||
return res.status(400).json({
|
||||
error: 'Invalid Excalidraw scene',
|
||||
issues: parsed.error.issues?.slice(0, 5)
|
||||
return res.status(400).json({
|
||||
error: 'Invalid Excalidraw scene',
|
||||
issues: parsed.error.issues?.slice(0, 5)
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// Convert to Obsidian format if target is .excalidraw.md
|
||||
if (isExcalidrawMd) {
|
||||
finalContent = toObsidianExcalidrawMd(parsed.data, existingFrontMatter);
|
||||
@@ -969,22 +902,22 @@ app.put('/api/files', express.json({ limit: '10mb' }), express.text({ limit: '10
|
||||
// Plain JSON for .excalidraw or .json files
|
||||
finalContent = JSON.stringify(parsed.data, null, 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Handle text/markdown payload (already formatted)
|
||||
else if (contentType === 'text/markdown') {
|
||||
finalContent = typeof req.body === 'string' ? req.body : String(req.body);
|
||||
}
|
||||
}
|
||||
else {
|
||||
console.warn('[PUT /api/files] unsupported content-type', contentType);
|
||||
return res.status(400).json({ error: 'Unsupported content type' });
|
||||
return res.status(400).json({ error: 'Unsupported content type. Use application/json for Excalidraw or text/markdown for text files.' });
|
||||
}
|
||||
|
||||
|
||||
// Check size limit
|
||||
if (Buffer.byteLength(finalContent, 'utf-8') > 10 * 1024 * 1024) {
|
||||
console.warn('[PUT /api/files] payload too large path=%s size=%d', rel, Buffer.byteLength(finalContent, 'utf-8'));
|
||||
return res.status(413).json({ error: 'Payload too large' });
|
||||
return res.status(413).json({ error: 'Payload too large (max 10MB)' });
|
||||
}
|
||||
|
||||
|
||||
// Check for conflicts with If-Match
|
||||
const hasExisting = fs.existsSync(abs);
|
||||
const ifMatch = req.headers['if-match'];
|
||||
@@ -993,34 +926,44 @@ app.put('/api/files', express.json({ limit: '10mb' }), express.text({ limit: '10
|
||||
const currentRev = calculateSimpleHash(current);
|
||||
if (ifMatch !== currentRev) {
|
||||
console.warn('[PUT /api/files] conflict path=%s ifMatch=%s current=%s', rel, ifMatch, currentRev);
|
||||
return res.status(409).json({ error: 'Conflict detected' });
|
||||
return res.status(409).json({ error: 'Conflict detected. File was modified externally.' });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Atomic write with backup
|
||||
const temp = abs + '.tmp';
|
||||
const backup = abs + '.bak';
|
||||
|
||||
|
||||
try {
|
||||
if (hasExisting) fs.copyFileSync(abs, backup);
|
||||
fs.writeFileSync(temp, finalContent, 'utf-8');
|
||||
fs.renameSync(temp, abs);
|
||||
console.log('[PUT /api/files] wrote file path=%s bytes=%d', rel, Buffer.byteLength(finalContent, 'utf-8'));
|
||||
} catch (e) {
|
||||
// Cleanup temp file
|
||||
if (fs.existsSync(temp)) try { fs.unlinkSync(temp); } catch {}
|
||||
// Restore backup if it exists
|
||||
if (hasExisting && fs.existsSync(backup)) try { fs.copyFileSync(backup, abs); } catch {}
|
||||
console.error('[PUT /api/files] write error path=%s', rel, e);
|
||||
throw e;
|
||||
return res.status(500).json({ error: 'Failed to write file. Check file permissions and disk space.' });
|
||||
}
|
||||
|
||||
|
||||
// Cleanup backup
|
||||
if (fs.existsSync(backup)) {
|
||||
try { fs.unlinkSync(backup); } catch {}
|
||||
}
|
||||
|
||||
const rev = calculateSimpleHash(finalContent);
|
||||
res.setHeader('ETag', rev);
|
||||
res.json({ rev });
|
||||
|
||||
res.json({ rev, path: rel, size: Buffer.byteLength(finalContent, 'utf-8') });
|
||||
|
||||
} catch (error) {
|
||||
const code = typeof error?.status === 'number' ? error.status : 500;
|
||||
console.error('PUT /api/files error:', error);
|
||||
res.status(code).json({ error: 'Internal server error' });
|
||||
res.status(code).json({
|
||||
error: 'Internal server error',
|
||||
message: error.message || 'An unexpected error occurred while saving the file.'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1460,13 +1403,55 @@ app.use((req, res) => {
|
||||
return sendIndex(req, res);
|
||||
});
|
||||
|
||||
// Phase 3: Setup performance monitoring endpoint
|
||||
setupPerformanceEndpoint(app, performanceMonitor, metadataCache, meilisearchCircuitBreaker);
|
||||
|
||||
// Créer le répertoire de la voûte s'il n'existe pas
|
||||
if (!fs.existsSync(vaultDir)) {
|
||||
fs.mkdirSync(vaultDir, { recursive: true });
|
||||
console.log('Created vault directory:', vaultDir);
|
||||
}
|
||||
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`ObsiViewer server running on http://0.0.0.0:${PORT}`);
|
||||
console.log(`Vault directory: ${vaultDir}`);
|
||||
// Phase 3: Deferred Meilisearch indexing (non-blocking)
|
||||
let indexingState = { inProgress: false, completed: false };
|
||||
const scheduleIndexing = async () => {
|
||||
if (indexingState.inProgress) return;
|
||||
indexingState.inProgress = true;
|
||||
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
console.time('[Meilisearch] Background indexing');
|
||||
await fullReindex(vaultDir);
|
||||
console.timeEnd('[Meilisearch] Background indexing');
|
||||
indexingState.completed = true;
|
||||
console.log('[Meilisearch] ✅ Background indexing completed');
|
||||
} catch (error) {
|
||||
console.error('[Meilisearch] ❌ Background indexing failed:', error.message);
|
||||
indexingState.completed = false;
|
||||
// Retry after 5 minutes
|
||||
setTimeout(() => {
|
||||
indexingState.inProgress = false;
|
||||
scheduleIndexing();
|
||||
}, 5 * 60 * 1000);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const server = app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`🚀 ObsiViewer server running on http://0.0.0.0:${PORT}`);
|
||||
console.log(`📁 Vault directory: ${vaultDir}`);
|
||||
console.log(`📊 Performance monitoring: http://0.0.0.0:${PORT}/__perf`);
|
||||
|
||||
// Schedule background indexing (non-blocking)
|
||||
scheduleIndexing();
|
||||
console.log('✅ Server ready - Meilisearch indexing in background');
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
process.on('SIGINT', () => {
|
||||
console.log('\n🛑 Shutting down server...');
|
||||
server.close(() => {
|
||||
console.log('✅ Server shutdown complete');
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* MetadataCache - TTL + simple LRU cache
|
||||
*
|
||||
* Features:
|
||||
* - TTL-based expiration
|
||||
* - Simple FIFO eviction when maxItems exceeded
|
||||
* - read-through helper for async producers
|
||||
* - Pseudo-LRU via Map re-insertion
|
||||
*/
|
||||
|
||||
export class MetadataCache {
|
||||
constructor({ ttlMs = 5 * 60 * 1000, maxItems = 10_000 } = {}) {
|
||||
this.ttlMs = ttlMs;
|
||||
this.maxItems = maxItems;
|
||||
this.store = new Map(); // key -> { value, exp }
|
||||
this.stats = {
|
||||
hits: 0,
|
||||
misses: 0,
|
||||
evictions: 0,
|
||||
sets: 0
|
||||
};
|
||||
}
|
||||
|
||||
_now() {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict oldest entries if cache exceeds maxItems
|
||||
*/
|
||||
_evictIfNeeded() {
|
||||
if (this.store.size <= this.maxItems) return;
|
||||
|
||||
const toEvict = this.store.size - this.maxItems + Math.floor(this.maxItems * 0.1);
|
||||
let evicted = 0;
|
||||
|
||||
for (const k of this.store.keys()) {
|
||||
this.store.delete(k);
|
||||
evicted++;
|
||||
if (evicted >= toEvict) break;
|
||||
}
|
||||
|
||||
this.stats.evictions += evicted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get value from cache
|
||||
* Returns { hit: boolean, value: any }
|
||||
*/
|
||||
get(key) {
|
||||
const entry = this.store.get(key);
|
||||
|
||||
if (!entry) {
|
||||
this.stats.misses++;
|
||||
return { hit: false, value: undefined };
|
||||
}
|
||||
|
||||
// Check expiration
|
||||
if (entry.exp < this._now()) {
|
||||
this.store.delete(key);
|
||||
this.stats.misses++;
|
||||
return { hit: false, value: undefined };
|
||||
}
|
||||
|
||||
// Touch for pseudo-LRU: re-insert at end
|
||||
this.store.delete(key);
|
||||
this.store.set(key, entry);
|
||||
this.stats.hits++;
|
||||
return { hit: true, value: entry.value };
|
||||
}
|
||||
|
||||
/**
|
||||
* Set value in cache with optional TTL override
|
||||
*/
|
||||
set(key, value, { ttlMs = this.ttlMs } = {}) {
|
||||
const exp = this._now() + ttlMs;
|
||||
this.store.set(key, { value, exp });
|
||||
this.stats.sets++;
|
||||
this._evictIfNeeded();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete specific key
|
||||
*/
|
||||
delete(key) {
|
||||
this.store.delete(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear entire cache
|
||||
*/
|
||||
clear() {
|
||||
this.store.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-through helper: get from cache or call producer
|
||||
* Returns { value, hit: boolean }
|
||||
*/
|
||||
async remember(key, producer, { ttlMs = this.ttlMs } = {}) {
|
||||
const { hit, value } = this.get(key);
|
||||
if (hit) return { value, hit: true };
|
||||
|
||||
const fresh = await producer();
|
||||
this.set(key, fresh, { ttlMs });
|
||||
return { value: fresh, hit: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const total = this.stats.hits + this.stats.misses;
|
||||
const hitRate = total > 0 ? (this.stats.hits / total) * 100 : 0;
|
||||
|
||||
return {
|
||||
size: this.store.size,
|
||||
maxItems: this.maxItems,
|
||||
ttlMs: this.ttlMs,
|
||||
hits: this.stats.hits,
|
||||
misses: this.stats.misses,
|
||||
hitRate: Math.round(hitRate * 10) / 10,
|
||||
evictions: this.stats.evictions,
|
||||
sets: this.stats.sets
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset statistics
|
||||
*/
|
||||
resetStats() {
|
||||
this.stats = { hits: 0, misses: 0, evictions: 0, sets: 0 };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* PerformanceMonitor - Track latencies, errors, and cache metrics
|
||||
*
|
||||
* Features:
|
||||
* - Request timing (avg, p95)
|
||||
* - Cache hit/miss tracking
|
||||
* - Error counting
|
||||
* - Simple ring buffer for timings
|
||||
*/
|
||||
|
||||
export class PerformanceMonitor {
|
||||
constructor(clock = () => Date.now()) {
|
||||
this.clock = clock;
|
||||
this.counters = {
|
||||
cacheHit: 0,
|
||||
cacheMiss: 0,
|
||||
reqCount: 0,
|
||||
reqError: 0,
|
||||
meilisearchRetry: 0,
|
||||
filesystemRetry: 0
|
||||
};
|
||||
this.timings = []; // Ring buffer for latencies
|
||||
this.maxTimings = 500;
|
||||
this.startTime = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark request start, returns timestamp
|
||||
*/
|
||||
markRequestStart() {
|
||||
return this.clock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark request end, calculate duration
|
||||
* Returns duration in ms
|
||||
*/
|
||||
markRequestEnd(startTs, ok = true) {
|
||||
const dur = this.clock() - startTs;
|
||||
|
||||
if (this.timings.length >= this.maxTimings) {
|
||||
this.timings.shift();
|
||||
}
|
||||
this.timings.push(dur);
|
||||
|
||||
this.counters.reqCount += 1;
|
||||
if (!ok) this.counters.reqError += 1;
|
||||
|
||||
return dur;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark cache hit or miss
|
||||
*/
|
||||
markCache(hit) {
|
||||
if (hit) {
|
||||
this.counters.cacheHit += 1;
|
||||
} else {
|
||||
this.counters.cacheMiss += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark retry event
|
||||
*/
|
||||
markRetry(source = 'unknown') {
|
||||
if (source === 'meilisearch') {
|
||||
this.counters.meilisearchRetry += 1;
|
||||
} else if (source === 'filesystem') {
|
||||
this.counters.filesystemRetry += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get performance snapshot
|
||||
*/
|
||||
snapshot() {
|
||||
const arr = this.timings.slice();
|
||||
const n = arr.length || 1;
|
||||
const sum = arr.reduce((a, b) => a + b, 0);
|
||||
const avg = sum / n;
|
||||
|
||||
// Calculate p95
|
||||
const sorted = arr.slice().sort((a, b) => a - b);
|
||||
const p95Idx = Math.min(n - 1, Math.floor(n * 0.95));
|
||||
const p95 = sorted[p95Idx] || 0;
|
||||
|
||||
const uptime = this.clock() - this.startTime;
|
||||
const cacheTotal = this.counters.cacheHit + this.counters.cacheMiss;
|
||||
const cacheHitRate = cacheTotal > 0
|
||||
? Math.round((this.counters.cacheHit / cacheTotal) * 1000) / 10
|
||||
: 0;
|
||||
|
||||
const errorRate = this.counters.reqCount > 0
|
||||
? Math.round((this.counters.reqError / this.counters.reqCount) * 1000) / 10
|
||||
: 0;
|
||||
|
||||
return {
|
||||
uptime,
|
||||
requests: {
|
||||
total: this.counters.reqCount,
|
||||
errors: this.counters.reqError,
|
||||
errorRate: `${errorRate}%`
|
||||
},
|
||||
cache: {
|
||||
hits: this.counters.cacheHit,
|
||||
misses: this.counters.cacheMiss,
|
||||
hitRate: `${cacheHitRate}%`
|
||||
},
|
||||
retries: {
|
||||
meilisearch: this.counters.meilisearchRetry,
|
||||
filesystem: this.counters.filesystemRetry
|
||||
},
|
||||
latency: {
|
||||
avgMs: Math.round(avg),
|
||||
p95Ms: Math.round(p95),
|
||||
samples: this.timings.length
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all counters
|
||||
*/
|
||||
reset() {
|
||||
this.counters = {
|
||||
cacheHit: 0,
|
||||
cacheMiss: 0,
|
||||
reqCount: 0,
|
||||
reqError: 0,
|
||||
meilisearchRetry: 0,
|
||||
filesystemRetry: 0
|
||||
};
|
||||
this.timings = [];
|
||||
this.startTime = this.clock();
|
||||
}
|
||||
}
|
||||
+106
-22
@@ -20,41 +20,125 @@ export const PERFORMANCE_CONFIG = {
|
||||
};
|
||||
|
||||
/**
|
||||
* Simple in-memory cache for metadata
|
||||
* Advanced in-memory cache for metadata with metrics and intelligent invalidation
|
||||
*/
|
||||
export class MetadataCache {
|
||||
constructor(ttl = PERFORMANCE_CONFIG.METADATA_CACHE_TTL) {
|
||||
this.cache = null;
|
||||
constructor(ttl = PERFORMANCE_CONFIG.METADATA_CACHE_TTL, maxSize = 10000) {
|
||||
this.cache = new Map();
|
||||
this.ttl = ttl;
|
||||
this.timestamp = null;
|
||||
this.maxSize = maxSize;
|
||||
this.hits = 0;
|
||||
this.misses = 0;
|
||||
this.isLoading = false;
|
||||
this.loadPromise = null;
|
||||
}
|
||||
|
||||
set(data) {
|
||||
this.cache = data;
|
||||
this.timestamp = Date.now();
|
||||
}
|
||||
|
||||
get() {
|
||||
if (!this.cache || !this.timestamp) {
|
||||
return null;
|
||||
/**
|
||||
* Get cached metadata for a vault
|
||||
*/
|
||||
async getMetadata(vaultDir, loader) {
|
||||
const now = Date.now();
|
||||
const cacheKey = this.getCacheKey(vaultDir);
|
||||
const cached = this.cache.get(cacheKey);
|
||||
|
||||
// Cache valide ?
|
||||
if (cached && (now - cached.timestamp) < this.ttl) {
|
||||
this.hits++;
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
if (Date.now() - this.timestamp > this.ttl) {
|
||||
this.cache = null;
|
||||
this.timestamp = null;
|
||||
return null;
|
||||
// Cache miss - recharger
|
||||
this.misses++;
|
||||
return await this.loadFreshMetadata(vaultDir, cacheKey, loader);
|
||||
}
|
||||
|
||||
/**
|
||||
* Charger les métadonnées fraiches
|
||||
*/
|
||||
async loadFreshMetadata(vaultDir, cacheKey, loader) {
|
||||
// Éviter les chargements concurrents
|
||||
if (this.isLoading && this.loadPromise) {
|
||||
return this.loadPromise;
|
||||
}
|
||||
|
||||
return this.cache;
|
||||
this.isLoading = true;
|
||||
|
||||
try {
|
||||
this.loadPromise = loader();
|
||||
const metadata = await this.loadPromise;
|
||||
|
||||
// Stocker en cache
|
||||
this.cache.set(cacheKey, {
|
||||
data: metadata,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
// Nettoyer le cache si trop gros
|
||||
this.cleanupIfNeeded();
|
||||
|
||||
return metadata;
|
||||
} finally {
|
||||
this.isLoading = false;
|
||||
this.loadPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
invalidate() {
|
||||
this.cache = null;
|
||||
this.timestamp = null;
|
||||
/**
|
||||
* Invalider le cache pour une voûte
|
||||
*/
|
||||
invalidate(vaultDir = null) {
|
||||
if (vaultDir) {
|
||||
const cacheKey = this.getCacheKey(vaultDir);
|
||||
this.cache.delete(cacheKey);
|
||||
} else {
|
||||
this.cache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
isValid() {
|
||||
return this.cache !== null && (Date.now() - this.timestamp) < this.ttl;
|
||||
/**
|
||||
* Générer une clé de cache unique
|
||||
*/
|
||||
getCacheKey(vaultDir) {
|
||||
return `metadata_${vaultDir.replace(/[/\\]/g, '_')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Nettoyer le cache si nécessaire (LRU simple)
|
||||
*/
|
||||
cleanupIfNeeded() {
|
||||
if (this.cache.size > this.maxSize) {
|
||||
// Supprimer les entrées les plus anciennes (LRU simple)
|
||||
const entries = Array.from(this.cache.entries());
|
||||
entries.sort((a, b) => a[1].timestamp - b[1].timestamp);
|
||||
|
||||
const toRemove = entries.slice(0, Math.floor(this.maxSize * 0.1));
|
||||
toRemove.forEach(([key]) => this.cache.delete(key));
|
||||
|
||||
console.log(`[Cache] Cleaned up ${toRemove.length} old entries`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Obtenir les statistiques du cache
|
||||
*/
|
||||
getStats() {
|
||||
const total = this.hits + this.misses;
|
||||
return {
|
||||
size: this.cache.size,
|
||||
hitRate: total > 0 ? (this.hits / total) * 100 : 0,
|
||||
hits: this.hits,
|
||||
misses: this.misses,
|
||||
maxSize: this.maxSize,
|
||||
ttl: this.ttl
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Réinitialiser les métriques
|
||||
*/
|
||||
resetStats() {
|
||||
this.hits = 0;
|
||||
this.misses = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Phase 3 Startup Configuration
|
||||
*
|
||||
* Add these lines to server/index.mjs before app.listen():
|
||||
*
|
||||
* 1. Setup performance endpoint
|
||||
* 2. Setup deferred Meilisearch indexing
|
||||
* 3. Start server with monitoring
|
||||
*/
|
||||
|
||||
// ============================================================================
|
||||
// BEFORE app.listen() - Add these lines:
|
||||
// ============================================================================
|
||||
|
||||
// Setup performance monitoring endpoint
|
||||
setupPerformanceEndpoint(app, performanceMonitor, metadataCache, meilisearchCircuitBreaker);
|
||||
|
||||
// Setup deferred Meilisearch indexing (non-blocking)
|
||||
const { scheduleIndexing } = await setupDeferredIndexing(vaultDir, fullReindex);
|
||||
|
||||
// Start server
|
||||
const server = app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`🚀 ObsiViewer server running on http://0.0.0.0:${PORT}`);
|
||||
console.log(`📁 Vault directory: ${vaultDir}`);
|
||||
console.log(`📊 Performance monitoring: http://0.0.0.0:${PORT}/__perf`);
|
||||
|
||||
// Schedule background indexing (non-blocking)
|
||||
scheduleIndexing();
|
||||
console.log('✅ Server ready - Meilisearch indexing in background');
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
process.on('SIGINT', () => {
|
||||
console.log('\n🛑 Shutting down server...');
|
||||
server.close(() => {
|
||||
console.log('✅ Server shutdown complete');
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// NOTES:
|
||||
// ============================================================================
|
||||
// - The server starts immediately without waiting for Meilisearch indexing
|
||||
// - Indexing happens in the background via setImmediate()
|
||||
// - If indexing fails, it retries after 5 minutes
|
||||
// - Performance metrics available at /__perf endpoint
|
||||
// - Cache hit rate should reach > 80% after 5 minutes of usage
|
||||
// - Circuit breaker protects against cascading failures
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Retry utilities with exponential backoff
|
||||
*
|
||||
* Features:
|
||||
* - Simple retry with fixed delay
|
||||
* - Exponential backoff with jitter
|
||||
* - Callback hooks for logging/monitoring
|
||||
*/
|
||||
|
||||
const sleep = (ms) => new Promise(res => setTimeout(res, ms));
|
||||
|
||||
/**
|
||||
* Simple retry with fixed delay
|
||||
*
|
||||
* @param {Function} fn - Async function to retry
|
||||
* @param {Object} options
|
||||
* @param {number} options.retries - Number of retries (default: 3)
|
||||
* @param {number} options.delayMs - Fixed delay between retries (default: 100)
|
||||
* @param {Function} options.onRetry - Callback on retry attempt
|
||||
* @returns {Promise} Result of fn
|
||||
*/
|
||||
export async function retry(fn, { retries = 3, delayMs = 100, onRetry } = {}) {
|
||||
let lastErr;
|
||||
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
|
||||
if (i === retries) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (onRetry) {
|
||||
onRetry({ attempt: i + 1, maxAttempts: retries + 1, err, delay: delayMs });
|
||||
}
|
||||
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry with exponential backoff and jitter
|
||||
*
|
||||
* @param {Function} fn - Async function to retry
|
||||
* @param {Object} options
|
||||
* @param {number} options.retries - Number of retries (default: 3)
|
||||
* @param {number} options.baseDelayMs - Base delay for exponential backoff (default: 100)
|
||||
* @param {number} options.maxDelayMs - Maximum delay cap (default: 2000)
|
||||
* @param {boolean} options.jitter - Add random jitter (default: true)
|
||||
* @param {Function} options.onRetry - Callback on retry attempt
|
||||
* @returns {Promise} Result of fn
|
||||
*/
|
||||
export async function retryWithBackoff(fn, {
|
||||
retries = 3,
|
||||
baseDelayMs = 100,
|
||||
maxDelayMs = 2000,
|
||||
jitter = true,
|
||||
onRetry
|
||||
} = {}) {
|
||||
let lastErr;
|
||||
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
|
||||
if (i === retries) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Exponential backoff: baseDelay * 2^attempt
|
||||
const exponential = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, i));
|
||||
|
||||
// Add jitter: random value between 50% and 100% of exponential
|
||||
const delay = jitter
|
||||
? Math.floor(exponential * (0.5 + Math.random() * 0.5))
|
||||
: exponential;
|
||||
|
||||
if (onRetry) {
|
||||
onRetry({
|
||||
attempt: i + 1,
|
||||
maxAttempts: retries + 1,
|
||||
delay,
|
||||
err,
|
||||
exponential
|
||||
});
|
||||
}
|
||||
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry with circuit breaker pattern
|
||||
* Fails fast after consecutive errors threshold
|
||||
*
|
||||
* @param {Function} fn - Async function to retry
|
||||
* @param {Object} options
|
||||
* @param {number} options.retries - Retries per attempt (default: 2)
|
||||
* @param {number} options.failureThreshold - Failures before circuit opens (default: 5)
|
||||
* @param {number} options.resetTimeoutMs - Time before circuit half-open (default: 30000)
|
||||
* @param {Function} options.onRetry - Callback on retry
|
||||
* @param {Function} options.onCircuitOpen - Callback when circuit opens
|
||||
* @returns {Promise} Result of fn
|
||||
*/
|
||||
export class CircuitBreaker {
|
||||
constructor({
|
||||
retries = 2,
|
||||
failureThreshold = 5,
|
||||
resetTimeoutMs = 30_000
|
||||
} = {}) {
|
||||
this.retries = retries;
|
||||
this.failureThreshold = failureThreshold;
|
||||
this.resetTimeoutMs = resetTimeoutMs;
|
||||
this.failureCount = 0;
|
||||
this.state = 'closed'; // 'closed' | 'open' | 'half-open'
|
||||
this.lastFailureTime = null;
|
||||
}
|
||||
|
||||
async execute(fn, { onRetry, onCircuitOpen } = {}) {
|
||||
// Check if circuit should reset
|
||||
if (this.state === 'open') {
|
||||
const timeSinceFailure = Date.now() - this.lastFailureTime;
|
||||
if (timeSinceFailure >= this.resetTimeoutMs) {
|
||||
this.state = 'half-open';
|
||||
this.failureCount = 0;
|
||||
} else {
|
||||
throw new Error(`Circuit breaker is open (reset in ${this.resetTimeoutMs - timeSinceFailure}ms)`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await retryWithBackoff(fn, { retries: this.retries, onRetry });
|
||||
|
||||
// Success: reset failure count
|
||||
if (this.state === 'half-open') {
|
||||
this.state = 'closed';
|
||||
}
|
||||
this.failureCount = 0;
|
||||
|
||||
return result;
|
||||
} catch (err) {
|
||||
this.failureCount++;
|
||||
this.lastFailureTime = Date.now();
|
||||
|
||||
if (this.failureCount >= this.failureThreshold) {
|
||||
this.state = 'open';
|
||||
if (onCircuitOpen) {
|
||||
onCircuitOpen({ failureCount: this.failureCount });
|
||||
}
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.state = 'closed';
|
||||
this.failureCount = 0;
|
||||
this.lastFailureTime = null;
|
||||
}
|
||||
|
||||
getState() {
|
||||
return {
|
||||
state: this.state,
|
||||
failureCount: this.failureCount,
|
||||
failureThreshold: this.failureThreshold
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user