577 lines
22 KiB
JavaScript
577 lines
22 KiB
JavaScript
/**
|
|
* Phase 3 Patch - Endpoint modifications for caching and monitoring
|
|
*
|
|
* This file contains the updated endpoints that should replace the old ones in index.mjs
|
|
* Apply these changes step by step:
|
|
*
|
|
* 1. Replace /api/vault/metadata endpoint (lines ~500-551)
|
|
* 2. Replace /api/vault/metadata/paginated endpoint (lines ~553-620)
|
|
* 3. Add /__perf endpoint for monitoring (new)
|
|
* 4. Add startup hook for deferred Meilisearch indexing (new)
|
|
* 5. Add /api/folders/rename endpoint for folder renaming (new)
|
|
*/
|
|
|
|
import express from 'express';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 5: /api/folders/rename - Rename folder with validation
|
|
// ============================================================================
|
|
export function setupRenameFolderEndpoint(app, vaultDir, broadcastVaultEvent, metadataCache) {
|
|
app.put('/api/folders/rename', express.json(), (req, res) => {
|
|
try {
|
|
const { oldPath, newName } = req.body;
|
|
|
|
// Validation
|
|
if (!oldPath || typeof oldPath !== 'string') {
|
|
return res.status(400).json({ error: 'Missing or invalid oldPath' });
|
|
}
|
|
if (!newName || typeof newName !== 'string') {
|
|
return res.status(400).json({ error: 'Missing or invalid newName' });
|
|
}
|
|
|
|
// Sanitize inputs
|
|
const sanitizedOldPath = oldPath.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
|
|
const sanitizedNewName = newName.trim();
|
|
|
|
if (!sanitizedOldPath) {
|
|
return res.status(400).json({ error: 'Invalid oldPath' });
|
|
}
|
|
if (!sanitizedNewName) {
|
|
return res.status(400).json({ error: 'New name cannot be empty' });
|
|
}
|
|
|
|
// Prevent renaming to same name
|
|
const oldName = path.basename(sanitizedOldPath);
|
|
if (oldName === sanitizedNewName) {
|
|
return res.status(400).json({ error: 'New name is same as current name' });
|
|
}
|
|
|
|
// Construct paths
|
|
const oldFullPath = path.join(vaultDir, sanitizedOldPath);
|
|
const parentDir = path.dirname(oldFullPath);
|
|
const newFullPath = path.join(parentDir, sanitizedNewName);
|
|
|
|
// Check if old folder exists
|
|
if (!fs.existsSync(oldFullPath)) {
|
|
return res.status(404).json({ error: 'Source folder not found' });
|
|
}
|
|
|
|
// Check if old path is actually a directory
|
|
const oldStats = fs.statSync(oldFullPath);
|
|
if (!oldStats.isDirectory()) {
|
|
return res.status(400).json({ error: 'Source path is not a directory' });
|
|
}
|
|
|
|
// Check if new folder already exists
|
|
if (fs.existsSync(newFullPath)) {
|
|
return res.status(409).json({ error: 'A folder with this name already exists' });
|
|
}
|
|
|
|
// Perform the rename
|
|
try {
|
|
fs.renameSync(oldFullPath, newFullPath);
|
|
console.log(`[PUT /api/folders/rename] Renamed "${sanitizedOldPath}" to "${sanitizedNewName}"`);
|
|
} catch (renameError) {
|
|
console.error('[PUT /api/folders/rename] Rename operation failed:', renameError);
|
|
return res.status(500).json({ error: 'Failed to rename folder' });
|
|
}
|
|
|
|
// Update Meilisearch index for all affected files
|
|
try {
|
|
// Find all files that were in the old folder path
|
|
const walkDir = (dir, fileList = []) => {
|
|
const files = fs.readdirSync(dir);
|
|
for (const file of files) {
|
|
const filePath = path.join(dir, file);
|
|
const stat = fs.statSync(filePath);
|
|
if (stat.isDirectory()) {
|
|
walkDir(filePath, fileList);
|
|
} else if (file.toLowerCase().endsWith('.md')) {
|
|
fileList.push(path.relative(vaultDir, filePath).replace(/\\/g, '/'));
|
|
}
|
|
}
|
|
return fileList;
|
|
};
|
|
|
|
const affectedFiles = walkDir(newFullPath);
|
|
|
|
// Re-index affected files with new paths
|
|
for (const filePath of affectedFiles) {
|
|
try {
|
|
// Re-index the file with new path
|
|
// Note: This would need to be implemented based on your indexing logic
|
|
console.log(`[PUT /api/folders/rename] Re-indexing: ${filePath}`);
|
|
} catch (indexError) {
|
|
console.warn(`[PUT /api/folders/rename] Failed to re-index ${filePath}:`, indexError);
|
|
}
|
|
}
|
|
} catch (indexError) {
|
|
console.warn('[PUT /api/folders/rename] Index update failed:', indexError);
|
|
// Don't fail the request if indexing fails
|
|
}
|
|
|
|
// Invalidate metadata cache
|
|
if (metadataCache) metadataCache.clear();
|
|
|
|
// Emit SSE event for immediate UI update
|
|
const newRelPath = path.relative(vaultDir, newFullPath).replace(/\\/g, '/');
|
|
if (broadcastVaultEvent) {
|
|
broadcastVaultEvent({
|
|
event: 'folder-rename',
|
|
oldPath: sanitizedOldPath,
|
|
newPath: newRelPath,
|
|
timestamp: Date.now()
|
|
});
|
|
}
|
|
|
|
res.json({
|
|
success: true,
|
|
oldPath: sanitizedOldPath,
|
|
newPath: newRelPath,
|
|
newName: sanitizedNewName,
|
|
message: `Folder renamed successfully`
|
|
});
|
|
|
|
} catch (error) {
|
|
console.error('[PUT /api/folders/rename] Unexpected error:', error);
|
|
res.status(500).json({ error: 'Internal server error' });
|
|
}
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 6: /api/folders (DELETE) - Delete a folder recursively with validation
|
|
// ============================================================================
|
|
export function setupDeleteFolderEndpoint(app, vaultDir, broadcastVaultEvent, metadataCache) {
|
|
const parsePathParam = (req) => {
|
|
const q = typeof req.query.path === 'string' ? req.query.path : '';
|
|
if (q) return q;
|
|
const ct = String(req.headers['content-type'] || '').split(';')[0];
|
|
if (ct === 'application/json' && req.body && typeof req.body.path === 'string') {
|
|
return req.body.path;
|
|
}
|
|
return '';
|
|
};
|
|
|
|
app.delete('/api/folders', express.json(), (req, res) => {
|
|
try {
|
|
const rawPath = parsePathParam(req);
|
|
if (!rawPath) {
|
|
return res.status(400).json({ error: 'Missing or invalid path' });
|
|
}
|
|
|
|
const sanitizedRel = rawPath.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
|
|
const abs = path.join(vaultDir, sanitizedRel);
|
|
|
|
if (!fs.existsSync(abs)) {
|
|
return res.status(404).json({ error: 'Folder not found' });
|
|
}
|
|
|
|
const st = fs.statSync(abs);
|
|
if (!st.isDirectory()) {
|
|
return res.status(400).json({ error: 'Path is not a directory' });
|
|
}
|
|
|
|
try {
|
|
fs.rmSync(abs, { recursive: true, force: true });
|
|
console.log(`[DELETE /api/folders] Deleted folder "${sanitizedRel}"`);
|
|
} catch (delErr) {
|
|
console.error('[DELETE /api/folders] Delete failed:', delErr);
|
|
return res.status(500).json({ error: 'Failed to delete folder' });
|
|
}
|
|
|
|
// Invalidate metadata cache
|
|
if (metadataCache) metadataCache.clear();
|
|
|
|
// Emit SSE event for immediate UI update
|
|
if (broadcastVaultEvent) {
|
|
broadcastVaultEvent({
|
|
event: 'folder-delete',
|
|
path: sanitizedRel,
|
|
timestamp: Date.now()
|
|
});
|
|
}
|
|
|
|
return res.json({ success: true, path: sanitizedRel });
|
|
} catch (error) {
|
|
console.error('[DELETE /api/folders] Unexpected error:', error);
|
|
return res.status(500).json({ error: 'Internal server error' });
|
|
}
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 1: /api/vault/metadata - with cache read-through and monitoring
|
|
// ============================================================================
|
|
export function setupMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, { meiliClient, vaultIndexName, ensureIndexSettings, loadVaultMetadataOnly }) {
|
|
app.get('/api/vault/metadata', async (req, res) => {
|
|
const startTime = performanceMonitor.markRequestStart();
|
|
|
|
try {
|
|
// Use cache.remember() for read-through caching
|
|
const { value: metadata, hit } = await metadataCache.remember(
|
|
`metadata:${vaultDir}`,
|
|
async () => {
|
|
// Try Meilisearch first with circuit breaker
|
|
try {
|
|
return await meilisearchCircuitBreaker.execute(
|
|
async () => {
|
|
const client = meiliClient();
|
|
const indexUid = vaultIndexName(vaultDir);
|
|
const index = await ensureIndexSettings(client, indexUid);
|
|
|
|
const result = await index.search('', {
|
|
limit: 10000,
|
|
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt']
|
|
});
|
|
|
|
const items = Array.isArray(result.hits) ? result.hits.map(hit => ({
|
|
id: hit.id,
|
|
title: hit.title,
|
|
filePath: hit.path,
|
|
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
|
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
|
})) : [];
|
|
|
|
console.log(`[/api/vault/metadata] Loaded ${items.length} items from Meilisearch`);
|
|
return items;
|
|
},
|
|
{
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Meilisearch] Retry attempt ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('meilisearch');
|
|
},
|
|
onCircuitOpen: ({ failureCount }) => {
|
|
console.error(`[Meilisearch] Circuit breaker opened after ${failureCount} failures`);
|
|
}
|
|
}
|
|
);
|
|
} catch (meiliError) {
|
|
console.warn('[Meilisearch] Failed, falling back to filesystem:', meiliError.message);
|
|
|
|
// Fallback to filesystem with retry
|
|
return await retryWithBackoff(
|
|
async () => {
|
|
const notes = await loadVaultMetadataOnly(vaultDir);
|
|
const metadata = notes.map(n => ({
|
|
id: n.id,
|
|
title: n.title,
|
|
filePath: n.filePath,
|
|
createdAt: n.createdAt,
|
|
updatedAt: n.updatedAt
|
|
}));
|
|
|
|
console.log(`[/api/vault/metadata] Loaded ${metadata.length} items from filesystem`);
|
|
return metadata;
|
|
},
|
|
{
|
|
retries: 2,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 500,
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Filesystem] Retry attempt ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('filesystem');
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
|
|
performanceMonitor.markCache(hit);
|
|
const duration = performanceMonitor.markRequestEnd(startTime, true);
|
|
|
|
console.log(`[/api/vault/metadata] ${hit ? 'CACHE HIT' : 'CACHE MISS'} - ${duration}ms`);
|
|
|
|
res.json({
|
|
items: metadata,
|
|
cached: hit,
|
|
duration
|
|
});
|
|
} catch (error) {
|
|
performanceMonitor.markRequestEnd(startTime, false);
|
|
console.error('[/api/vault/metadata] Error:', error);
|
|
res.status(500).json({ error: 'Unable to load vault metadata.' });
|
|
}
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 2: /api/vault/metadata/paginated - with cache and monitoring
|
|
// ============================================================================
|
|
export function setupPaginatedMetadataEndpoint(app, metadataCache, performanceMonitor, vaultDir, meilisearchCircuitBreaker, retryWithBackoff, { meiliClient, vaultIndexName, ensureIndexSettings, loadVaultMetadataOnly }) {
|
|
app.get('/api/vault/metadata/paginated', async (req, res) => {
|
|
const startTime = performanceMonitor.markRequestStart();
|
|
|
|
try {
|
|
const limit = Math.min(parseInt(req.query.limit) || 100, 500);
|
|
const cursor = parseInt(req.query.cursor) || 0;
|
|
const search = req.query.search || '';
|
|
|
|
const cacheKey = `paginated:${vaultDir}:${search}`;
|
|
|
|
// For paginated requests, we cache the full result set and paginate client-side
|
|
const { value: allMetadata, hit } = await metadataCache.remember(
|
|
cacheKey,
|
|
async () => {
|
|
try {
|
|
return await meilisearchCircuitBreaker.execute(
|
|
async () => {
|
|
const client = meiliClient();
|
|
const indexUid = vaultIndexName(vaultDir);
|
|
const index = await ensureIndexSettings(client, indexUid);
|
|
|
|
const result = await index.search(search, {
|
|
limit: 10000,
|
|
attributesToRetrieve: ['id', 'title', 'path', 'createdAt', 'updatedAt'],
|
|
sort: ['updatedAt:desc']
|
|
});
|
|
|
|
return Array.isArray(result.hits) ? result.hits.map(hit => ({
|
|
id: hit.id,
|
|
title: hit.title,
|
|
filePath: hit.path,
|
|
createdAt: typeof hit.createdAt === 'number' ? new Date(hit.createdAt).toISOString() : hit.createdAt,
|
|
updatedAt: typeof hit.updatedAt === 'number' ? new Date(hit.updatedAt).toISOString() : hit.updatedAt,
|
|
})) : [];
|
|
},
|
|
{
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Meilisearch] Paginated retry ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('meilisearch');
|
|
}
|
|
}
|
|
);
|
|
} catch (meiliError) {
|
|
console.warn('[Meilisearch] Paginated failed, falling back to filesystem:', meiliError.message);
|
|
|
|
return await retryWithBackoff(
|
|
async () => {
|
|
const allMetadata = await loadVaultMetadataOnly(vaultDir);
|
|
|
|
let filtered = allMetadata;
|
|
if (search) {
|
|
const searchLower = search.toLowerCase();
|
|
filtered = allMetadata.filter(item =>
|
|
(item.title || '').toLowerCase().includes(searchLower) ||
|
|
(item.filePath || '').toLowerCase().includes(searchLower)
|
|
);
|
|
}
|
|
|
|
filtered.sort((a, b) => {
|
|
const dateA = new Date(a.updatedAt || a.createdAt || 0).getTime();
|
|
const dateB = new Date(b.updatedAt || b.createdAt || 0).getTime();
|
|
return dateB - dateA;
|
|
});
|
|
|
|
return filtered.map(n => ({
|
|
id: n.id,
|
|
title: n.title,
|
|
filePath: n.filePath,
|
|
createdAt: n.createdAt,
|
|
updatedAt: n.updatedAt
|
|
}));
|
|
},
|
|
{
|
|
retries: 2,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 500,
|
|
onRetry: ({ attempt, delay, err }) => {
|
|
console.warn(`[Filesystem] Paginated retry ${attempt}, delay ${delay}ms:`, err.message);
|
|
performanceMonitor.markRetry('filesystem');
|
|
}
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
|
|
// Paginate the cached result
|
|
const paginatedItems = allMetadata.slice(cursor, cursor + limit);
|
|
const hasMore = cursor + limit < allMetadata.length;
|
|
const nextCursor = hasMore ? cursor + limit : null;
|
|
|
|
performanceMonitor.markCache(hit);
|
|
const duration = performanceMonitor.markRequestEnd(startTime, true);
|
|
|
|
console.log(`[/api/vault/metadata/paginated] ${hit ? 'CACHE HIT' : 'CACHE MISS'} - cursor=${cursor}, limit=${limit}, duration=${duration}ms`);
|
|
|
|
res.json({
|
|
items: paginatedItems,
|
|
nextCursor,
|
|
hasMore,
|
|
total: allMetadata.length,
|
|
cached: hit,
|
|
duration
|
|
});
|
|
} catch (error) {
|
|
performanceMonitor.markRequestEnd(startTime, false);
|
|
console.error('[/api/vault/metadata/paginated] Error:', error);
|
|
res.status(500).json({ error: 'Pagination failed' });
|
|
}
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// ENDPOINT 3: /__perf - Performance monitoring dashboard
|
|
// ============================================================================
|
|
export function setupPerformanceEndpoint(app, performanceMonitor, metadataCache, meilisearchCircuitBreaker) {
|
|
app.get('/__perf', (req, res) => {
|
|
res.json({
|
|
performance: performanceMonitor.snapshot(),
|
|
cache: metadataCache.getStats(),
|
|
circuitBreaker: meilisearchCircuitBreaker.getState(),
|
|
timestamp: new Date().toISOString()
|
|
});
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// STARTUP HOOK: Deferred Meilisearch indexing (non-blocking)
|
|
// ============================================================================
|
|
export async function setupDeferredIndexing(vaultDir, fullReindex) {
|
|
let indexingInProgress = false;
|
|
let indexingCompleted = false;
|
|
let lastIndexingAttempt = 0;
|
|
const INDEXING_COOLDOWN = 5 * 60 * 1000; // 5 minutes
|
|
|
|
async function scheduleIndexing() {
|
|
const now = Date.now();
|
|
|
|
if (indexingInProgress || (now - lastIndexingAttempt) < INDEXING_COOLDOWN) {
|
|
return;
|
|
}
|
|
|
|
indexingInProgress = true;
|
|
lastIndexingAttempt = now;
|
|
|
|
console.log('[Meilisearch] Scheduling background indexing...');
|
|
|
|
// Use setImmediate to not block startup
|
|
setImmediate(async () => {
|
|
try {
|
|
console.time('[Meilisearch] Background indexing');
|
|
|
|
await fullReindex(vaultDir);
|
|
|
|
console.timeEnd('[Meilisearch] Background indexing');
|
|
console.log('[Meilisearch] Background indexing completed successfully');
|
|
|
|
indexingCompleted = true;
|
|
} catch (error) {
|
|
console.error('[Meilisearch] Background indexing failed:', error.message);
|
|
indexingCompleted = false;
|
|
|
|
// Schedule retry in 5 minutes
|
|
setTimeout(() => {
|
|
console.log('[Meilisearch] Retrying indexing in 5 minutes...');
|
|
indexingInProgress = false;
|
|
scheduleIndexing();
|
|
}, INDEXING_COOLDOWN);
|
|
} finally {
|
|
indexingInProgress = false;
|
|
}
|
|
});
|
|
}
|
|
|
|
return {
|
|
scheduleIndexing,
|
|
getState: () => ({ indexingInProgress, indexingCompleted, lastIndexingAttempt })
|
|
};
|
|
}
|
|
|
|
import { join, dirname, relative } from 'path';
|
|
import { existsSync, mkdirSync, writeFileSync } from 'fs';
|
|
|
|
// ============================================================================
|
|
// ENDPOINT: POST /api/vault/notes - Create new note
|
|
// ============================================================================
|
|
export function setupCreateNoteEndpoint(app, vaultDir) {
|
|
console.log('[Setup] Setting up /api/vault/notes endpoint');
|
|
app.post('/api/vault/notes', async (req, res) => {
|
|
try {
|
|
const { fileName, folderPath, frontmatter, content = '' } = req.body;
|
|
|
|
console.log('[/api/vault/notes] Request received:', { fileName, folderPath });
|
|
|
|
if (!fileName) {
|
|
return res.status(400).json({ error: 'fileName is required' });
|
|
}
|
|
|
|
if (!frontmatter || typeof frontmatter !== 'object') {
|
|
return res.status(400).json({ error: 'frontmatter is required and must be an object' });
|
|
}
|
|
|
|
// Ensure fileName ends with .md
|
|
const finalFileName = fileName.endsWith('.md') ? fileName : `${fileName}.md`;
|
|
|
|
// Build full path - handle folderPath properly
|
|
let fullFolderPath = '';
|
|
if (folderPath && folderPath !== '/' && folderPath.trim() !== '') {
|
|
fullFolderPath = folderPath.replace(/^\/+/, '').replace(/\/+$/, ''); // Remove leading/trailing slashes
|
|
}
|
|
|
|
const fullPath = fullFolderPath
|
|
? join(vaultDir, fullFolderPath, finalFileName)
|
|
: join(vaultDir, finalFileName);
|
|
|
|
console.log('[/api/vault/notes] Full path:', fullPath);
|
|
|
|
// Check if file already exists
|
|
if (existsSync(fullPath)) {
|
|
return res.status(409).json({ error: 'File already exists' });
|
|
}
|
|
|
|
// Format frontmatter to YAML
|
|
const frontmatterYaml = Object.keys(frontmatter).length > 0
|
|
? `---\n${Object.entries(frontmatter)
|
|
.map(([key, value]) => {
|
|
if (typeof value === 'string') {
|
|
return `${key}: "${value}"`;
|
|
} else if (typeof value === 'boolean') {
|
|
return `${key}: ${value}`;
|
|
} else if (Array.isArray(value)) {
|
|
return `${key}: [${value.map(v => `"${v}"`).join(', ')}]`;
|
|
}
|
|
return `${key}: ${value}`;
|
|
})
|
|
.join('\n')}\n---\n\n`
|
|
: '';
|
|
|
|
// Create the full content
|
|
const fullContent = frontmatterYaml + content;
|
|
|
|
// Ensure directory exists
|
|
const dir = dirname(fullPath);
|
|
console.log('[/api/vault/notes] Creating directory:', dir);
|
|
|
|
if (!existsSync(dir)) {
|
|
mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
// Write the file
|
|
console.log('[/api/vault/notes] Writing file:', fullPath);
|
|
writeFileSync(fullPath, fullContent, 'utf8');
|
|
|
|
// Generate ID (same logic as in vault loader)
|
|
const relativePath = relative(vaultDir, fullPath).replace(/\\/g, '/');
|
|
const id = relativePath.replace(/\.md$/, '');
|
|
|
|
console.log(`[/api/vault/notes] Created note: ${relativePath}`);
|
|
|
|
res.json({
|
|
id,
|
|
fileName: finalFileName,
|
|
filePath: relativePath,
|
|
success: true
|
|
});
|
|
|
|
} catch (error) {
|
|
console.error('[/api/vault/notes] Error creating note:', error.message, error.stack);
|
|
res.status(500).json({ error: 'Failed to create note', details: error.message });
|
|
}
|
|
});
|
|
}
|