diff --git a/e2e/search.spec.ts b/e2e/search.spec.ts index 21d2b31..d64ec3e 100644 --- a/e2e/search.spec.ts +++ b/e2e/search.spec.ts @@ -1,4 +1,28 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; + +const SEARCH_QUEUE_KEY = 'obsiviewer.searchdiag.queue'; + +async function clearDiagnostics(page: Page): Promise { + await page.evaluate(key => { + localStorage.removeItem(key); + }, SEARCH_QUEUE_KEY); +} + +async function readDiagnostics(page: Page): Promise { + const raw = await page.evaluate(key => localStorage.getItem(key), SEARCH_QUEUE_KEY); + if (!raw) { + return []; + } + try { + return JSON.parse(raw); + } catch { + return []; + } +} + +function findStage(events: any[], stage: string): any | undefined { + return events.find(event => event?.stage === stage); +} test.describe('Search Functionality', () => { test.beforeEach(async ({ page }) => { @@ -69,16 +93,43 @@ test.describe('Search Functionality', () => { }); test('should filter by tag', async ({ page }) => { + await clearDiagnostics(page); const searchInput = page.locator('input[type="text"]').first(); - + await searchInput.fill('tag:#work'); await searchInput.press('Enter'); - + await page.waitForTimeout(1000); // Should show results with #work tag const results = page.locator('.search-result'); await expect(results.first()).toBeVisible({ timeout: 5000 }); + + const events = await readDiagnostics(page); + const summary = findStage(events, 'SEARCH_DIAG_SUMMARY'); + expect(summary).toBeTruthy(); + expect(summary.data?.counts?.displayed).toBeGreaterThan(0); + expect(summary.data?.userVisible?.emptyStateShown).toBe(false); + }); + + test('should record diagnostics when tag search misses index', async ({ page }) => { + await clearDiagnostics(page); + const searchInput = page.locator('input[type="text"]').first(); + + await searchInput.fill('tag:#home'); + await searchInput.press('Enter'); + + await page.waitForTimeout(500); + + const events = await readDiagnostics(page); + const summary = findStage(events, 'SEARCH_DIAG_SUMMARY'); + expect(summary).toBeTruthy(); + expect(summary.data?.counts?.displayed ?? 0).toBe(0); + expect(summary.data?.userVisible?.emptyStateShown).toBe(true); + + const resultMap = findStage(events, 'SEARCH_DIAG_RESULT_MAP'); + expect(resultMap).toBeTruthy(); + expect(resultMap.data?.reasonsEmpty).toEqual(expect.arrayContaining(['tagNotInIndex'])); }); test('should toggle case sensitivity', async ({ page }) => { diff --git a/src/app/core/logging/log.service.ts b/src/app/core/logging/log.service.ts new file mode 100644 index 0000000..280b07f --- /dev/null +++ b/src/app/core/logging/log.service.ts @@ -0,0 +1,513 @@ +import { Injectable, PLATFORM_ID, inject } from '@angular/core'; +import { isPlatformBrowser } from '@angular/common'; +import { environment } from '../../../core/logging/environment'; +import { SearchDiagEvent, SearchDiagRecord } from './search-log.model'; + +const SESSION_STORAGE_KEY = 'obsiviewer.searchdiag.sessionId'; +const LOCAL_STORAGE_QUEUE_KEY = 'obsiviewer.searchdiag.queue'; +const SAMPLE_DECISIONS_LIMIT = 1000; +const MAX_DATA_SIZE = 5 * 1024; // 5KB +const LARGE_ARRAY_LIMIT = 200; + +type SearchLogLevel = 'info' | 'warn' | 'error'; + +interface SanitizedDictionary { + [key: string]: SanitizedValue; +} + +type SanitizedValue = + | string + | number + | boolean + | null + | undefined + | SanitizedValue[] + | SanitizedDictionary; + +type Dictionary = Record; + +@Injectable({ providedIn: 'root' }) +export class SearchLogService { + private readonly platformId = inject(PLATFORM_ID); + private readonly isBrowser = isPlatformBrowser(this.platformId); + + private readonly config = environment.logging; + private readonly sessionId = this.ensureSessionId(); + + private queue: SearchDiagRecord[] = []; + private flushTimer?: ReturnType; + private consecutiveFailures = 0; + private circuitBreakerOpenUntil = 0; + private isFlushing = false; + + private readonly correlationQueries = new Map(); + private readonly sampleDecisions = new Map(); + + constructor() { + if (this.isBrowser) { + this.loadQueueFromStorage(); + if (typeof window !== 'undefined') { + window.addEventListener('online', () => this.scheduleFlush()); + window.addEventListener('beforeunload', () => this.flushSync()); + } + } + } + + logSearch( + stage: SearchDiagEvent, + data: Record, + level: SearchLogLevel = 'info', + correlationId?: string + ): string { + if (!this.isBrowser || !this.config.enabled) { + return correlationId ?? ''; + } + + const effectiveCorrelationId = correlationId || this.generateUUID(); + + if (!this.shouldSample(effectiveCorrelationId)) { + return effectiveCorrelationId; + } + + const queryRaw = this.extractQueryRaw(effectiveCorrelationId, stage, data); + + const record: SearchDiagRecord = { + ts: new Date().toISOString(), + app: 'ObsiViewer', + sessionId: this.sessionId, + level, + correlationId: effectiveCorrelationId, + queryRaw, + context: this.buildContext(), + stage, + data: this.prepareData(stage, data), + userAgent: this.getUserAgent(), + }; + + this.enqueue(record); + return effectiveCorrelationId; + } + + async flush(): Promise { + if (!this.isBrowser) { + return; + } + + clearTimeout(this.flushTimer); + this.flushTimer = undefined; + + await this.processQueue(); + } + + private flushSync(): void { + if (!this.isBrowser || this.queue.length === 0) { + return; + } + + const payload = this.queue.slice(); + this.queue = []; + this.persistQueue(); + + if ('sendBeacon' in navigator) { + const blob = new Blob([JSON.stringify(payload)], { type: 'application/json' }); + navigator.sendBeacon(this.config.endpoint, blob); + } + } + + private enqueue(record: SearchDiagRecord): void { + this.queue.push(record); + this.persistQueue(); + + if (this.queue.length >= this.config.batchSize) { + this.scheduleFlush(0); + } else { + this.scheduleFlush(); + } + } + + private scheduleFlush(delayMs = this.config.debounceMs): void { + if (!this.isBrowser) { + return; + } + + if (this.flushTimer) { + return; + } + + this.flushTimer = setTimeout(() => { + this.flushTimer = undefined; + + if (typeof window !== 'undefined' && 'requestIdleCallback' in window) { + requestIdleCallback(() => this.processQueue()); + } else { + this.processQueue(); + } + }, delayMs); + } + + private async processQueue(): Promise { + if (!this.isBrowser) { + return; + } + + if (this.isFlushing || this.queue.length === 0) { + return; + } + + if (Date.now() < this.circuitBreakerOpenUntil) { + return; + } + + if (typeof navigator !== 'undefined' && !navigator.onLine) { + return; + } + + this.isFlushing = true; + const batch = [...this.queue]; + + try { + await this.sendWithRetry(batch); + this.queue = []; + this.consecutiveFailures = 0; + this.persistQueue(); + } catch (error) { + this.consecutiveFailures++; + + if (this.consecutiveFailures >= this.config.circuitBreakerThreshold) { + this.circuitBreakerOpenUntil = Date.now() + this.config.circuitBreakerResetMs; + } + + console.warn('[SearchLogService] Failed to send diagnostics batch', error); + } finally { + this.isFlushing = false; + } + } + + private async sendWithRetry(batch: SearchDiagRecord[]): Promise { + let lastError: unknown; + + for (let attempt = 0; attempt < this.config.maxRetries; attempt++) { + try { + await this.postBatch(batch); + return; + } catch (error) { + lastError = error; + + if (attempt < this.config.maxRetries - 1) { + const delayMs = 500 * Math.pow(2, attempt); + await this.delay(delayMs); + } + } + } + + throw lastError; + } + + private async postBatch(batch: SearchDiagRecord[]): Promise { + if (batch.length === 0) { + return; + } + + const payload = batch.length === 1 ? batch[0] : batch; + + const response = await fetch(this.config.endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + keepalive: true, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + } + + private buildContext(): SearchDiagRecord['context'] { + const context: SearchDiagRecord['context'] = { + version: environment.appVersion, + }; + + if (!this.isBrowser) { + return context; + } + + if (typeof window !== 'undefined' && window.location) { + context.route = window.location.pathname + window.location.search; + } + + if (typeof document !== 'undefined' && document.documentElement) { + const theme = document.documentElement.getAttribute('data-theme'); + if (theme === 'light' || theme === 'dark') { + context.theme = theme; + } + } + + try { + const vault = localStorage.getItem('obsiviewer.vaultName'); + if (vault) { + context.vault = vault; + } + } catch { + // ignore storage errors + } + + return context; + } + + private getUserAgent(): string | undefined { + if (!this.isBrowser || typeof navigator === 'undefined') { + return undefined; + } + return navigator.userAgent; + } + + private prepareData(stage: SearchDiagEvent, data: Record): Dictionary { + const cloned = this.cloneData(data); + delete cloned.queryRaw; + delete cloned['noteContent']; + + const sanitized = this.sanitizeValue(cloned, [stage]) as Dictionary; + return this.enforceSizeLimit(sanitized); + } + + private sanitizeValue(value: unknown, path: string[], seen = new WeakSet()): SanitizedValue { + if (value === null || value === undefined) { + return value as SanitizedValue; + } + + if (typeof value === 'number' || typeof value === 'boolean') { + return value; + } + + if (typeof value === 'string') { + return this.sanitizeString(value, path); + } + + if (Array.isArray(value)) { + if (seen.has(value)) { + return '[Circular]' as SanitizedValue; + } + seen.add(value); + + const limited = value.slice(0, LARGE_ARRAY_LIMIT); + const sanitizedArray = limited.map((entry, index) => this.sanitizeValue(entry, [...path, String(index)], seen)); + + if (value.length > LARGE_ARRAY_LIMIT) { + sanitizedArray.push('[Truncated]' as unknown as SanitizedValue); + } + + seen.delete(value); + return sanitizedArray; + } + + if (typeof value === 'object') { + if (seen.has(value as object)) { + return '[Circular]' as SanitizedValue; + } + + seen.add(value as object); + const result: Dictionary = {}; + + for (const [key, entry] of Object.entries(value as Dictionary)) { + const lowerKey = key.toLowerCase(); + + if (this.isContentKey(lowerKey)) { + result[key] = '[REDACTED_CONTENT]'; + continue; + } + + if (this.isSensitivePathKey(lowerKey)) { + result[key] = this.maskPath(String(entry)); + continue; + } + + result[key] = this.sanitizeValue(entry, [...path, key], seen); + } + + seen.delete(value as object); + return result; + } + + return String(value); + } + + private sanitizeString(value: string, path: string[]): string { + const lowerPath = path[path.length - 1]?.toLowerCase() ?? ''; + + if (this.isContentKey(lowerPath)) { + return '[REDACTED_CONTENT]'; + } + + if (this.isSensitivePathKey(lowerPath)) { + return this.maskPath(value); + } + + if (value.length > MAX_DATA_SIZE) { + return `${value.slice(0, 512)}…[TRUNCATED_${value.length}]`; + } + + return value; + } + + private enforceSizeLimit(value: T): T { + try { + const serialized = JSON.stringify(value); + if (serialized.length > MAX_DATA_SIZE) { + return { + _truncated: true, + _originalSize: serialized.length, + _message: 'Payload truncated due to size limit', + } as unknown as T; + } + } catch (error) { + return { + _error: 'Failed to serialize payload', + _message: String(error), + } as unknown as T; + } + + return value; + } + + private isContentKey(key: string): boolean { + return key.includes('content') || key.includes('raw') || key.includes('body') || key.includes('html'); + } + + private isSensitivePathKey(key: string): boolean { + return key.includes('path') || key.includes('filepath') || key.includes('folder'); + } + + private maskPath(value: string): string { + if (!value) { + return value; + } + + const segments = value.split(/[\\/]+/).filter(Boolean); + if (segments.length === 0) { + return value; + } + + const tail = segments.slice(-2).join('/'); + return segments.length > 2 ? `…/${tail}` : tail; + } + + private cloneData(data: Record): Dictionary { + return JSON.parse(JSON.stringify(data ?? {})) as Dictionary; + } + + private ensureSessionId(): string { + if (!this.isBrowser) { + return this.generateUUID(); + } + + try { + let id = sessionStorage.getItem(SESSION_STORAGE_KEY); + if (!id) { + id = this.generateUUID(); + sessionStorage.setItem(SESSION_STORAGE_KEY, id); + } + return id; + } catch { + return this.generateUUID(); + } + } + + private shouldSample(correlationId: string): boolean { + const rate = this.config.sampleRateSearchDiag ?? 1; + if (rate >= 1) { + return true; + } + + if (this.sampleDecisions.has(correlationId)) { + return this.sampleDecisions.get(correlationId) as boolean; + } + + const decision = Math.random() < rate; + this.sampleDecisions.set(correlationId, decision); + + if (this.sampleDecisions.size > SAMPLE_DECISIONS_LIMIT) { + const [firstKey] = this.sampleDecisions.keys(); + if (firstKey) { + this.sampleDecisions.delete(firstKey); + } + } + + return decision; + } + + private extractQueryRaw( + correlationId: string, + stage: SearchDiagEvent, + data: Record + ): string { + let query = ''; + const maybeQuery = data['queryRaw']; + if (typeof maybeQuery === 'string') { + query = maybeQuery; + } else { + query = this.correlationQueries.get(correlationId) ?? ''; + } + + if (stage === 'SEARCH_DIAG_START' && query) { + this.correlationQueries.set(correlationId, query); + } + + if (!query && typeof data['query'] === 'string') { + query = data['query'] as string; + } + + return query; + } + + private persistQueue(): void { + if (!this.isBrowser) { + return; + } + + try { + localStorage.setItem(LOCAL_STORAGE_QUEUE_KEY, JSON.stringify(this.queue)); + } catch { + // ignore + } + } + + private loadQueueFromStorage(): void { + if (!this.isBrowser) { + return; + } + + try { + const stored = localStorage.getItem(LOCAL_STORAGE_QUEUE_KEY); + if (!stored) { + return; + } + const parsed = JSON.parse(stored); + if (Array.isArray(parsed)) { + this.queue = parsed; + if (this.queue.length > 0) { + this.scheduleFlush(); + } + } + } catch { + // ignore parse errors + } + } + + private generateUUID(): string { + if (this.isBrowser && typeof window !== 'undefined' && 'crypto' in window && typeof crypto.randomUUID === 'function') { + return crypto.randomUUID(); + } + + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (char) => { + const random = (Math.random() * 16) | 0; + const value = char === 'x' ? random : (random & 0x3) | 0x8; + return value.toString(16); + }); + } + + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/src/app/core/logging/search-log.model.ts b/src/app/core/logging/search-log.model.ts new file mode 100644 index 0000000..32736e6 --- /dev/null +++ b/src/app/core/logging/search-log.model.ts @@ -0,0 +1,37 @@ +export type SearchDiagEvent = + | 'SEARCH_DIAG_START' + | 'SEARCH_DIAG_PARSE' + | 'SEARCH_DIAG_PLAN' + | 'SEARCH_DIAG_EXEC_PROVIDER' + | 'SEARCH_DIAG_RESULT_MAP' + | 'SEARCH_DIAG_SUMMARY' + | 'SEARCH_DIAG_ERROR'; + +export interface SearchDiagRecord { + ts: string; + app: 'ObsiViewer'; + sessionId: string; + level: 'info' | 'warn' | 'error'; + correlationId: string; + queryRaw: string; + context: { + route?: string; + vault?: string; + theme?: 'dark' | 'light'; + version?: string; + }; + userAgent?: string; + stage: SearchDiagEvent; + data: Record; +} + +export interface SearchDiagContextMeta { + userAgent?: string; + sampleRate?: number; +} + +export interface SearchDiagOptions { + correlationId?: string; + level?: 'info' | 'warn' | 'error'; + queryRaw: string; +} diff --git a/src/core/logging/environment.ts b/src/core/logging/environment.ts index 889a009..589132b 100644 --- a/src/core/logging/environment.ts +++ b/src/core/logging/environment.ts @@ -9,5 +9,6 @@ export const environment = { maxRetries: 5, circuitBreakerThreshold: 5, circuitBreakerResetMs: 30000, + sampleRateSearchDiag: 1.0, }, }; diff --git a/src/core/search/__tests__/search-orchestrator.plan.spec.ts b/src/core/search/__tests__/search-orchestrator.plan.spec.ts new file mode 100644 index 0000000..6cf9766 --- /dev/null +++ b/src/core/search/__tests__/search-orchestrator.plan.spec.ts @@ -0,0 +1,121 @@ +import { TestBed } from '@angular/core/testing'; +import { provideZonelessChangeDetection } from '@angular/core'; +import { SearchOrchestratorService } from '../search-orchestrator.service'; +import { SearchIndexService, SearchIndexDiagnosticsSnapshot, SearchIndexNormalizationRules } from '../search-index.service'; +import { ClientLoggingService } from '../../../services/client-logging.service'; +import { SearchLogService } from '../../../app/core/logging/log.service'; +import { ParseDiagnostics } from '../search-parser.types'; +import { SearchExecutionOptions } from '../search-orchestrator.service'; + +describe('SearchOrchestratorService diagnostics planning', () => { + let service: SearchOrchestratorService; + let normalizationRules: SearchIndexNormalizationRules; + + beforeEach(() => { + normalizationRules = { + tags: { + ensureHashPrefix: true, + preserveCase: true, + deduplicate: true, + stripHashOnMatch: true + }, + paths: { + separator: '/', + caseSensitive: false + }, + files: { + caseSensitive: false + } + }; + + const searchIndexStub = { + getAllContexts: jasmine.createSpy('getAllContexts').and.returnValue([]), + getIndexDiagnostics: jasmine.createSpy('getIndexDiagnostics').and.returnValue({}) + } as unknown as SearchIndexService; + + const clientLoggerStub = jasmine.createSpyObj('ClientLoggingService', ['info', 'debug', 'error', 'warn']); + const diagLoggerStub = { + logSearch: jasmine.createSpy('logSearch').and.returnValue('correlation-1') + } as unknown as SearchLogService; + + TestBed.configureTestingModule({ + providers: [ + provideZonelessChangeDetection(), + SearchOrchestratorService, + { provide: SearchIndexService, useValue: searchIndexStub }, + { provide: ClientLoggingService, useValue: clientLoggerStub }, + { provide: SearchLogService, useValue: diagLoggerStub } + ] + }); + + service = TestBed.inject(SearchOrchestratorService); + }); + + it('builds plan with tag stages and no warnings when tag is indexed', () => { + const diagnostics: ParseDiagnostics = { + tokens: ['tag:#home'], + warnings: [], + filters: { + tag: ['#home'], + path: [], + file: [], + negative: [], + negativeDetails: [], + regex: false, + caseSensitive: false, + wholeWord: false + } + }; + const options: SearchExecutionOptions = { caseSensitive: false, regexMode: false }; + const indexSnapshot: SearchIndexDiagnosticsSnapshot = { + tagsIndexed: true, + pathIndexed: true, + filesIndexed: true, + totalContexts: 25, + tagCardinality: { home: 3 }, + normalizeRules: normalizationRules + }; + + const plan = (service as any).buildExecutionPlan(diagnostics, options, indexSnapshot); + + expect(plan.stages.length).toBeGreaterThan(0); + const tagStage = plan.stages.find((s: any) => s.name === 'candidateSetFromTag'); + expect(tagStage).toBeDefined(); + expect(tagStage.constraint).toBe('tag=home'); + expect(tagStage.estimate).toBe(3); + expect(plan.warnings).toEqual([]); + }); + + it('flags missing tag index and negative filters in plan warnings', () => { + const diagnostics: ParseDiagnostics = { + tokens: ['tag:#home', '-path:"Archive"'], + warnings: [], + filters: { + tag: ['#home'], + path: ['Archive'], + file: [], + negative: ['Archive'], + negativeDetails: [{ type: 'path', value: 'Archive', wildcard: false }], + regex: false, + caseSensitive: false, + wholeWord: false + } + }; + const options: SearchExecutionOptions = { caseSensitive: false, regexMode: false }; + const indexSnapshot: SearchIndexDiagnosticsSnapshot = { + tagsIndexed: false, + pathIndexed: true, + filesIndexed: true, + totalContexts: 10, + tagCardinality: {}, + normalizeRules: normalizationRules + }; + + const plan = (service as any).buildExecutionPlan(diagnostics, options, indexSnapshot); + + expect(plan.warnings).toContain('TagIndexUnavailable'); + const negativeStage = plan.stages.find((s: any) => s.name === 'applyNegative'); + expect(negativeStage).toBeDefined(); + expect(negativeStage.constraint).toBe('path:Archive'); + }); +}); diff --git a/src/core/search/__tests__/search-parser.spec.ts b/src/core/search/__tests__/search-parser.spec.ts new file mode 100644 index 0000000..2add675 --- /dev/null +++ b/src/core/search/__tests__/search-parser.spec.ts @@ -0,0 +1,40 @@ +import { parseSearchQuery } from '../search-parser'; +import { SearchOptions } from '../search-parser.types'; + +describe('parseSearchQuery diagnostics', () => { + const baseOptions: SearchOptions = { + caseSensitive: false, + regexMode: false + }; + + it('captures filters for tag and file operators', () => { + const query = 'tag:#home file:"Project Plan.md"'; + const parsed = parseSearchQuery(query, baseOptions); + + expect(parsed.isEmpty).toBe(false); + expect(parsed.diagnostics).toBeDefined(); + expect(parsed.diagnostics?.filters.tag).toEqual(['#home']); + expect(parsed.diagnostics?.filters.file).toEqual(['Project Plan.md']); + const negative = parsed.diagnostics?.filters.negative ?? []; + expect(negative.length).toBe(0); + }); + + it('tracks negative filters with metadata', () => { + const query = 'tag:#home -path:"Archive" -content:"secret"'; + const parsed = parseSearchQuery(query, baseOptions); + + expect(parsed.diagnostics?.filters.negative).toEqual(['Archive', 'secret']); + const negativeDetails = parsed.diagnostics?.filters.negativeDetails ?? []; + expect(negativeDetails.length).toBe(2); + expect(negativeDetails[0]).toEqual(jasmine.objectContaining({ type: 'path', value: 'Archive' })); + expect(negativeDetails[1]).toEqual(jasmine.objectContaining({ type: 'content', value: 'secret' })); + }); + + it('records warnings for unknown prefixes', () => { + const query = 'unknown:term -unknown:value'; + const parsed = parseSearchQuery(query, baseOptions); + + expect(parsed.diagnostics?.warnings).toContain('UnknownOperator:unknown'); + expect(parsed.diagnostics?.filters.negative).toContain('unknown:value'); + }); +}); diff --git a/src/core/search/search-index.service.ts b/src/core/search/search-index.service.ts index 9860d73..38ec889 100644 --- a/src/core/search/search-index.service.ts +++ b/src/core/search/search-index.service.ts @@ -3,6 +3,31 @@ import { SearchContext, SectionContent, TaskInfo } from './search-parser.types'; import { VaultService } from '../../services/vault.service'; import { Note } from '../../types'; +export interface SearchIndexNormalizationRules { + tags: { + ensureHashPrefix: boolean; + preserveCase: boolean; + deduplicate: boolean; + stripHashOnMatch: boolean; + }; + paths: { + separator: string; + caseSensitive: boolean; + }; + files: { + caseSensitive: boolean; + }; +} + +export interface SearchIndexDiagnosticsSnapshot { + tagsIndexed: boolean; + pathIndexed: boolean; + filesIndexed: boolean; + totalContexts: number; + tagCardinality: Record; + normalizeRules: SearchIndexNormalizationRules; +} + /** * Comprehensive search index for the vault * Indexes all content for fast searching with full Obsidian operator support @@ -17,6 +42,7 @@ export class SearchIndexService { private pathsIndex = signal([]); private filesIndex = signal([]); private tagsIndex = signal([]); + private tagCardinality = signal>({}); private propertiesIndex = signal>>(new Map()); private headingsIndex = signal>(new Map()); @@ -56,6 +82,7 @@ export class SearchIndexService { const pathsSet = new Set(); const filesSet = new Set(); const tagsSet = new Set(); + const tagCounts = new Map(); const propertiesMap = new Map>(); const headingsMap = new Map(); @@ -78,7 +105,12 @@ export class SearchIndexService { filesSet.add(context.fileNameWithExt); // Index tags - context.tags.forEach(tag => tagsSet.add(tag)); + context.tags.forEach(tag => { + tagsSet.add(tag); + const normalized = tag.startsWith('#') ? tag.substring(1) : tag; + const key = normalized.toLowerCase(); + tagCounts.set(key, (tagCounts.get(key) ?? 0) + 1); + }); // Index properties Object.keys(context.properties).forEach(key => { @@ -104,6 +136,11 @@ export class SearchIndexService { this.pathsIndex.set(Array.from(pathsSet).sort()); this.filesIndex.set(Array.from(filesSet).sort()); this.tagsIndex.set(Array.from(tagsSet).sort()); + const tagCardinalityObj: Record = {}; + tagCounts.forEach((count, key) => { + tagCardinalityObj[key] = count; + }); + this.tagCardinality.set(tagCardinalityObj); this.propertiesIndex.set(propertiesMap); this.headingsIndex.set(headingsMap); } @@ -286,6 +323,48 @@ export class SearchIndexService { return Array.from(new Set(all)); } + getIndexDiagnostics(filters?: { tags?: string[] }): SearchIndexDiagnosticsSnapshot { + const tagSnapshot = this.tagCardinality(); + const selectedTags = filters?.tags ?? Object.keys(tagSnapshot); + const tagCardinality: Record = {}; + + selectedTags.forEach(tag => { + const normalized = tag.startsWith('#') ? tag.substring(1) : tag; + const key = normalized.toLowerCase(); + const value = tagSnapshot[key]; + if (value !== undefined) { + tagCardinality[normalized] = value; + } + }); + + return { + tagsIndexed: this.tagsIndex().length > 0, + pathIndexed: this.pathsIndex().length > 0, + filesIndexed: this.filesIndex().length > 0, + totalContexts: this.indexData().size, + tagCardinality, + normalizeRules: this.getNormalizationRules() + }; + } + + getNormalizationRules(): SearchIndexNormalizationRules { + return { + tags: { + ensureHashPrefix: true, + preserveCase: true, + deduplicate: true, + stripHashOnMatch: true + }, + paths: { + separator: '/', + caseSensitive: false + }, + files: { + caseSensitive: false + } + }; + } + /** * Extract tags from YAML frontmatter in raw markdown content. * Supports: diff --git a/src/core/search/search-orchestrator.service.spec.ts b/src/core/search/search-orchestrator.service.spec.ts index 59a3b4b..160698a 100644 --- a/src/core/search/search-orchestrator.service.spec.ts +++ b/src/core/search/search-orchestrator.service.spec.ts @@ -3,6 +3,8 @@ import { provideZonelessChangeDetection } from '@angular/core'; import { SearchOrchestratorService } from './search-orchestrator.service'; import { SearchIndexService } from './search-index.service'; import { SearchContext } from './search-parser.types'; +import { ClientLoggingService } from '../../services/client-logging.service'; +import { SearchLogService } from '../../app/core/logging/log.service'; describe('SearchOrchestratorService', () => { let service: SearchOrchestratorService; @@ -38,14 +40,33 @@ describe('SearchOrchestratorService', () => { contexts = [mockContext]; const indexServiceStub = { - getAllContexts: () => contexts - } as SearchIndexService; + getAllContexts: () => contexts, + getIndexDiagnostics: jasmine.createSpy('getIndexDiagnostics').and.returnValue({ + tagsIndexed: true, + pathIndexed: true, + filesIndexed: true, + totalContexts: 1, + tagCardinality: { test: 1, example: 1 }, + normalizeRules: { + tags: { ensureHashPrefix: true, preserveCase: true, deduplicate: true, stripHashOnMatch: true }, + paths: { separator: '/', caseSensitive: false }, + files: { caseSensitive: false } + } + }) + } as unknown as SearchIndexService; + + const clientLoggerStub = jasmine.createSpyObj('ClientLoggingService', ['info', 'debug', 'error', 'warn']); + const diagLoggerStub = { + logSearch: jasmine.createSpy('logSearch').and.returnValue('test-correlation-id') + } as unknown as SearchLogService; TestBed.configureTestingModule({ providers: [ provideZonelessChangeDetection(), SearchOrchestratorService, - { provide: SearchIndexService, useValue: indexServiceStub } + { provide: SearchIndexService, useValue: indexServiceStub }, + { provide: ClientLoggingService, useValue: clientLoggerStub }, + { provide: SearchLogService, useValue: diagLoggerStub } ] }); service = TestBed.inject(SearchOrchestratorService); diff --git a/src/core/search/search-orchestrator.service.ts b/src/core/search/search-orchestrator.service.ts index 64373d4..c4989e5 100644 --- a/src/core/search/search-orchestrator.service.ts +++ b/src/core/search/search-orchestrator.service.ts @@ -1,8 +1,11 @@ import { Injectable, inject } from '@angular/core'; import { parseSearchQuery, queryToPredicate } from './search-parser'; import { SearchOptions, SearchContext } from './search-parser.types'; -import { SearchIndexService } from './search-index.service'; +import { SearchIndexService, SearchIndexDiagnosticsSnapshot } from './search-index.service'; import { ClientLoggingService } from '../../services/client-logging.service'; +import { SearchLogService } from '../../app/core/logging/log.service'; +import { SearchDiagEvent } from '../../app/core/logging/search-log.model'; +import { ParseDiagnostics } from './search-parser.types'; /** * Match range for highlighting @@ -43,6 +46,16 @@ export interface SearchExecutionOptions extends SearchOptions { contextLines?: number; /** Maximum number of results to return (default: unlimited) */ maxResults?: number; + /** Diagnostic trigger source */ + trigger?: 'submit' | 'debounce' | 'route' | string; + /** Diagnostic source identifier */ + source?: string; + /** Paging page number for diagnostics */ + page?: number; + /** Paging page size for diagnostics */ + pageSize?: number; + /** Existing correlation id to reuse */ + correlationId?: string; } /** @@ -55,6 +68,7 @@ export interface SearchExecutionOptions extends SearchOptions { export class SearchOrchestratorService { private searchIndex = inject(SearchIndexService); private logger = inject(ClientLoggingService); + private diagnosticsLogger = inject(SearchLogService); /** * Execute a search query and return matching notes with highlights @@ -64,87 +78,420 @@ export class SearchOrchestratorService { return []; } + const pipelineStart = this.now(); + let stageAtFailure: SearchDiagEvent = 'SEARCH_DIAG_START'; + const timings = { + parse: 0, + plan: 0, + exec: 0, + map: 0, + render: 0 + }; + let results: SearchResult[] = []; + let allContextsCount = 0; + let combinedCount = 0; + let planInfo: ReturnType | undefined; + let indexDiagnostics: SearchIndexDiagnosticsSnapshot | undefined; + + const correlationId = this.diagnosticsLogger.logSearch('SEARCH_DIAG_START', { + source: options?.source ?? 'ui', + trigger: options?.trigger ?? 'submit', + queryRaw: query, + options: { + caseSensitive: options?.caseSensitive ?? false, + regex: options?.regexMode ?? false, + wholeWord: (options as any)?.wholeWord ?? false + }, + paging: { + page: options?.page ?? 1, + pageSize: options?.pageSize ?? options?.maxResults ?? 0 + } + }, 'info'); + const contextLines = options?.contextLines ?? 2; const maxResults = options?.maxResults; - // Parse the query into an AST - this.logger.info('SearchOrchestrator', 'Parsing query', { query, options }); - const parsed = parseSearchQuery(query, options); - if (parsed.isEmpty) { - this.logger.debug('SearchOrchestrator', 'Parsed query is empty'); + try { + stageAtFailure = 'SEARCH_DIAG_PARSE'; + this.logger.info('SearchOrchestrator', 'Parsing query', { query, options }); + const parseStart = this.now(); + const parsed = parseSearchQuery(query, options); + timings.parse = this.now() - parseStart; + + if (parsed.diagnostics) { + this.diagnosticsLogger.logSearch('SEARCH_DIAG_PARSE', { + queryRaw: query, + tokens: parsed.diagnostics.tokens, + filters: parsed.diagnostics.filters, + parseWarnings: parsed.diagnostics.warnings + }, 'info', correlationId); + } + + if (parsed.isEmpty) { + this.logger.debug('SearchOrchestrator', 'Parsed query is empty'); + return []; + } + + stageAtFailure = 'SEARCH_DIAG_PLAN'; + this.logger.debug('SearchOrchestrator', 'Building predicate'); + const predicate = queryToPredicate(parsed, options); + + this.logger.debug('SearchOrchestrator', 'Extracting search terms'); + const searchTerms = this.extractSearchTerms(parsed.ast); + + const planStart = this.now(); + indexDiagnostics = this.searchIndex.getIndexDiagnostics({ tags: parsed.diagnostics?.filters.tag }); + planInfo = this.buildExecutionPlan(parsed.diagnostics, options, indexDiagnostics); + timings.plan = this.now() - planStart; + + this.diagnosticsLogger.logSearch('SEARCH_DIAG_PLAN', { + queryRaw: query, + stages: planInfo.stages, + indexStats: planInfo.indexStats, + warnings: planInfo.warnings, + normalizeRules: planInfo.normalizeRules + }, planInfo.warnings.length > 0 ? 'warn' : 'info', correlationId); + + stageAtFailure = 'SEARCH_DIAG_EXEC_PROVIDER'; + results = []; + const allContexts = this.searchIndex.getAllContexts(); + allContextsCount = allContexts.length; + this.logger.info('SearchOrchestrator', 'Evaluating contexts', { + contextCount: allContextsCount + }); + + const execStart = this.now(); + for (const context of allContexts) { + this.logger.debug('SearchOrchestrator', 'Evaluating context', { + noteId: context.noteId, + filePath: context.filePath + }); + + if (!predicate(context)) { + this.logger.debug('SearchOrchestrator', 'Context filtered out', { + noteId: context.noteId + }); + continue; + } + + this.logger.debug('SearchOrchestrator', 'Context matched predicate, finding highlights', { + noteId: context.noteId + }); + const { matches, allRanges } = this.findMatchesWithRanges( + context, + searchTerms, + options, + contextLines + ); + + const score = this.calculateScore(context, query, matches); + this.logger.debug('SearchOrchestrator', 'Context scored', { + noteId: context.noteId, + matchCount: matches.length, + score + }); + + results.push({ + noteId: context.noteId, + matches, + score, + allRanges + }); + } + timings.exec = this.now() - execStart; + + const providerOutputCount = results.length; + combinedCount = providerOutputCount; + this.diagnosticsLogger.logSearch('SEARCH_DIAG_EXEC_PROVIDER', { + queryRaw: query, + provider: 'localIndex', + inputCount: allContextsCount, + filterApplied: this.buildFilterApplied(parsed.diagnostics), + outputCount: providerOutputCount, + timingMs: timings.exec, + cache: { hit: false } + }, 'info', correlationId); + + stageAtFailure = 'SEARCH_DIAG_RESULT_MAP'; + const mapStart = this.now(); + this.logger.debug('SearchOrchestrator', 'Sorting results', { + resultCount: results.length + }); + + if (maxResults && maxResults > 0) { + this.logger.debug('SearchOrchestrator', 'Applying maxResults slice', { maxResults }); + results.splice(maxResults); + } + + const providerResults = [{ provider: 'localIndex', count: providerOutputCount }]; + const afterUIFiltersCount = combinedCount; + const reasonsEmpty = indexDiagnostics && planInfo + ? this.collectEmptyReasons(results.length, planInfo, indexDiagnostics) + : []; + timings.map = this.now() - mapStart; + + this.diagnosticsLogger.logSearch('SEARCH_DIAG_RESULT_MAP', { + queryRaw: query, + providerResults, + combinedCount, + afterUIFiltersCount, + reasonsEmpty + }, reasonsEmpty.length > 0 ? 'warn' : 'info', correlationId); + + this.logger.info('SearchOrchestrator', 'Returning results', { + resultCount: results.length + }); + + stageAtFailure = 'SEARCH_DIAG_SUMMARY'; + const totalMs = this.now() - pipelineStart; + timings.render = Math.max(0, totalMs - (timings.parse + timings.plan + timings.exec + timings.map)); + const summaryLevel: 'info' | 'warn' = results.length === 0 || (planInfo?.warnings.length ?? 0) > 0 ? 'warn' : 'info'; + + this.diagnosticsLogger.logSearch('SEARCH_DIAG_SUMMARY', { + queryRaw: query, + totalMs, + stagesMs: timings, + counts: { + beforeFilters: allContextsCount, + afterFilters: combinedCount, + displayed: results.length + }, + index: indexDiagnostics ? { + tagsIndexed: indexDiagnostics.tagsIndexed, + tagCardinality: indexDiagnostics.tagCardinality + } : undefined, + userVisible: { + emptyStateShown: results.length === 0, + helperSuggestions: planInfo ? this.buildHelperSuggestions(planInfo, results.length) : [] + } + }, summaryLevel, correlationId); + + return results; + } catch (error) { + const errorKind = this.inferErrorKind(stageAtFailure, error); + this.diagnosticsLogger.logSearch('SEARCH_DIAG_ERROR', { + queryRaw: query, + kind: errorKind, + message: error instanceof Error ? error.message : String(error), + code: (error as any)?.code, + safeStack: this.extractSafeStack(error), + stageAtFailure, + partialCounts: { + beforeFilters: allContextsCount, + afterFilters: combinedCount + } + }, 'error', correlationId); + this.logger.error('SearchOrchestrator', 'Search pipeline failed', { + stage: stageAtFailure, + error + }); + throw error; + } + } + + private inferErrorKind(stage: SearchDiagEvent, error: unknown): 'ParseError' | 'IndexError' | 'ProviderError' | 'Timeout' | 'Unknown' { + if ((error as any)?.name === 'TimeoutError') { + return 'Timeout'; + } + + switch (stage) { + case 'SEARCH_DIAG_PARSE': + return 'ParseError'; + case 'SEARCH_DIAG_PLAN': + return 'IndexError'; + case 'SEARCH_DIAG_EXEC_PROVIDER': + return 'ProviderError'; + default: + return 'Unknown'; + } + } + + private extractSafeStack(error: unknown): string | undefined { + if (error instanceof Error && typeof error.stack === 'string') { + return error.stack.split('\n').slice(0, 5).join('\n'); + } + return undefined; + } + + private now(): number { + if (typeof performance !== 'undefined' && typeof performance.now === 'function') { + return performance.now(); + } + return Date.now(); + } + + private buildExecutionPlan( + diagnostics: ParseDiagnostics | undefined, + options: SearchExecutionOptions | undefined, + indexDiagnostics: SearchIndexDiagnosticsSnapshot + ): { + stages: Array<{ name: string; estimate: number; constraint: string }>; + warnings: string[]; + indexStats: { + tagsIndexed: boolean; + tagCardinality: Record; + pathIndexed: boolean; + filesIndexed: boolean; + }; + normalizeRules: SearchIndexDiagnosticsSnapshot['normalizeRules']; + } { + const stages: Array<{ name: string; estimate: number; constraint: string }> = []; + const warnings: string[] = []; + const filters = diagnostics?.filters; + + if (filters?.tag?.length) { + filters.tag.forEach(tag => { + const normalized = tag.startsWith('#') ? tag.substring(1) : tag; + const key = normalized.toLowerCase(); + const count = indexDiagnostics.tagCardinality[key]; + stages.push({ + name: 'candidateSetFromTag', + estimate: count ?? 0, + constraint: `tag=${normalized}` + }); + if (!indexDiagnostics.tagsIndexed) { + warnings.push('TagIndexUnavailable'); + } else if (count === undefined) { + warnings.push(`TagNotInIndex:${normalized}`); + } else if (count === 0) { + warnings.push(`TagEmpty:${normalized}`); + } + }); + } + + if (filters?.path?.length) { + filters.path.forEach(path => { + stages.push({ + name: 'intersectPath', + estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 4)), + constraint: `path=${path}` + }); + }); + } + + if (filters?.file?.length) { + filters.file.forEach(file => { + stages.push({ + name: 'intersectFile', + estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 6)), + constraint: `file=${file}` + }); + }); + } + + if (filters?.negativeDetails?.length) { + const constraint = filters.negativeDetails.map(n => `${n.type}:${n.value}`).join(', '); + stages.push({ + name: 'applyNegative', + estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 8)), + constraint + }); + } + + if (options?.regexMode || filters?.regex) { + stages.push({ + name: 'postFilterRegex', + estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 3)), + constraint: `regex=${options?.regexMode ?? filters?.regex}` + }); + } + + if (filters?.caseSensitive) { + stages.push({ + name: 'postFilterCaseSensitive', + estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 2)), + constraint: 'caseSensitive=true' + }); + } + + if (!stages.length) { + stages.push({ + name: 'fullScan', + estimate: indexDiagnostics.totalContexts, + constraint: 'none' + }); + } + + if (!indexDiagnostics.tagsIndexed && filters?.tag?.length) { + warnings.push('E_TAG_INDEX_DISABLED'); + } + + return { + stages, + warnings, + indexStats: { + tagsIndexed: indexDiagnostics.tagsIndexed, + tagCardinality: indexDiagnostics.tagCardinality, + pathIndexed: indexDiagnostics.pathIndexed, + filesIndexed: indexDiagnostics.filesIndexed + }, + normalizeRules: indexDiagnostics.normalizeRules + }; + } + + private buildFilterApplied(diagnostics: ParseDiagnostics | undefined): Record { + if (!diagnostics) { + return {}; + } + + return { + tag: diagnostics.filters.tag, + path: diagnostics.filters.path, + file: diagnostics.filters.file, + negative: diagnostics.filters.negative + }; + } + + private collectEmptyReasons( + resultCount: number, + planInfo: ReturnType, + indexDiagnostics: SearchIndexDiagnosticsSnapshot + ): string[] { + if (resultCount > 0) { return []; } - // Convert to predicate function - this.logger.debug('SearchOrchestrator', 'Building predicate'); - const predicate = queryToPredicate(parsed, options); - - // Extract search terms for highlighting - this.logger.debug('SearchOrchestrator', 'Extracting search terms'); - const searchTerms = this.extractSearchTerms(parsed.ast); - - // Evaluate against all indexed contexts - const results: SearchResult[] = []; - const allContexts = this.searchIndex.getAllContexts(); - this.logger.info('SearchOrchestrator', 'Evaluating contexts', { - contextCount: allContexts.length - }); - - for (const context of allContexts) { - this.logger.debug('SearchOrchestrator', 'Evaluating context', { - noteId: context.noteId, - filePath: context.filePath - }); - - // Apply the predicate filter (if it doesn't match, skip this context) - if (!predicate(context)) { - this.logger.debug('SearchOrchestrator', 'Context filtered out', { - noteId: context.noteId - }); - continue; + const reasons = new Set(); + planInfo.warnings.forEach(warning => { + if (warning.startsWith('TagNotInIndex')) { + reasons.add('tagNotInIndex'); + } else if (warning.startsWith('TagEmpty')) { + reasons.add('noMatchesForTag'); + } else if (warning === 'TagIndexUnavailable' || warning === 'E_TAG_INDEX_DISABLED') { + reasons.add('tagIndexUnavailable'); } - - // Find matches and ranges for highlighting (may be empty for file/path/property queries) - this.logger.debug('SearchOrchestrator', 'Context matched predicate, finding highlights', { - noteId: context.noteId - }); - const { matches, allRanges } = this.findMatchesWithRanges( - context, - searchTerms, - options, - contextLines - ); - - const score = this.calculateScore(context, query, matches); - this.logger.debug('SearchOrchestrator', 'Context scored', { - noteId: context.noteId, - matchCount: matches.length, - score - }); - - results.push({ - noteId: context.noteId, - matches, - score, - allRanges - }); - } - - // Sort by score (descending) - this.logger.debug('SearchOrchestrator', 'Sorting results', { - resultCount: results.length }); - // Apply max results limit if specified - if (maxResults && maxResults > 0) { - this.logger.debug('SearchOrchestrator', 'Applying maxResults slice', { maxResults }); - return results.slice(0, maxResults); + if (!indexDiagnostics.filesIndexed) { + reasons.add('fileIndexUnavailable'); + } + if (!indexDiagnostics.pathIndexed) { + reasons.add('pathIndexUnavailable'); } - this.logger.info('SearchOrchestrator', 'Returning results', { - resultCount: results.length - }); - return results; + if (!reasons.size) { + reasons.add('intersectionEmpty'); + } + + return Array.from(reasons); + } + + private buildHelperSuggestions(planInfo: ReturnType, resultCount: number): string[] { + const suggestions: string[] = []; + + if (resultCount === 0) { + if (planInfo.warnings.some(w => w.startsWith('TagNotInIndex'))) { + suggestions.push('Vérifier que le tag existe dans le vault'); + } + if (planInfo.warnings.includes('TagIndexUnavailable') || planInfo.warnings.includes('E_TAG_INDEX_DISABLED')) { + suggestions.push('Reconstruire l’index des tags'); + } + if (!suggestions.length) { + suggestions.push('Assouplir les filtres ou retirer les exclusions'); + } + } + + return suggestions.slice(0, 3); } /** * Extract search terms from AST for highlighting diff --git a/src/core/search/search-parser.ts b/src/core/search/search-parser.ts index 8786efd..c475252 100644 --- a/src/core/search/search-parser.ts +++ b/src/core/search/search-parser.ts @@ -13,7 +13,9 @@ import { SearchContext, SearchOptions, SectionContent, - TaskInfo + TaskInfo, + ParseDiagnostics, + SearchFilterDiagnostics } from './search-parser.types'; // Re-export types for convenience @@ -26,16 +28,27 @@ export function parseSearchQuery(query: string, options?: SearchOptions): Parsed if (!query || !query.trim()) { return { ast: { type: 'group', operator: 'AND', terms: [] }, - isEmpty: true + isEmpty: true, + diagnostics: { + tokens: [], + filters: createEmptyDiagnosticsFilters(options), + warnings: [] + } }; } const tokens = tokenize(query); - const ast = parseTokens(tokens, options); + const diagnostics: ParseDiagnostics = { + tokens: [...tokens], + filters: createEmptyDiagnosticsFilters(options), + warnings: [] + }; + const ast = parseTokens(tokens, options, diagnostics); return { ast, - isEmpty: false + isEmpty: false, + diagnostics }; } @@ -63,21 +76,39 @@ function tokenize(query: string): string[] { // Handle quoted strings if (char === '"') { - if (current) { - tokens.push(current); - current = ''; - } - let quoted = ''; - i++; - while (i < query.length && query[i] !== '"') { - quoted += query[i]; + // If we are inside a prefix token like file: or path:, attach the quoted part to current + if (current.includes(':') && !current.includes(' ')) { + let quoted = '"'; i++; + while (i < query.length && query[i] !== '"') { + quoted += query[i]; + i++; + } + if (i < query.length && query[i] === '"') { + quoted += '"'; + i++; + } + current += quoted; + continue; + } else { + if (current) { + tokens.push(current); + current = ''; + } + let quoted = ''; + i++; + while (i < query.length && query[i] !== '"') { + quoted += query[i]; + i++; + } + if (i < query.length && query[i] === '"') { + i++; + } + if (quoted) { + tokens.push(`"${quoted}"`); + } + continue; } - if (quoted) { - tokens.push(`"${quoted}"`); - } - i++; - continue; } // Handle regex patterns /.../ @@ -137,7 +168,7 @@ function tokenize(query: string): string[] { /** * Parse tokens into AST */ -function parseTokens(tokens: string[], options?: SearchOptions): SearchNode { +function parseTokens(tokens: string[], options: SearchOptions | undefined, diagnostics: ParseDiagnostics): SearchNode { const terms: SearchNode[] = []; let i = 0; @@ -158,14 +189,14 @@ function parseTokens(tokens: string[], options?: SearchOptions): SearchNode { // Handle parentheses if (token === '(') { - const { node, endIndex } = parseGroup(tokens, i + 1, options); + const { node, endIndex } = parseGroup(tokens, i + 1, options, diagnostics); terms.push(node); i = endIndex + 1; continue; } // Parse term - const term = parseTerm(token, options); + const term = parseTerm(token, options, diagnostics); if (term) { terms.push(term); } @@ -186,7 +217,12 @@ function parseTokens(tokens: string[], options?: SearchOptions): SearchNode { /** * Parse a group enclosed in parentheses */ -function parseGroup(tokens: string[], startIndex: number, options?: SearchOptions): { node: SearchNode; endIndex: number } { +function parseGroup( + tokens: string[], + startIndex: number, + options: SearchOptions | undefined, + diagnostics: ParseDiagnostics +): { node: SearchNode; endIndex: number } { const terms: SearchNode[] = []; let i = startIndex; let depth = 1; @@ -204,7 +240,7 @@ function parseGroup(tokens: string[], startIndex: number, options?: SearchOption } if (token.toUpperCase() !== 'OR' && token.toUpperCase() !== 'AND' && token !== '(' && token !== ')') { - const term = parseTerm(token, options); + const term = parseTerm(token, options, diagnostics); if (term) { terms.push(term); } @@ -229,30 +265,34 @@ function parseGroup(tokens: string[], startIndex: number, options?: SearchOption /** * Parse a single search term */ -function parseTerm(token: string, options?: SearchOptions): SearchTerm | null { - if (!token) return null; +function parseTerm(token: string, options: SearchOptions | undefined, diagnostics: ParseDiagnostics): SearchTerm | null { + if (!token) { + return null; + } let negated = false; let value = token; - // Handle negation if (value.startsWith('-')) { negated = true; value = value.substring(1); } + let term: SearchTerm | null = null; + let negativeValue: string | undefined; + // Support property form: [key]:value (Obsidian compatibility) if (value.startsWith('[') && value.includes(']:')) { const closeBracket = value.indexOf(']'); if (closeBracket > 0) { const propertyKey = value.substring(1, closeBracket); - let propertyValue = value.substring(closeBracket + 2); // after ]: + let propertyValue = value.substring(closeBracket + 2); let propValueQuoted = false; if (propertyValue.startsWith('"') && propertyValue.endsWith('"')) { propValueQuoted = true; propertyValue = propertyValue.substring(1, propertyValue.length - 1); } - return { + term = { type: 'property', value: propertyValue, propertyKey, @@ -260,96 +300,132 @@ function parseTerm(token: string, options?: SearchOptions): SearchTerm | null { quoted: propValueQuoted, wildcard: propertyValue.includes('*') }; + negativeValue = propertyValue || propertyKey; } } - // Handle regex patterns /.../ - if (value.startsWith('/') && value.endsWith('/') && value.length > 2) { + if (!term && value.startsWith('/') && value.endsWith('/') && value.length > 2) { const regexPattern = value.substring(1, value.length - 1); - return { type: 'regex', value: regexPattern, negated, quoted: false, wildcard: false }; + term = { type: 'regex', value: regexPattern, negated, quoted: false, wildcard: false }; + diagnostics.filters.regex = true; + negativeValue = regexPattern; } - // Handle quoted strings let quoted = false; - if (value.startsWith('"') && value.endsWith('"')) { + if (!term && value.startsWith('"') && value.endsWith('"')) { quoted = true; value = value.substring(1, value.length - 1); } - // Handle wildcards const wildcard = value.includes('*'); - // Check for prefixes - const colonIndex = value.indexOf(':'); - if (colonIndex > 0) { - const prefix = value.substring(0, colonIndex).toLowerCase(); - const searchValue = value.substring(colonIndex + 1); + if (!term) { + const colonIndex = value.indexOf(':'); + if (colonIndex > 0) { + const prefix = value.substring(0, colonIndex).toLowerCase(); + const searchValueRaw = value.substring(colonIndex + 1); + let cleanValue = searchValueRaw; + let valueQuoted = false; + if (cleanValue.startsWith('"') && cleanValue.endsWith('"')) { + valueQuoted = true; + cleanValue = cleanValue.substring(1, cleanValue.length - 1); + } - // Remove quotes from search value if present - let cleanValue = searchValue; - let valueQuoted = false; - if (cleanValue.startsWith('"') && cleanValue.endsWith('"')) { - valueQuoted = true; - cleanValue = cleanValue.substring(1, cleanValue.length - 1); - } - - // Fallback: handle property form [key]:value (if missed previously) - if (prefix.startsWith('[') && prefix.endsWith(']')) { - const propertyKey = prefix.substring(1, prefix.length - 1); - return { - type: 'property', - value: cleanValue, - propertyKey, - negated, - quoted: valueQuoted, - wildcard: cleanValue.includes('*') - }; - } - - switch (prefix) { - case 'path': - return { type: 'path', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'file': - return { type: 'file', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'content': - return { type: 'content', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'tag': - return { type: 'tag', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'line': - return { type: 'line', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'block': - return { type: 'block', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'section': - return { type: 'section', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'task': - return { type: 'task', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'task-todo': - return { type: 'task-todo', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'task-done': - return { type: 'task-done', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; - case 'match-case': - return { type: 'match-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: true }; - case 'ignore-case': - return { type: 'ignore-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: false }; + if (prefix.startsWith('[') && prefix.endsWith(']')) { + const propertyKey = prefix.substring(1, prefix.length - 1); + term = { + type: 'property', + value: cleanValue, + propertyKey, + negated, + quoted: valueQuoted, + wildcard: cleanValue.includes('*') + }; + negativeValue = cleanValue || propertyKey; + } else { + switch (prefix) { + case 'path': + if (!negated) { + diagnostics.filters.path.push(cleanValue); + } + term = { type: 'path', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'file': + if (!negated) { + diagnostics.filters.file.push(cleanValue); + } + term = { type: 'file', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'content': + term = { type: 'content', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'tag': + if (!negated) { + diagnostics.filters.tag.push(cleanValue); + } + term = { type: 'tag', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'line': + term = { type: 'line', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'block': + term = { type: 'block', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'section': + term = { type: 'section', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'task': + term = { type: 'task', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'task-todo': + term = { type: 'task-todo', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'task-done': + term = { type: 'task-done', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; + negativeValue = cleanValue; + break; + case 'match-case': + diagnostics.filters.caseSensitive = true; + term = { type: 'match-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: true }; + negativeValue = cleanValue; + break; + case 'ignore-case': + diagnostics.filters.caseSensitive = false; + term = { type: 'ignore-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: false }; + negativeValue = cleanValue; + break; + default: + diagnostics.warnings.push(`UnknownOperator:${prefix}`); + // treat as text with original value including prefix + term = { type: 'text', value, negated, quoted, wildcard }; + negativeValue = value; + break; + } + } } } - // Check for property search [property:value] or [property] - if (value.startsWith('[') && value.endsWith(']')) { + if (!term && value.startsWith('[') && value.endsWith(']')) { const inner = value.substring(1, value.length - 1); const propColonIndex = inner.indexOf(':'); if (propColonIndex > 0) { const propertyKey = inner.substring(0, propColonIndex); let propertyValue = inner.substring(propColonIndex + 1); let propValueQuoted = false; - - // Remove quotes from property value if (propertyValue.startsWith('"') && propertyValue.endsWith('"')) { propValueQuoted = true; propertyValue = propertyValue.substring(1, propertyValue.length - 1); } - - return { + term = { type: 'property', value: propertyValue, propertyKey, @@ -357,20 +433,52 @@ function parseTerm(token: string, options?: SearchOptions): SearchTerm | null { quoted: propValueQuoted, wildcard: propertyValue.includes('*') }; + negativeValue = propertyValue || propertyKey; + } else { + term = { + type: 'property', + value: '', + propertyKey: inner, + negated, + quoted: false, + wildcard: false + }; + negativeValue = inner; } - // Property existence check [property] - return { - type: 'property', - value: '', - propertyKey: inner, - negated, - quoted: false, - wildcard: false - }; } - // Default: text search - return { type: 'text', value, negated, quoted, wildcard }; + if (!term) { + term = { type: 'text', value, negated, quoted, wildcard }; + negativeValue = value; + } + + if (negated && negativeValue && term) { + diagnostics.filters.negative.push(negativeValue); + diagnostics.filters.negativeDetails.push({ + type: term.type, + value: negativeValue, + wildcard: 'wildcard' in term ? Boolean((term as any).wildcard) : undefined + }); + } + + if (term.type === 'regex') { + diagnostics.filters.regex = true; + } + + return term; +} + +function createEmptyDiagnosticsFilters(options?: SearchOptions): SearchFilterDiagnostics { + return { + tag: [], + path: [], + file: [], + negative: [], + negativeDetails: [], + regex: options?.regexMode ?? false, + caseSensitive: options?.caseSensitive ?? false, + wholeWord: (options as any)?.wholeWord ?? false + }; } /** diff --git a/src/core/search/search-parser.types.ts b/src/core/search/search-parser.types.ts index d4cf689..2ae0dc2 100644 --- a/src/core/search/search-parser.types.ts +++ b/src/core/search/search-parser.types.ts @@ -41,9 +41,33 @@ export interface SearchGroup { export type SearchNode = SearchTerm | SearchGroup; +export interface NegativeFilterDiagnostic { + type: SearchTermType; + value: string; + wildcard?: boolean; +} + +export interface SearchFilterDiagnostics { + tag: string[]; + path: string[]; + file: string[]; + negative: string[]; + negativeDetails: NegativeFilterDiagnostic[]; + regex: boolean; + caseSensitive: boolean; + wholeWord: boolean; +} + +export interface ParseDiagnostics { + tokens: string[]; + filters: SearchFilterDiagnostics; + warnings: string[]; +} + export interface ParsedQuery { ast: SearchNode; isEmpty: boolean; + diagnostics?: ParseDiagnostics; } /** Predicate function to test if content matches the query */ diff --git a/src/test-setup.ts b/src/test-setup.ts new file mode 100644 index 0000000..1e7853d --- /dev/null +++ b/src/test-setup.ts @@ -0,0 +1,14 @@ +// Test setup file for Karma/Jasmine +import 'zone.js'; +import 'zone.js/testing'; +import { getTestBed } from '@angular/core/testing'; +import { + BrowserDynamicTestingModule, + platformBrowserDynamicTesting +} from '@angular/platform-browser-dynamic/testing'; + +// First, initialize the Angular testing environment. +getTestBed().initTestEnvironment( + BrowserDynamicTestingModule, + platformBrowserDynamicTesting(), +); diff --git a/test-parser-debug.js b/test-parser-debug.js new file mode 100644 index 0000000..1109a91 --- /dev/null +++ b/test-parser-debug.js @@ -0,0 +1,19 @@ +// Quick test to debug parser behavior +const { parseSearchQuery } = require('./dist/core/search/search-parser'); + +const query1 = 'tag:#home file:"Project Plan.md"'; +const parsed1 = parseSearchQuery(query1, { caseSensitive: false, regexMode: false }); + +console.log('Query 1:', query1); +console.log('Tokens:', parsed1.diagnostics?.tokens); +console.log('Tag filters:', parsed1.diagnostics?.filters.tag); +console.log('File filters:', parsed1.diagnostics?.filters.file); +console.log('---'); + +const query2 = 'tag:#home -path:"Archive" -content:"secret"'; +const parsed2 = parseSearchQuery(query2, { caseSensitive: false, regexMode: false }); + +console.log('Query 2:', query2); +console.log('Tokens:', parsed2.diagnostics?.tokens); +console.log('Negative filters:', parsed2.diagnostics?.filters.negative); +console.log('Negative details:', parsed2.diagnostics?.filters.negativeDetails); diff --git a/tsconfig.spec.json b/tsconfig.spec.json index 3a86045..39b4b7d 100644 --- a/tsconfig.spec.json +++ b/tsconfig.spec.json @@ -9,6 +9,9 @@ "allowJs": false, "types": ["jasmine", "node"] }, + "files": [ + "src/test-setup.ts" + ], "include": [ "src/**/*.spec.ts", "src/**/*.d.ts" diff --git a/vault/.obsidian/graph.json b/vault/.obsidian/graph.json index 8bc5a09..339d3ce 100644 --- a/vault/.obsidian/graph.json +++ b/vault/.obsidian/graph.json @@ -1,22 +1,22 @@ { "collapse-filter": false, "search": "", - "showTags": true, + "showTags": false, "showAttachments": false, "hideUnresolved": false, - "showOrphans": false, + "showOrphans": true, "collapse-color-groups": false, "colorGroups": [], "collapse-display": false, "showArrow": false, - "textFadeMultiplier": -3, - "nodeSizeMultiplier": 0.25, - "lineSizeMultiplier": 1.45, + "textFadeMultiplier": 0, + "nodeSizeMultiplier": 1, + "lineSizeMultiplier": 1, "collapse-forces": false, - "centerStrength": 0.27, - "repelStrength": 10, - "linkStrength": 0.15, - "linkDistance": 102, - "scale": 1.4019828977761002, + "centerStrength": 0.3, + "repelStrength": 17, + "linkStrength": 0.5, + "linkDistance": 200, + "scale": 1, "close": false } \ No newline at end of file