feat: add search diagnostics logging and tag cardinality tracking

This commit is contained in:
Bruno Charest 2025-10-05 16:24:19 -04:00
parent 1ddfd18af3
commit 989f3ee25a
15 changed files with 1567 additions and 189 deletions

View File

@ -1,4 +1,28 @@
import { test, expect } from '@playwright/test'; import { test, expect, Page } from '@playwright/test';
const SEARCH_QUEUE_KEY = 'obsiviewer.searchdiag.queue';
async function clearDiagnostics(page: Page): Promise<void> {
await page.evaluate(key => {
localStorage.removeItem(key);
}, SEARCH_QUEUE_KEY);
}
async function readDiagnostics(page: Page): Promise<any[]> {
const raw = await page.evaluate(key => localStorage.getItem(key), SEARCH_QUEUE_KEY);
if (!raw) {
return [];
}
try {
return JSON.parse(raw);
} catch {
return [];
}
}
function findStage(events: any[], stage: string): any | undefined {
return events.find(event => event?.stage === stage);
}
test.describe('Search Functionality', () => { test.describe('Search Functionality', () => {
test.beforeEach(async ({ page }) => { test.beforeEach(async ({ page }) => {
@ -69,6 +93,7 @@ test.describe('Search Functionality', () => {
}); });
test('should filter by tag', async ({ page }) => { test('should filter by tag', async ({ page }) => {
await clearDiagnostics(page);
const searchInput = page.locator('input[type="text"]').first(); const searchInput = page.locator('input[type="text"]').first();
await searchInput.fill('tag:#work'); await searchInput.fill('tag:#work');
@ -79,6 +104,32 @@ test.describe('Search Functionality', () => {
// Should show results with #work tag // Should show results with #work tag
const results = page.locator('.search-result'); const results = page.locator('.search-result');
await expect(results.first()).toBeVisible({ timeout: 5000 }); await expect(results.first()).toBeVisible({ timeout: 5000 });
const events = await readDiagnostics(page);
const summary = findStage(events, 'SEARCH_DIAG_SUMMARY');
expect(summary).toBeTruthy();
expect(summary.data?.counts?.displayed).toBeGreaterThan(0);
expect(summary.data?.userVisible?.emptyStateShown).toBe(false);
});
test('should record diagnostics when tag search misses index', async ({ page }) => {
await clearDiagnostics(page);
const searchInput = page.locator('input[type="text"]').first();
await searchInput.fill('tag:#home');
await searchInput.press('Enter');
await page.waitForTimeout(500);
const events = await readDiagnostics(page);
const summary = findStage(events, 'SEARCH_DIAG_SUMMARY');
expect(summary).toBeTruthy();
expect(summary.data?.counts?.displayed ?? 0).toBe(0);
expect(summary.data?.userVisible?.emptyStateShown).toBe(true);
const resultMap = findStage(events, 'SEARCH_DIAG_RESULT_MAP');
expect(resultMap).toBeTruthy();
expect(resultMap.data?.reasonsEmpty).toEqual(expect.arrayContaining(['tagNotInIndex']));
}); });
test('should toggle case sensitivity', async ({ page }) => { test('should toggle case sensitivity', async ({ page }) => {

View File

@ -0,0 +1,513 @@
import { Injectable, PLATFORM_ID, inject } from '@angular/core';
import { isPlatformBrowser } from '@angular/common';
import { environment } from '../../../core/logging/environment';
import { SearchDiagEvent, SearchDiagRecord } from './search-log.model';
const SESSION_STORAGE_KEY = 'obsiviewer.searchdiag.sessionId';
const LOCAL_STORAGE_QUEUE_KEY = 'obsiviewer.searchdiag.queue';
const SAMPLE_DECISIONS_LIMIT = 1000;
const MAX_DATA_SIZE = 5 * 1024; // 5KB
const LARGE_ARRAY_LIMIT = 200;
type SearchLogLevel = 'info' | 'warn' | 'error';
interface SanitizedDictionary {
[key: string]: SanitizedValue;
}
type SanitizedValue =
| string
| number
| boolean
| null
| undefined
| SanitizedValue[]
| SanitizedDictionary;
type Dictionary<T = unknown> = Record<string, T>;
@Injectable({ providedIn: 'root' })
export class SearchLogService {
private readonly platformId = inject(PLATFORM_ID);
private readonly isBrowser = isPlatformBrowser(this.platformId);
private readonly config = environment.logging;
private readonly sessionId = this.ensureSessionId();
private queue: SearchDiagRecord[] = [];
private flushTimer?: ReturnType<typeof setTimeout>;
private consecutiveFailures = 0;
private circuitBreakerOpenUntil = 0;
private isFlushing = false;
private readonly correlationQueries = new Map<string, string>();
private readonly sampleDecisions = new Map<string, boolean>();
constructor() {
if (this.isBrowser) {
this.loadQueueFromStorage();
if (typeof window !== 'undefined') {
window.addEventListener('online', () => this.scheduleFlush());
window.addEventListener('beforeunload', () => this.flushSync());
}
}
}
logSearch(
stage: SearchDiagEvent,
data: Record<string, unknown>,
level: SearchLogLevel = 'info',
correlationId?: string
): string {
if (!this.isBrowser || !this.config.enabled) {
return correlationId ?? '';
}
const effectiveCorrelationId = correlationId || this.generateUUID();
if (!this.shouldSample(effectiveCorrelationId)) {
return effectiveCorrelationId;
}
const queryRaw = this.extractQueryRaw(effectiveCorrelationId, stage, data);
const record: SearchDiagRecord = {
ts: new Date().toISOString(),
app: 'ObsiViewer',
sessionId: this.sessionId,
level,
correlationId: effectiveCorrelationId,
queryRaw,
context: this.buildContext(),
stage,
data: this.prepareData(stage, data),
userAgent: this.getUserAgent(),
};
this.enqueue(record);
return effectiveCorrelationId;
}
async flush(): Promise<void> {
if (!this.isBrowser) {
return;
}
clearTimeout(this.flushTimer);
this.flushTimer = undefined;
await this.processQueue();
}
private flushSync(): void {
if (!this.isBrowser || this.queue.length === 0) {
return;
}
const payload = this.queue.slice();
this.queue = [];
this.persistQueue();
if ('sendBeacon' in navigator) {
const blob = new Blob([JSON.stringify(payload)], { type: 'application/json' });
navigator.sendBeacon(this.config.endpoint, blob);
}
}
private enqueue(record: SearchDiagRecord): void {
this.queue.push(record);
this.persistQueue();
if (this.queue.length >= this.config.batchSize) {
this.scheduleFlush(0);
} else {
this.scheduleFlush();
}
}
private scheduleFlush(delayMs = this.config.debounceMs): void {
if (!this.isBrowser) {
return;
}
if (this.flushTimer) {
return;
}
this.flushTimer = setTimeout(() => {
this.flushTimer = undefined;
if (typeof window !== 'undefined' && 'requestIdleCallback' in window) {
requestIdleCallback(() => this.processQueue());
} else {
this.processQueue();
}
}, delayMs);
}
private async processQueue(): Promise<void> {
if (!this.isBrowser) {
return;
}
if (this.isFlushing || this.queue.length === 0) {
return;
}
if (Date.now() < this.circuitBreakerOpenUntil) {
return;
}
if (typeof navigator !== 'undefined' && !navigator.onLine) {
return;
}
this.isFlushing = true;
const batch = [...this.queue];
try {
await this.sendWithRetry(batch);
this.queue = [];
this.consecutiveFailures = 0;
this.persistQueue();
} catch (error) {
this.consecutiveFailures++;
if (this.consecutiveFailures >= this.config.circuitBreakerThreshold) {
this.circuitBreakerOpenUntil = Date.now() + this.config.circuitBreakerResetMs;
}
console.warn('[SearchLogService] Failed to send diagnostics batch', error);
} finally {
this.isFlushing = false;
}
}
private async sendWithRetry(batch: SearchDiagRecord[]): Promise<void> {
let lastError: unknown;
for (let attempt = 0; attempt < this.config.maxRetries; attempt++) {
try {
await this.postBatch(batch);
return;
} catch (error) {
lastError = error;
if (attempt < this.config.maxRetries - 1) {
const delayMs = 500 * Math.pow(2, attempt);
await this.delay(delayMs);
}
}
}
throw lastError;
}
private async postBatch(batch: SearchDiagRecord[]): Promise<void> {
if (batch.length === 0) {
return;
}
const payload = batch.length === 1 ? batch[0] : batch;
const response = await fetch(this.config.endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
keepalive: true,
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
}
private buildContext(): SearchDiagRecord['context'] {
const context: SearchDiagRecord['context'] = {
version: environment.appVersion,
};
if (!this.isBrowser) {
return context;
}
if (typeof window !== 'undefined' && window.location) {
context.route = window.location.pathname + window.location.search;
}
if (typeof document !== 'undefined' && document.documentElement) {
const theme = document.documentElement.getAttribute('data-theme');
if (theme === 'light' || theme === 'dark') {
context.theme = theme;
}
}
try {
const vault = localStorage.getItem('obsiviewer.vaultName');
if (vault) {
context.vault = vault;
}
} catch {
// ignore storage errors
}
return context;
}
private getUserAgent(): string | undefined {
if (!this.isBrowser || typeof navigator === 'undefined') {
return undefined;
}
return navigator.userAgent;
}
private prepareData(stage: SearchDiagEvent, data: Record<string, unknown>): Dictionary {
const cloned = this.cloneData(data);
delete cloned.queryRaw;
delete cloned['noteContent'];
const sanitized = this.sanitizeValue(cloned, [stage]) as Dictionary;
return this.enforceSizeLimit(sanitized);
}
private sanitizeValue(value: unknown, path: string[], seen = new WeakSet<object>()): SanitizedValue {
if (value === null || value === undefined) {
return value as SanitizedValue;
}
if (typeof value === 'number' || typeof value === 'boolean') {
return value;
}
if (typeof value === 'string') {
return this.sanitizeString(value, path);
}
if (Array.isArray(value)) {
if (seen.has(value)) {
return '[Circular]' as SanitizedValue;
}
seen.add(value);
const limited = value.slice(0, LARGE_ARRAY_LIMIT);
const sanitizedArray = limited.map((entry, index) => this.sanitizeValue(entry, [...path, String(index)], seen));
if (value.length > LARGE_ARRAY_LIMIT) {
sanitizedArray.push('[Truncated]' as unknown as SanitizedValue);
}
seen.delete(value);
return sanitizedArray;
}
if (typeof value === 'object') {
if (seen.has(value as object)) {
return '[Circular]' as SanitizedValue;
}
seen.add(value as object);
const result: Dictionary<SanitizedValue> = {};
for (const [key, entry] of Object.entries(value as Dictionary)) {
const lowerKey = key.toLowerCase();
if (this.isContentKey(lowerKey)) {
result[key] = '[REDACTED_CONTENT]';
continue;
}
if (this.isSensitivePathKey(lowerKey)) {
result[key] = this.maskPath(String(entry));
continue;
}
result[key] = this.sanitizeValue(entry, [...path, key], seen);
}
seen.delete(value as object);
return result;
}
return String(value);
}
private sanitizeString(value: string, path: string[]): string {
const lowerPath = path[path.length - 1]?.toLowerCase() ?? '';
if (this.isContentKey(lowerPath)) {
return '[REDACTED_CONTENT]';
}
if (this.isSensitivePathKey(lowerPath)) {
return this.maskPath(value);
}
if (value.length > MAX_DATA_SIZE) {
return `${value.slice(0, 512)}…[TRUNCATED_${value.length}]`;
}
return value;
}
private enforceSizeLimit<T>(value: T): T {
try {
const serialized = JSON.stringify(value);
if (serialized.length > MAX_DATA_SIZE) {
return {
_truncated: true,
_originalSize: serialized.length,
_message: 'Payload truncated due to size limit',
} as unknown as T;
}
} catch (error) {
return {
_error: 'Failed to serialize payload',
_message: String(error),
} as unknown as T;
}
return value;
}
private isContentKey(key: string): boolean {
return key.includes('content') || key.includes('raw') || key.includes('body') || key.includes('html');
}
private isSensitivePathKey(key: string): boolean {
return key.includes('path') || key.includes('filepath') || key.includes('folder');
}
private maskPath(value: string): string {
if (!value) {
return value;
}
const segments = value.split(/[\\/]+/).filter(Boolean);
if (segments.length === 0) {
return value;
}
const tail = segments.slice(-2).join('/');
return segments.length > 2 ? `…/${tail}` : tail;
}
private cloneData(data: Record<string, unknown>): Dictionary {
return JSON.parse(JSON.stringify(data ?? {})) as Dictionary;
}
private ensureSessionId(): string {
if (!this.isBrowser) {
return this.generateUUID();
}
try {
let id = sessionStorage.getItem(SESSION_STORAGE_KEY);
if (!id) {
id = this.generateUUID();
sessionStorage.setItem(SESSION_STORAGE_KEY, id);
}
return id;
} catch {
return this.generateUUID();
}
}
private shouldSample(correlationId: string): boolean {
const rate = this.config.sampleRateSearchDiag ?? 1;
if (rate >= 1) {
return true;
}
if (this.sampleDecisions.has(correlationId)) {
return this.sampleDecisions.get(correlationId) as boolean;
}
const decision = Math.random() < rate;
this.sampleDecisions.set(correlationId, decision);
if (this.sampleDecisions.size > SAMPLE_DECISIONS_LIMIT) {
const [firstKey] = this.sampleDecisions.keys();
if (firstKey) {
this.sampleDecisions.delete(firstKey);
}
}
return decision;
}
private extractQueryRaw(
correlationId: string,
stage: SearchDiagEvent,
data: Record<string, unknown>
): string {
let query = '';
const maybeQuery = data['queryRaw'];
if (typeof maybeQuery === 'string') {
query = maybeQuery;
} else {
query = this.correlationQueries.get(correlationId) ?? '';
}
if (stage === 'SEARCH_DIAG_START' && query) {
this.correlationQueries.set(correlationId, query);
}
if (!query && typeof data['query'] === 'string') {
query = data['query'] as string;
}
return query;
}
private persistQueue(): void {
if (!this.isBrowser) {
return;
}
try {
localStorage.setItem(LOCAL_STORAGE_QUEUE_KEY, JSON.stringify(this.queue));
} catch {
// ignore
}
}
private loadQueueFromStorage(): void {
if (!this.isBrowser) {
return;
}
try {
const stored = localStorage.getItem(LOCAL_STORAGE_QUEUE_KEY);
if (!stored) {
return;
}
const parsed = JSON.parse(stored);
if (Array.isArray(parsed)) {
this.queue = parsed;
if (this.queue.length > 0) {
this.scheduleFlush();
}
}
} catch {
// ignore parse errors
}
}
private generateUUID(): string {
if (this.isBrowser && typeof window !== 'undefined' && 'crypto' in window && typeof crypto.randomUUID === 'function') {
return crypto.randomUUID();
}
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (char) => {
const random = (Math.random() * 16) | 0;
const value = char === 'x' ? random : (random & 0x3) | 0x8;
return value.toString(16);
});
}
private delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}

View File

@ -0,0 +1,37 @@
export type SearchDiagEvent =
| 'SEARCH_DIAG_START'
| 'SEARCH_DIAG_PARSE'
| 'SEARCH_DIAG_PLAN'
| 'SEARCH_DIAG_EXEC_PROVIDER'
| 'SEARCH_DIAG_RESULT_MAP'
| 'SEARCH_DIAG_SUMMARY'
| 'SEARCH_DIAG_ERROR';
export interface SearchDiagRecord {
ts: string;
app: 'ObsiViewer';
sessionId: string;
level: 'info' | 'warn' | 'error';
correlationId: string;
queryRaw: string;
context: {
route?: string;
vault?: string;
theme?: 'dark' | 'light';
version?: string;
};
userAgent?: string;
stage: SearchDiagEvent;
data: Record<string, unknown>;
}
export interface SearchDiagContextMeta {
userAgent?: string;
sampleRate?: number;
}
export interface SearchDiagOptions {
correlationId?: string;
level?: 'info' | 'warn' | 'error';
queryRaw: string;
}

View File

@ -9,5 +9,6 @@ export const environment = {
maxRetries: 5, maxRetries: 5,
circuitBreakerThreshold: 5, circuitBreakerThreshold: 5,
circuitBreakerResetMs: 30000, circuitBreakerResetMs: 30000,
sampleRateSearchDiag: 1.0,
}, },
}; };

View File

@ -0,0 +1,121 @@
import { TestBed } from '@angular/core/testing';
import { provideZonelessChangeDetection } from '@angular/core';
import { SearchOrchestratorService } from '../search-orchestrator.service';
import { SearchIndexService, SearchIndexDiagnosticsSnapshot, SearchIndexNormalizationRules } from '../search-index.service';
import { ClientLoggingService } from '../../../services/client-logging.service';
import { SearchLogService } from '../../../app/core/logging/log.service';
import { ParseDiagnostics } from '../search-parser.types';
import { SearchExecutionOptions } from '../search-orchestrator.service';
describe('SearchOrchestratorService diagnostics planning', () => {
let service: SearchOrchestratorService;
let normalizationRules: SearchIndexNormalizationRules;
beforeEach(() => {
normalizationRules = {
tags: {
ensureHashPrefix: true,
preserveCase: true,
deduplicate: true,
stripHashOnMatch: true
},
paths: {
separator: '/',
caseSensitive: false
},
files: {
caseSensitive: false
}
};
const searchIndexStub = {
getAllContexts: jasmine.createSpy('getAllContexts').and.returnValue([]),
getIndexDiagnostics: jasmine.createSpy('getIndexDiagnostics').and.returnValue({})
} as unknown as SearchIndexService;
const clientLoggerStub = jasmine.createSpyObj<ClientLoggingService>('ClientLoggingService', ['info', 'debug', 'error', 'warn']);
const diagLoggerStub = {
logSearch: jasmine.createSpy('logSearch').and.returnValue('correlation-1')
} as unknown as SearchLogService;
TestBed.configureTestingModule({
providers: [
provideZonelessChangeDetection(),
SearchOrchestratorService,
{ provide: SearchIndexService, useValue: searchIndexStub },
{ provide: ClientLoggingService, useValue: clientLoggerStub },
{ provide: SearchLogService, useValue: diagLoggerStub }
]
});
service = TestBed.inject(SearchOrchestratorService);
});
it('builds plan with tag stages and no warnings when tag is indexed', () => {
const diagnostics: ParseDiagnostics = {
tokens: ['tag:#home'],
warnings: [],
filters: {
tag: ['#home'],
path: [],
file: [],
negative: [],
negativeDetails: [],
regex: false,
caseSensitive: false,
wholeWord: false
}
};
const options: SearchExecutionOptions = { caseSensitive: false, regexMode: false };
const indexSnapshot: SearchIndexDiagnosticsSnapshot = {
tagsIndexed: true,
pathIndexed: true,
filesIndexed: true,
totalContexts: 25,
tagCardinality: { home: 3 },
normalizeRules: normalizationRules
};
const plan = (service as any).buildExecutionPlan(diagnostics, options, indexSnapshot);
expect(plan.stages.length).toBeGreaterThan(0);
const tagStage = plan.stages.find((s: any) => s.name === 'candidateSetFromTag');
expect(tagStage).toBeDefined();
expect(tagStage.constraint).toBe('tag=home');
expect(tagStage.estimate).toBe(3);
expect(plan.warnings).toEqual([]);
});
it('flags missing tag index and negative filters in plan warnings', () => {
const diagnostics: ParseDiagnostics = {
tokens: ['tag:#home', '-path:"Archive"'],
warnings: [],
filters: {
tag: ['#home'],
path: ['Archive'],
file: [],
negative: ['Archive'],
negativeDetails: [{ type: 'path', value: 'Archive', wildcard: false }],
regex: false,
caseSensitive: false,
wholeWord: false
}
};
const options: SearchExecutionOptions = { caseSensitive: false, regexMode: false };
const indexSnapshot: SearchIndexDiagnosticsSnapshot = {
tagsIndexed: false,
pathIndexed: true,
filesIndexed: true,
totalContexts: 10,
tagCardinality: {},
normalizeRules: normalizationRules
};
const plan = (service as any).buildExecutionPlan(diagnostics, options, indexSnapshot);
expect(plan.warnings).toContain('TagIndexUnavailable');
const negativeStage = plan.stages.find((s: any) => s.name === 'applyNegative');
expect(negativeStage).toBeDefined();
expect(negativeStage.constraint).toBe('path:Archive');
});
});

View File

@ -0,0 +1,40 @@
import { parseSearchQuery } from '../search-parser';
import { SearchOptions } from '../search-parser.types';
describe('parseSearchQuery diagnostics', () => {
const baseOptions: SearchOptions = {
caseSensitive: false,
regexMode: false
};
it('captures filters for tag and file operators', () => {
const query = 'tag:#home file:"Project Plan.md"';
const parsed = parseSearchQuery(query, baseOptions);
expect(parsed.isEmpty).toBe(false);
expect(parsed.diagnostics).toBeDefined();
expect(parsed.diagnostics?.filters.tag).toEqual(['#home']);
expect(parsed.diagnostics?.filters.file).toEqual(['Project Plan.md']);
const negative = parsed.diagnostics?.filters.negative ?? [];
expect(negative.length).toBe(0);
});
it('tracks negative filters with metadata', () => {
const query = 'tag:#home -path:"Archive" -content:"secret"';
const parsed = parseSearchQuery(query, baseOptions);
expect(parsed.diagnostics?.filters.negative).toEqual(['Archive', 'secret']);
const negativeDetails = parsed.diagnostics?.filters.negativeDetails ?? [];
expect(negativeDetails.length).toBe(2);
expect(negativeDetails[0]).toEqual(jasmine.objectContaining({ type: 'path', value: 'Archive' }));
expect(negativeDetails[1]).toEqual(jasmine.objectContaining({ type: 'content', value: 'secret' }));
});
it('records warnings for unknown prefixes', () => {
const query = 'unknown:term -unknown:value';
const parsed = parseSearchQuery(query, baseOptions);
expect(parsed.diagnostics?.warnings).toContain('UnknownOperator:unknown');
expect(parsed.diagnostics?.filters.negative).toContain('unknown:value');
});
});

View File

@ -3,6 +3,31 @@ import { SearchContext, SectionContent, TaskInfo } from './search-parser.types';
import { VaultService } from '../../services/vault.service'; import { VaultService } from '../../services/vault.service';
import { Note } from '../../types'; import { Note } from '../../types';
export interface SearchIndexNormalizationRules {
tags: {
ensureHashPrefix: boolean;
preserveCase: boolean;
deduplicate: boolean;
stripHashOnMatch: boolean;
};
paths: {
separator: string;
caseSensitive: boolean;
};
files: {
caseSensitive: boolean;
};
}
export interface SearchIndexDiagnosticsSnapshot {
tagsIndexed: boolean;
pathIndexed: boolean;
filesIndexed: boolean;
totalContexts: number;
tagCardinality: Record<string, number>;
normalizeRules: SearchIndexNormalizationRules;
}
/** /**
* Comprehensive search index for the vault * Comprehensive search index for the vault
* Indexes all content for fast searching with full Obsidian operator support * Indexes all content for fast searching with full Obsidian operator support
@ -17,6 +42,7 @@ export class SearchIndexService {
private pathsIndex = signal<string[]>([]); private pathsIndex = signal<string[]>([]);
private filesIndex = signal<string[]>([]); private filesIndex = signal<string[]>([]);
private tagsIndex = signal<string[]>([]); private tagsIndex = signal<string[]>([]);
private tagCardinality = signal<Record<string, number>>({});
private propertiesIndex = signal<Map<string, Set<string>>>(new Map()); private propertiesIndex = signal<Map<string, Set<string>>>(new Map());
private headingsIndex = signal<Map<string, string[]>>(new Map()); private headingsIndex = signal<Map<string, string[]>>(new Map());
@ -56,6 +82,7 @@ export class SearchIndexService {
const pathsSet = new Set<string>(); const pathsSet = new Set<string>();
const filesSet = new Set<string>(); const filesSet = new Set<string>();
const tagsSet = new Set<string>(); const tagsSet = new Set<string>();
const tagCounts = new Map<string, number>();
const propertiesMap = new Map<string, Set<string>>(); const propertiesMap = new Map<string, Set<string>>();
const headingsMap = new Map<string, string[]>(); const headingsMap = new Map<string, string[]>();
@ -78,7 +105,12 @@ export class SearchIndexService {
filesSet.add(context.fileNameWithExt); filesSet.add(context.fileNameWithExt);
// Index tags // Index tags
context.tags.forEach(tag => tagsSet.add(tag)); context.tags.forEach(tag => {
tagsSet.add(tag);
const normalized = tag.startsWith('#') ? tag.substring(1) : tag;
const key = normalized.toLowerCase();
tagCounts.set(key, (tagCounts.get(key) ?? 0) + 1);
});
// Index properties // Index properties
Object.keys(context.properties).forEach(key => { Object.keys(context.properties).forEach(key => {
@ -104,6 +136,11 @@ export class SearchIndexService {
this.pathsIndex.set(Array.from(pathsSet).sort()); this.pathsIndex.set(Array.from(pathsSet).sort());
this.filesIndex.set(Array.from(filesSet).sort()); this.filesIndex.set(Array.from(filesSet).sort());
this.tagsIndex.set(Array.from(tagsSet).sort()); this.tagsIndex.set(Array.from(tagsSet).sort());
const tagCardinalityObj: Record<string, number> = {};
tagCounts.forEach((count, key) => {
tagCardinalityObj[key] = count;
});
this.tagCardinality.set(tagCardinalityObj);
this.propertiesIndex.set(propertiesMap); this.propertiesIndex.set(propertiesMap);
this.headingsIndex.set(headingsMap); this.headingsIndex.set(headingsMap);
} }
@ -286,6 +323,48 @@ export class SearchIndexService {
return Array.from(new Set(all)); return Array.from(new Set(all));
} }
getIndexDiagnostics(filters?: { tags?: string[] }): SearchIndexDiagnosticsSnapshot {
const tagSnapshot = this.tagCardinality();
const selectedTags = filters?.tags ?? Object.keys(tagSnapshot);
const tagCardinality: Record<string, number> = {};
selectedTags.forEach(tag => {
const normalized = tag.startsWith('#') ? tag.substring(1) : tag;
const key = normalized.toLowerCase();
const value = tagSnapshot[key];
if (value !== undefined) {
tagCardinality[normalized] = value;
}
});
return {
tagsIndexed: this.tagsIndex().length > 0,
pathIndexed: this.pathsIndex().length > 0,
filesIndexed: this.filesIndex().length > 0,
totalContexts: this.indexData().size,
tagCardinality,
normalizeRules: this.getNormalizationRules()
};
}
getNormalizationRules(): SearchIndexNormalizationRules {
return {
tags: {
ensureHashPrefix: true,
preserveCase: true,
deduplicate: true,
stripHashOnMatch: true
},
paths: {
separator: '/',
caseSensitive: false
},
files: {
caseSensitive: false
}
};
}
/** /**
* Extract tags from YAML frontmatter in raw markdown content. * Extract tags from YAML frontmatter in raw markdown content.
* Supports: * Supports:

View File

@ -3,6 +3,8 @@ import { provideZonelessChangeDetection } from '@angular/core';
import { SearchOrchestratorService } from './search-orchestrator.service'; import { SearchOrchestratorService } from './search-orchestrator.service';
import { SearchIndexService } from './search-index.service'; import { SearchIndexService } from './search-index.service';
import { SearchContext } from './search-parser.types'; import { SearchContext } from './search-parser.types';
import { ClientLoggingService } from '../../services/client-logging.service';
import { SearchLogService } from '../../app/core/logging/log.service';
describe('SearchOrchestratorService', () => { describe('SearchOrchestratorService', () => {
let service: SearchOrchestratorService; let service: SearchOrchestratorService;
@ -38,14 +40,33 @@ describe('SearchOrchestratorService', () => {
contexts = [mockContext]; contexts = [mockContext];
const indexServiceStub = { const indexServiceStub = {
getAllContexts: () => contexts getAllContexts: () => contexts,
} as SearchIndexService; getIndexDiagnostics: jasmine.createSpy('getIndexDiagnostics').and.returnValue({
tagsIndexed: true,
pathIndexed: true,
filesIndexed: true,
totalContexts: 1,
tagCardinality: { test: 1, example: 1 },
normalizeRules: {
tags: { ensureHashPrefix: true, preserveCase: true, deduplicate: true, stripHashOnMatch: true },
paths: { separator: '/', caseSensitive: false },
files: { caseSensitive: false }
}
})
} as unknown as SearchIndexService;
const clientLoggerStub = jasmine.createSpyObj<ClientLoggingService>('ClientLoggingService', ['info', 'debug', 'error', 'warn']);
const diagLoggerStub = {
logSearch: jasmine.createSpy('logSearch').and.returnValue('test-correlation-id')
} as unknown as SearchLogService;
TestBed.configureTestingModule({ TestBed.configureTestingModule({
providers: [ providers: [
provideZonelessChangeDetection(), provideZonelessChangeDetection(),
SearchOrchestratorService, SearchOrchestratorService,
{ provide: SearchIndexService, useValue: indexServiceStub } { provide: SearchIndexService, useValue: indexServiceStub },
{ provide: ClientLoggingService, useValue: clientLoggerStub },
{ provide: SearchLogService, useValue: diagLoggerStub }
] ]
}); });
service = TestBed.inject(SearchOrchestratorService); service = TestBed.inject(SearchOrchestratorService);

View File

@ -1,8 +1,11 @@
import { Injectable, inject } from '@angular/core'; import { Injectable, inject } from '@angular/core';
import { parseSearchQuery, queryToPredicate } from './search-parser'; import { parseSearchQuery, queryToPredicate } from './search-parser';
import { SearchOptions, SearchContext } from './search-parser.types'; import { SearchOptions, SearchContext } from './search-parser.types';
import { SearchIndexService } from './search-index.service'; import { SearchIndexService, SearchIndexDiagnosticsSnapshot } from './search-index.service';
import { ClientLoggingService } from '../../services/client-logging.service'; import { ClientLoggingService } from '../../services/client-logging.service';
import { SearchLogService } from '../../app/core/logging/log.service';
import { SearchDiagEvent } from '../../app/core/logging/search-log.model';
import { ParseDiagnostics } from './search-parser.types';
/** /**
* Match range for highlighting * Match range for highlighting
@ -43,6 +46,16 @@ export interface SearchExecutionOptions extends SearchOptions {
contextLines?: number; contextLines?: number;
/** Maximum number of results to return (default: unlimited) */ /** Maximum number of results to return (default: unlimited) */
maxResults?: number; maxResults?: number;
/** Diagnostic trigger source */
trigger?: 'submit' | 'debounce' | 'route' | string;
/** Diagnostic source identifier */
source?: string;
/** Paging page number for diagnostics */
page?: number;
/** Paging page size for diagnostics */
pageSize?: number;
/** Existing correlation id to reuse */
correlationId?: string;
} }
/** /**
@ -55,6 +68,7 @@ export interface SearchExecutionOptions extends SearchOptions {
export class SearchOrchestratorService { export class SearchOrchestratorService {
private searchIndex = inject(SearchIndexService); private searchIndex = inject(SearchIndexService);
private logger = inject(ClientLoggingService); private logger = inject(ClientLoggingService);
private diagnosticsLogger = inject(SearchLogService);
/** /**
* Execute a search query and return matching notes with highlights * Execute a search query and return matching notes with highlights
@ -64,87 +78,420 @@ export class SearchOrchestratorService {
return []; return [];
} }
const pipelineStart = this.now();
let stageAtFailure: SearchDiagEvent = 'SEARCH_DIAG_START';
const timings = {
parse: 0,
plan: 0,
exec: 0,
map: 0,
render: 0
};
let results: SearchResult[] = [];
let allContextsCount = 0;
let combinedCount = 0;
let planInfo: ReturnType<typeof this.buildExecutionPlan> | undefined;
let indexDiagnostics: SearchIndexDiagnosticsSnapshot | undefined;
const correlationId = this.diagnosticsLogger.logSearch('SEARCH_DIAG_START', {
source: options?.source ?? 'ui',
trigger: options?.trigger ?? 'submit',
queryRaw: query,
options: {
caseSensitive: options?.caseSensitive ?? false,
regex: options?.regexMode ?? false,
wholeWord: (options as any)?.wholeWord ?? false
},
paging: {
page: options?.page ?? 1,
pageSize: options?.pageSize ?? options?.maxResults ?? 0
}
}, 'info');
const contextLines = options?.contextLines ?? 2; const contextLines = options?.contextLines ?? 2;
const maxResults = options?.maxResults; const maxResults = options?.maxResults;
// Parse the query into an AST try {
this.logger.info('SearchOrchestrator', 'Parsing query', { query, options }); stageAtFailure = 'SEARCH_DIAG_PARSE';
const parsed = parseSearchQuery(query, options); this.logger.info('SearchOrchestrator', 'Parsing query', { query, options });
if (parsed.isEmpty) { const parseStart = this.now();
this.logger.debug('SearchOrchestrator', 'Parsed query is empty'); const parsed = parseSearchQuery(query, options);
timings.parse = this.now() - parseStart;
if (parsed.diagnostics) {
this.diagnosticsLogger.logSearch('SEARCH_DIAG_PARSE', {
queryRaw: query,
tokens: parsed.diagnostics.tokens,
filters: parsed.diagnostics.filters,
parseWarnings: parsed.diagnostics.warnings
}, 'info', correlationId);
}
if (parsed.isEmpty) {
this.logger.debug('SearchOrchestrator', 'Parsed query is empty');
return [];
}
stageAtFailure = 'SEARCH_DIAG_PLAN';
this.logger.debug('SearchOrchestrator', 'Building predicate');
const predicate = queryToPredicate(parsed, options);
this.logger.debug('SearchOrchestrator', 'Extracting search terms');
const searchTerms = this.extractSearchTerms(parsed.ast);
const planStart = this.now();
indexDiagnostics = this.searchIndex.getIndexDiagnostics({ tags: parsed.diagnostics?.filters.tag });
planInfo = this.buildExecutionPlan(parsed.diagnostics, options, indexDiagnostics);
timings.plan = this.now() - planStart;
this.diagnosticsLogger.logSearch('SEARCH_DIAG_PLAN', {
queryRaw: query,
stages: planInfo.stages,
indexStats: planInfo.indexStats,
warnings: planInfo.warnings,
normalizeRules: planInfo.normalizeRules
}, planInfo.warnings.length > 0 ? 'warn' : 'info', correlationId);
stageAtFailure = 'SEARCH_DIAG_EXEC_PROVIDER';
results = [];
const allContexts = this.searchIndex.getAllContexts();
allContextsCount = allContexts.length;
this.logger.info('SearchOrchestrator', 'Evaluating contexts', {
contextCount: allContextsCount
});
const execStart = this.now();
for (const context of allContexts) {
this.logger.debug('SearchOrchestrator', 'Evaluating context', {
noteId: context.noteId,
filePath: context.filePath
});
if (!predicate(context)) {
this.logger.debug('SearchOrchestrator', 'Context filtered out', {
noteId: context.noteId
});
continue;
}
this.logger.debug('SearchOrchestrator', 'Context matched predicate, finding highlights', {
noteId: context.noteId
});
const { matches, allRanges } = this.findMatchesWithRanges(
context,
searchTerms,
options,
contextLines
);
const score = this.calculateScore(context, query, matches);
this.logger.debug('SearchOrchestrator', 'Context scored', {
noteId: context.noteId,
matchCount: matches.length,
score
});
results.push({
noteId: context.noteId,
matches,
score,
allRanges
});
}
timings.exec = this.now() - execStart;
const providerOutputCount = results.length;
combinedCount = providerOutputCount;
this.diagnosticsLogger.logSearch('SEARCH_DIAG_EXEC_PROVIDER', {
queryRaw: query,
provider: 'localIndex',
inputCount: allContextsCount,
filterApplied: this.buildFilterApplied(parsed.diagnostics),
outputCount: providerOutputCount,
timingMs: timings.exec,
cache: { hit: false }
}, 'info', correlationId);
stageAtFailure = 'SEARCH_DIAG_RESULT_MAP';
const mapStart = this.now();
this.logger.debug('SearchOrchestrator', 'Sorting results', {
resultCount: results.length
});
if (maxResults && maxResults > 0) {
this.logger.debug('SearchOrchestrator', 'Applying maxResults slice', { maxResults });
results.splice(maxResults);
}
const providerResults = [{ provider: 'localIndex', count: providerOutputCount }];
const afterUIFiltersCount = combinedCount;
const reasonsEmpty = indexDiagnostics && planInfo
? this.collectEmptyReasons(results.length, planInfo, indexDiagnostics)
: [];
timings.map = this.now() - mapStart;
this.diagnosticsLogger.logSearch('SEARCH_DIAG_RESULT_MAP', {
queryRaw: query,
providerResults,
combinedCount,
afterUIFiltersCount,
reasonsEmpty
}, reasonsEmpty.length > 0 ? 'warn' : 'info', correlationId);
this.logger.info('SearchOrchestrator', 'Returning results', {
resultCount: results.length
});
stageAtFailure = 'SEARCH_DIAG_SUMMARY';
const totalMs = this.now() - pipelineStart;
timings.render = Math.max(0, totalMs - (timings.parse + timings.plan + timings.exec + timings.map));
const summaryLevel: 'info' | 'warn' = results.length === 0 || (planInfo?.warnings.length ?? 0) > 0 ? 'warn' : 'info';
this.diagnosticsLogger.logSearch('SEARCH_DIAG_SUMMARY', {
queryRaw: query,
totalMs,
stagesMs: timings,
counts: {
beforeFilters: allContextsCount,
afterFilters: combinedCount,
displayed: results.length
},
index: indexDiagnostics ? {
tagsIndexed: indexDiagnostics.tagsIndexed,
tagCardinality: indexDiagnostics.tagCardinality
} : undefined,
userVisible: {
emptyStateShown: results.length === 0,
helperSuggestions: planInfo ? this.buildHelperSuggestions(planInfo, results.length) : []
}
}, summaryLevel, correlationId);
return results;
} catch (error) {
const errorKind = this.inferErrorKind(stageAtFailure, error);
this.diagnosticsLogger.logSearch('SEARCH_DIAG_ERROR', {
queryRaw: query,
kind: errorKind,
message: error instanceof Error ? error.message : String(error),
code: (error as any)?.code,
safeStack: this.extractSafeStack(error),
stageAtFailure,
partialCounts: {
beforeFilters: allContextsCount,
afterFilters: combinedCount
}
}, 'error', correlationId);
this.logger.error('SearchOrchestrator', 'Search pipeline failed', {
stage: stageAtFailure,
error
});
throw error;
}
}
private inferErrorKind(stage: SearchDiagEvent, error: unknown): 'ParseError' | 'IndexError' | 'ProviderError' | 'Timeout' | 'Unknown' {
if ((error as any)?.name === 'TimeoutError') {
return 'Timeout';
}
switch (stage) {
case 'SEARCH_DIAG_PARSE':
return 'ParseError';
case 'SEARCH_DIAG_PLAN':
return 'IndexError';
case 'SEARCH_DIAG_EXEC_PROVIDER':
return 'ProviderError';
default:
return 'Unknown';
}
}
private extractSafeStack(error: unknown): string | undefined {
if (error instanceof Error && typeof error.stack === 'string') {
return error.stack.split('\n').slice(0, 5).join('\n');
}
return undefined;
}
private now(): number {
if (typeof performance !== 'undefined' && typeof performance.now === 'function') {
return performance.now();
}
return Date.now();
}
private buildExecutionPlan(
diagnostics: ParseDiagnostics | undefined,
options: SearchExecutionOptions | undefined,
indexDiagnostics: SearchIndexDiagnosticsSnapshot
): {
stages: Array<{ name: string; estimate: number; constraint: string }>;
warnings: string[];
indexStats: {
tagsIndexed: boolean;
tagCardinality: Record<string, number>;
pathIndexed: boolean;
filesIndexed: boolean;
};
normalizeRules: SearchIndexDiagnosticsSnapshot['normalizeRules'];
} {
const stages: Array<{ name: string; estimate: number; constraint: string }> = [];
const warnings: string[] = [];
const filters = diagnostics?.filters;
if (filters?.tag?.length) {
filters.tag.forEach(tag => {
const normalized = tag.startsWith('#') ? tag.substring(1) : tag;
const key = normalized.toLowerCase();
const count = indexDiagnostics.tagCardinality[key];
stages.push({
name: 'candidateSetFromTag',
estimate: count ?? 0,
constraint: `tag=${normalized}`
});
if (!indexDiagnostics.tagsIndexed) {
warnings.push('TagIndexUnavailable');
} else if (count === undefined) {
warnings.push(`TagNotInIndex:${normalized}`);
} else if (count === 0) {
warnings.push(`TagEmpty:${normalized}`);
}
});
}
if (filters?.path?.length) {
filters.path.forEach(path => {
stages.push({
name: 'intersectPath',
estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 4)),
constraint: `path=${path}`
});
});
}
if (filters?.file?.length) {
filters.file.forEach(file => {
stages.push({
name: 'intersectFile',
estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 6)),
constraint: `file=${file}`
});
});
}
if (filters?.negativeDetails?.length) {
const constraint = filters.negativeDetails.map(n => `${n.type}:${n.value}`).join(', ');
stages.push({
name: 'applyNegative',
estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 8)),
constraint
});
}
if (options?.regexMode || filters?.regex) {
stages.push({
name: 'postFilterRegex',
estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 3)),
constraint: `regex=${options?.regexMode ?? filters?.regex}`
});
}
if (filters?.caseSensitive) {
stages.push({
name: 'postFilterCaseSensitive',
estimate: Math.max(0, Math.floor(indexDiagnostics.totalContexts / 2)),
constraint: 'caseSensitive=true'
});
}
if (!stages.length) {
stages.push({
name: 'fullScan',
estimate: indexDiagnostics.totalContexts,
constraint: 'none'
});
}
if (!indexDiagnostics.tagsIndexed && filters?.tag?.length) {
warnings.push('E_TAG_INDEX_DISABLED');
}
return {
stages,
warnings,
indexStats: {
tagsIndexed: indexDiagnostics.tagsIndexed,
tagCardinality: indexDiagnostics.tagCardinality,
pathIndexed: indexDiagnostics.pathIndexed,
filesIndexed: indexDiagnostics.filesIndexed
},
normalizeRules: indexDiagnostics.normalizeRules
};
}
private buildFilterApplied(diagnostics: ParseDiagnostics | undefined): Record<string, unknown> {
if (!diagnostics) {
return {};
}
return {
tag: diagnostics.filters.tag,
path: diagnostics.filters.path,
file: diagnostics.filters.file,
negative: diagnostics.filters.negative
};
}
private collectEmptyReasons(
resultCount: number,
planInfo: ReturnType<typeof this.buildExecutionPlan>,
indexDiagnostics: SearchIndexDiagnosticsSnapshot
): string[] {
if (resultCount > 0) {
return []; return [];
} }
// Convert to predicate function const reasons = new Set<string>();
this.logger.debug('SearchOrchestrator', 'Building predicate'); planInfo.warnings.forEach(warning => {
const predicate = queryToPredicate(parsed, options); if (warning.startsWith('TagNotInIndex')) {
reasons.add('tagNotInIndex');
// Extract search terms for highlighting } else if (warning.startsWith('TagEmpty')) {
this.logger.debug('SearchOrchestrator', 'Extracting search terms'); reasons.add('noMatchesForTag');
const searchTerms = this.extractSearchTerms(parsed.ast); } else if (warning === 'TagIndexUnavailable' || warning === 'E_TAG_INDEX_DISABLED') {
reasons.add('tagIndexUnavailable');
// Evaluate against all indexed contexts
const results: SearchResult[] = [];
const allContexts = this.searchIndex.getAllContexts();
this.logger.info('SearchOrchestrator', 'Evaluating contexts', {
contextCount: allContexts.length
});
for (const context of allContexts) {
this.logger.debug('SearchOrchestrator', 'Evaluating context', {
noteId: context.noteId,
filePath: context.filePath
});
// Apply the predicate filter (if it doesn't match, skip this context)
if (!predicate(context)) {
this.logger.debug('SearchOrchestrator', 'Context filtered out', {
noteId: context.noteId
});
continue;
} }
// Find matches and ranges for highlighting (may be empty for file/path/property queries)
this.logger.debug('SearchOrchestrator', 'Context matched predicate, finding highlights', {
noteId: context.noteId
});
const { matches, allRanges } = this.findMatchesWithRanges(
context,
searchTerms,
options,
contextLines
);
const score = this.calculateScore(context, query, matches);
this.logger.debug('SearchOrchestrator', 'Context scored', {
noteId: context.noteId,
matchCount: matches.length,
score
});
results.push({
noteId: context.noteId,
matches,
score,
allRanges
});
}
// Sort by score (descending)
this.logger.debug('SearchOrchestrator', 'Sorting results', {
resultCount: results.length
}); });
// Apply max results limit if specified if (!indexDiagnostics.filesIndexed) {
if (maxResults && maxResults > 0) { reasons.add('fileIndexUnavailable');
this.logger.debug('SearchOrchestrator', 'Applying maxResults slice', { maxResults }); }
return results.slice(0, maxResults); if (!indexDiagnostics.pathIndexed) {
reasons.add('pathIndexUnavailable');
} }
this.logger.info('SearchOrchestrator', 'Returning results', { if (!reasons.size) {
resultCount: results.length reasons.add('intersectionEmpty');
}); }
return results;
return Array.from(reasons);
}
private buildHelperSuggestions(planInfo: ReturnType<typeof this.buildExecutionPlan>, resultCount: number): string[] {
const suggestions: string[] = [];
if (resultCount === 0) {
if (planInfo.warnings.some(w => w.startsWith('TagNotInIndex'))) {
suggestions.push('Vérifier que le tag existe dans le vault');
}
if (planInfo.warnings.includes('TagIndexUnavailable') || planInfo.warnings.includes('E_TAG_INDEX_DISABLED')) {
suggestions.push('Reconstruire lindex des tags');
}
if (!suggestions.length) {
suggestions.push('Assouplir les filtres ou retirer les exclusions');
}
}
return suggestions.slice(0, 3);
} }
/** /**
* Extract search terms from AST for highlighting * Extract search terms from AST for highlighting

View File

@ -13,7 +13,9 @@ import {
SearchContext, SearchContext,
SearchOptions, SearchOptions,
SectionContent, SectionContent,
TaskInfo TaskInfo,
ParseDiagnostics,
SearchFilterDiagnostics
} from './search-parser.types'; } from './search-parser.types';
// Re-export types for convenience // Re-export types for convenience
@ -26,16 +28,27 @@ export function parseSearchQuery(query: string, options?: SearchOptions): Parsed
if (!query || !query.trim()) { if (!query || !query.trim()) {
return { return {
ast: { type: 'group', operator: 'AND', terms: [] }, ast: { type: 'group', operator: 'AND', terms: [] },
isEmpty: true isEmpty: true,
diagnostics: {
tokens: [],
filters: createEmptyDiagnosticsFilters(options),
warnings: []
}
}; };
} }
const tokens = tokenize(query); const tokens = tokenize(query);
const ast = parseTokens(tokens, options); const diagnostics: ParseDiagnostics = {
tokens: [...tokens],
filters: createEmptyDiagnosticsFilters(options),
warnings: []
};
const ast = parseTokens(tokens, options, diagnostics);
return { return {
ast, ast,
isEmpty: false isEmpty: false,
diagnostics
}; };
} }
@ -63,21 +76,39 @@ function tokenize(query: string): string[] {
// Handle quoted strings // Handle quoted strings
if (char === '"') { if (char === '"') {
if (current) { // If we are inside a prefix token like file: or path:, attach the quoted part to current
tokens.push(current); if (current.includes(':') && !current.includes(' ')) {
current = ''; let quoted = '"';
}
let quoted = '';
i++;
while (i < query.length && query[i] !== '"') {
quoted += query[i];
i++; i++;
while (i < query.length && query[i] !== '"') {
quoted += query[i];
i++;
}
if (i < query.length && query[i] === '"') {
quoted += '"';
i++;
}
current += quoted;
continue;
} else {
if (current) {
tokens.push(current);
current = '';
}
let quoted = '';
i++;
while (i < query.length && query[i] !== '"') {
quoted += query[i];
i++;
}
if (i < query.length && query[i] === '"') {
i++;
}
if (quoted) {
tokens.push(`"${quoted}"`);
}
continue;
} }
if (quoted) {
tokens.push(`"${quoted}"`);
}
i++;
continue;
} }
// Handle regex patterns /.../ // Handle regex patterns /.../
@ -137,7 +168,7 @@ function tokenize(query: string): string[] {
/** /**
* Parse tokens into AST * Parse tokens into AST
*/ */
function parseTokens(tokens: string[], options?: SearchOptions): SearchNode { function parseTokens(tokens: string[], options: SearchOptions | undefined, diagnostics: ParseDiagnostics): SearchNode {
const terms: SearchNode[] = []; const terms: SearchNode[] = [];
let i = 0; let i = 0;
@ -158,14 +189,14 @@ function parseTokens(tokens: string[], options?: SearchOptions): SearchNode {
// Handle parentheses // Handle parentheses
if (token === '(') { if (token === '(') {
const { node, endIndex } = parseGroup(tokens, i + 1, options); const { node, endIndex } = parseGroup(tokens, i + 1, options, diagnostics);
terms.push(node); terms.push(node);
i = endIndex + 1; i = endIndex + 1;
continue; continue;
} }
// Parse term // Parse term
const term = parseTerm(token, options); const term = parseTerm(token, options, diagnostics);
if (term) { if (term) {
terms.push(term); terms.push(term);
} }
@ -186,7 +217,12 @@ function parseTokens(tokens: string[], options?: SearchOptions): SearchNode {
/** /**
* Parse a group enclosed in parentheses * Parse a group enclosed in parentheses
*/ */
function parseGroup(tokens: string[], startIndex: number, options?: SearchOptions): { node: SearchNode; endIndex: number } { function parseGroup(
tokens: string[],
startIndex: number,
options: SearchOptions | undefined,
diagnostics: ParseDiagnostics
): { node: SearchNode; endIndex: number } {
const terms: SearchNode[] = []; const terms: SearchNode[] = [];
let i = startIndex; let i = startIndex;
let depth = 1; let depth = 1;
@ -204,7 +240,7 @@ function parseGroup(tokens: string[], startIndex: number, options?: SearchOption
} }
if (token.toUpperCase() !== 'OR' && token.toUpperCase() !== 'AND' && token !== '(' && token !== ')') { if (token.toUpperCase() !== 'OR' && token.toUpperCase() !== 'AND' && token !== '(' && token !== ')') {
const term = parseTerm(token, options); const term = parseTerm(token, options, diagnostics);
if (term) { if (term) {
terms.push(term); terms.push(term);
} }
@ -229,30 +265,34 @@ function parseGroup(tokens: string[], startIndex: number, options?: SearchOption
/** /**
* Parse a single search term * Parse a single search term
*/ */
function parseTerm(token: string, options?: SearchOptions): SearchTerm | null { function parseTerm(token: string, options: SearchOptions | undefined, diagnostics: ParseDiagnostics): SearchTerm | null {
if (!token) return null; if (!token) {
return null;
}
let negated = false; let negated = false;
let value = token; let value = token;
// Handle negation
if (value.startsWith('-')) { if (value.startsWith('-')) {
negated = true; negated = true;
value = value.substring(1); value = value.substring(1);
} }
let term: SearchTerm | null = null;
let negativeValue: string | undefined;
// Support property form: [key]:value (Obsidian compatibility) // Support property form: [key]:value (Obsidian compatibility)
if (value.startsWith('[') && value.includes(']:')) { if (value.startsWith('[') && value.includes(']:')) {
const closeBracket = value.indexOf(']'); const closeBracket = value.indexOf(']');
if (closeBracket > 0) { if (closeBracket > 0) {
const propertyKey = value.substring(1, closeBracket); const propertyKey = value.substring(1, closeBracket);
let propertyValue = value.substring(closeBracket + 2); // after ]: let propertyValue = value.substring(closeBracket + 2);
let propValueQuoted = false; let propValueQuoted = false;
if (propertyValue.startsWith('"') && propertyValue.endsWith('"')) { if (propertyValue.startsWith('"') && propertyValue.endsWith('"')) {
propValueQuoted = true; propValueQuoted = true;
propertyValue = propertyValue.substring(1, propertyValue.length - 1); propertyValue = propertyValue.substring(1, propertyValue.length - 1);
} }
return { term = {
type: 'property', type: 'property',
value: propertyValue, value: propertyValue,
propertyKey, propertyKey,
@ -260,96 +300,132 @@ function parseTerm(token: string, options?: SearchOptions): SearchTerm | null {
quoted: propValueQuoted, quoted: propValueQuoted,
wildcard: propertyValue.includes('*') wildcard: propertyValue.includes('*')
}; };
negativeValue = propertyValue || propertyKey;
} }
} }
// Handle regex patterns /.../ if (!term && value.startsWith('/') && value.endsWith('/') && value.length > 2) {
if (value.startsWith('/') && value.endsWith('/') && value.length > 2) {
const regexPattern = value.substring(1, value.length - 1); const regexPattern = value.substring(1, value.length - 1);
return { type: 'regex', value: regexPattern, negated, quoted: false, wildcard: false }; term = { type: 'regex', value: regexPattern, negated, quoted: false, wildcard: false };
diagnostics.filters.regex = true;
negativeValue = regexPattern;
} }
// Handle quoted strings
let quoted = false; let quoted = false;
if (value.startsWith('"') && value.endsWith('"')) { if (!term && value.startsWith('"') && value.endsWith('"')) {
quoted = true; quoted = true;
value = value.substring(1, value.length - 1); value = value.substring(1, value.length - 1);
} }
// Handle wildcards
const wildcard = value.includes('*'); const wildcard = value.includes('*');
// Check for prefixes if (!term) {
const colonIndex = value.indexOf(':'); const colonIndex = value.indexOf(':');
if (colonIndex > 0) { if (colonIndex > 0) {
const prefix = value.substring(0, colonIndex).toLowerCase(); const prefix = value.substring(0, colonIndex).toLowerCase();
const searchValue = value.substring(colonIndex + 1); const searchValueRaw = value.substring(colonIndex + 1);
let cleanValue = searchValueRaw;
let valueQuoted = false;
if (cleanValue.startsWith('"') && cleanValue.endsWith('"')) {
valueQuoted = true;
cleanValue = cleanValue.substring(1, cleanValue.length - 1);
}
// Remove quotes from search value if present if (prefix.startsWith('[') && prefix.endsWith(']')) {
let cleanValue = searchValue; const propertyKey = prefix.substring(1, prefix.length - 1);
let valueQuoted = false; term = {
if (cleanValue.startsWith('"') && cleanValue.endsWith('"')) { type: 'property',
valueQuoted = true; value: cleanValue,
cleanValue = cleanValue.substring(1, cleanValue.length - 1); propertyKey,
} negated,
quoted: valueQuoted,
// Fallback: handle property form [key]:value (if missed previously) wildcard: cleanValue.includes('*')
if (prefix.startsWith('[') && prefix.endsWith(']')) { };
const propertyKey = prefix.substring(1, prefix.length - 1); negativeValue = cleanValue || propertyKey;
return { } else {
type: 'property', switch (prefix) {
value: cleanValue, case 'path':
propertyKey, if (!negated) {
negated, diagnostics.filters.path.push(cleanValue);
quoted: valueQuoted, }
wildcard: cleanValue.includes('*') term = { type: 'path', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
}; negativeValue = cleanValue;
} break;
case 'file':
switch (prefix) { if (!negated) {
case 'path': diagnostics.filters.file.push(cleanValue);
return { type: 'path', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; }
case 'file': term = { type: 'file', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
return { type: 'file', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; negativeValue = cleanValue;
case 'content': break;
return { type: 'content', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; case 'content':
case 'tag': term = { type: 'content', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
return { type: 'tag', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; negativeValue = cleanValue;
case 'line': break;
return { type: 'line', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; case 'tag':
case 'block': if (!negated) {
return { type: 'block', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; diagnostics.filters.tag.push(cleanValue);
case 'section': }
return { type: 'section', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; term = { type: 'tag', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
case 'task': negativeValue = cleanValue;
return { type: 'task', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; break;
case 'task-todo': case 'line':
return { type: 'task-todo', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; term = { type: 'line', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
case 'task-done': negativeValue = cleanValue;
return { type: 'task-done', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') }; break;
case 'match-case': case 'block':
return { type: 'match-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: true }; term = { type: 'block', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
case 'ignore-case': negativeValue = cleanValue;
return { type: 'ignore-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: false }; break;
case 'section':
term = { type: 'section', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
negativeValue = cleanValue;
break;
case 'task':
term = { type: 'task', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
negativeValue = cleanValue;
break;
case 'task-todo':
term = { type: 'task-todo', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
negativeValue = cleanValue;
break;
case 'task-done':
term = { type: 'task-done', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*') };
negativeValue = cleanValue;
break;
case 'match-case':
diagnostics.filters.caseSensitive = true;
term = { type: 'match-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: true };
negativeValue = cleanValue;
break;
case 'ignore-case':
diagnostics.filters.caseSensitive = false;
term = { type: 'ignore-case', value: cleanValue, negated, quoted: valueQuoted, wildcard: cleanValue.includes('*'), caseSensitive: false };
negativeValue = cleanValue;
break;
default:
diagnostics.warnings.push(`UnknownOperator:${prefix}`);
// treat as text with original value including prefix
term = { type: 'text', value, negated, quoted, wildcard };
negativeValue = value;
break;
}
}
} }
} }
// Check for property search [property:value] or [property] if (!term && value.startsWith('[') && value.endsWith(']')) {
if (value.startsWith('[') && value.endsWith(']')) {
const inner = value.substring(1, value.length - 1); const inner = value.substring(1, value.length - 1);
const propColonIndex = inner.indexOf(':'); const propColonIndex = inner.indexOf(':');
if (propColonIndex > 0) { if (propColonIndex > 0) {
const propertyKey = inner.substring(0, propColonIndex); const propertyKey = inner.substring(0, propColonIndex);
let propertyValue = inner.substring(propColonIndex + 1); let propertyValue = inner.substring(propColonIndex + 1);
let propValueQuoted = false; let propValueQuoted = false;
// Remove quotes from property value
if (propertyValue.startsWith('"') && propertyValue.endsWith('"')) { if (propertyValue.startsWith('"') && propertyValue.endsWith('"')) {
propValueQuoted = true; propValueQuoted = true;
propertyValue = propertyValue.substring(1, propertyValue.length - 1); propertyValue = propertyValue.substring(1, propertyValue.length - 1);
} }
term = {
return {
type: 'property', type: 'property',
value: propertyValue, value: propertyValue,
propertyKey, propertyKey,
@ -357,20 +433,52 @@ function parseTerm(token: string, options?: SearchOptions): SearchTerm | null {
quoted: propValueQuoted, quoted: propValueQuoted,
wildcard: propertyValue.includes('*') wildcard: propertyValue.includes('*')
}; };
negativeValue = propertyValue || propertyKey;
} else {
term = {
type: 'property',
value: '',
propertyKey: inner,
negated,
quoted: false,
wildcard: false
};
negativeValue = inner;
} }
// Property existence check [property]
return {
type: 'property',
value: '',
propertyKey: inner,
negated,
quoted: false,
wildcard: false
};
} }
// Default: text search if (!term) {
return { type: 'text', value, negated, quoted, wildcard }; term = { type: 'text', value, negated, quoted, wildcard };
negativeValue = value;
}
if (negated && negativeValue && term) {
diagnostics.filters.negative.push(negativeValue);
diagnostics.filters.negativeDetails.push({
type: term.type,
value: negativeValue,
wildcard: 'wildcard' in term ? Boolean((term as any).wildcard) : undefined
});
}
if (term.type === 'regex') {
diagnostics.filters.regex = true;
}
return term;
}
function createEmptyDiagnosticsFilters(options?: SearchOptions): SearchFilterDiagnostics {
return {
tag: [],
path: [],
file: [],
negative: [],
negativeDetails: [],
regex: options?.regexMode ?? false,
caseSensitive: options?.caseSensitive ?? false,
wholeWord: (options as any)?.wholeWord ?? false
};
} }
/** /**

View File

@ -41,9 +41,33 @@ export interface SearchGroup {
export type SearchNode = SearchTerm | SearchGroup; export type SearchNode = SearchTerm | SearchGroup;
export interface NegativeFilterDiagnostic {
type: SearchTermType;
value: string;
wildcard?: boolean;
}
export interface SearchFilterDiagnostics {
tag: string[];
path: string[];
file: string[];
negative: string[];
negativeDetails: NegativeFilterDiagnostic[];
regex: boolean;
caseSensitive: boolean;
wholeWord: boolean;
}
export interface ParseDiagnostics {
tokens: string[];
filters: SearchFilterDiagnostics;
warnings: string[];
}
export interface ParsedQuery { export interface ParsedQuery {
ast: SearchNode; ast: SearchNode;
isEmpty: boolean; isEmpty: boolean;
diagnostics?: ParseDiagnostics;
} }
/** Predicate function to test if content matches the query */ /** Predicate function to test if content matches the query */

14
src/test-setup.ts Normal file
View File

@ -0,0 +1,14 @@
// Test setup file for Karma/Jasmine
import 'zone.js';
import 'zone.js/testing';
import { getTestBed } from '@angular/core/testing';
import {
BrowserDynamicTestingModule,
platformBrowserDynamicTesting
} from '@angular/platform-browser-dynamic/testing';
// First, initialize the Angular testing environment.
getTestBed().initTestEnvironment(
BrowserDynamicTestingModule,
platformBrowserDynamicTesting(),
);

19
test-parser-debug.js Normal file
View File

@ -0,0 +1,19 @@
// Quick test to debug parser behavior
const { parseSearchQuery } = require('./dist/core/search/search-parser');
const query1 = 'tag:#home file:"Project Plan.md"';
const parsed1 = parseSearchQuery(query1, { caseSensitive: false, regexMode: false });
console.log('Query 1:', query1);
console.log('Tokens:', parsed1.diagnostics?.tokens);
console.log('Tag filters:', parsed1.diagnostics?.filters.tag);
console.log('File filters:', parsed1.diagnostics?.filters.file);
console.log('---');
const query2 = 'tag:#home -path:"Archive" -content:"secret"';
const parsed2 = parseSearchQuery(query2, { caseSensitive: false, regexMode: false });
console.log('Query 2:', query2);
console.log('Tokens:', parsed2.diagnostics?.tokens);
console.log('Negative filters:', parsed2.diagnostics?.filters.negative);
console.log('Negative details:', parsed2.diagnostics?.filters.negativeDetails);

View File

@ -9,6 +9,9 @@
"allowJs": false, "allowJs": false,
"types": ["jasmine", "node"] "types": ["jasmine", "node"]
}, },
"files": [
"src/test-setup.ts"
],
"include": [ "include": [
"src/**/*.spec.ts", "src/**/*.spec.ts",
"src/**/*.d.ts" "src/**/*.d.ts"

View File

@ -1,22 +1,22 @@
{ {
"collapse-filter": false, "collapse-filter": false,
"search": "", "search": "",
"showTags": true, "showTags": false,
"showAttachments": false, "showAttachments": false,
"hideUnresolved": false, "hideUnresolved": false,
"showOrphans": false, "showOrphans": true,
"collapse-color-groups": false, "collapse-color-groups": false,
"colorGroups": [], "colorGroups": [],
"collapse-display": false, "collapse-display": false,
"showArrow": false, "showArrow": false,
"textFadeMultiplier": -3, "textFadeMultiplier": 0,
"nodeSizeMultiplier": 0.25, "nodeSizeMultiplier": 1,
"lineSizeMultiplier": 1.45, "lineSizeMultiplier": 1,
"collapse-forces": false, "collapse-forces": false,
"centerStrength": 0.27, "centerStrength": 0.3,
"repelStrength": 10, "repelStrength": 17,
"linkStrength": 0.15, "linkStrength": 0.5,
"linkDistance": 102, "linkDistance": 200,
"scale": 1.4019828977761002, "scale": 1,
"close": false "close": false
} }