feat: add search diagnostics logging and tag cardinality tracking

2025-10-05 16:24:19 -04:00
parent 1ddfd18af3
commit 989f3ee25a
15 changed files with 1567 additions and 189 deletions
@@ -0,0 +1,513 @@
+import { Injectable, PLATFORM_ID, inject } from '@angular/core';
+import { isPlatformBrowser } from '@angular/common';
+import { environment } from '../../../core/logging/environment';
+import { SearchDiagEvent, SearchDiagRecord } from './search-log.model';
+
+const SESSION_STORAGE_KEY = 'obsiviewer.searchdiag.sessionId';
+const LOCAL_STORAGE_QUEUE_KEY = 'obsiviewer.searchdiag.queue';
+const SAMPLE_DECISIONS_LIMIT = 1000;
+const MAX_DATA_SIZE = 5 * 1024; // 5KB
+const LARGE_ARRAY_LIMIT = 200;
+
+type SearchLogLevel = 'info' | 'warn' | 'error';
+
+interface SanitizedDictionary {
+  [key: string]: SanitizedValue;
+}
+
+type SanitizedValue =
+  | string
+  | number
+  | boolean
+  | null
+  | undefined
+  | SanitizedValue[]
+  | SanitizedDictionary;
+
+type Dictionary<T = unknown> = Record<string, T>;
+
+@Injectable({ providedIn: 'root' })
+export class SearchLogService {
+  private readonly platformId = inject(PLATFORM_ID);
+  private readonly isBrowser = isPlatformBrowser(this.platformId);
+
+  private readonly config = environment.logging;
+  private readonly sessionId = this.ensureSessionId();
+
+  private queue: SearchDiagRecord[] = [];
+  private flushTimer?: ReturnType<typeof setTimeout>;
+  private consecutiveFailures = 0;
+  private circuitBreakerOpenUntil = 0;
+  private isFlushing = false;
+
+  private readonly correlationQueries = new Map<string, string>();
+  private readonly sampleDecisions = new Map<string, boolean>();
+
+  constructor() {
+    if (this.isBrowser) {
+      this.loadQueueFromStorage();
+      if (typeof window !== 'undefined') {
+        window.addEventListener('online', () => this.scheduleFlush());
+        window.addEventListener('beforeunload', () => this.flushSync());
+      }
+    }
+  }
+
+  logSearch(
+    stage: SearchDiagEvent,
+    data: Record<string, unknown>,
+    level: SearchLogLevel = 'info',
+    correlationId?: string
+  ): string {
+    if (!this.isBrowser || !this.config.enabled) {
+      return correlationId ?? '';
+    }
+
+    const effectiveCorrelationId = correlationId || this.generateUUID();
+
+    if (!this.shouldSample(effectiveCorrelationId)) {
+      return effectiveCorrelationId;
+    }
+
+    const queryRaw = this.extractQueryRaw(effectiveCorrelationId, stage, data);
+
+    const record: SearchDiagRecord = {
+      ts: new Date().toISOString(),
+      app: 'ObsiViewer',
+      sessionId: this.sessionId,
+      level,
+      correlationId: effectiveCorrelationId,
+      queryRaw,
+      context: this.buildContext(),
+      stage,
+      data: this.prepareData(stage, data),
+      userAgent: this.getUserAgent(),
+    };
+
+    this.enqueue(record);
+    return effectiveCorrelationId;
+  }
+
+  async flush(): Promise<void> {
+    if (!this.isBrowser) {
+      return;
+    }
+
+    clearTimeout(this.flushTimer);
+    this.flushTimer = undefined;
+
+    await this.processQueue();
+  }
+
+  private flushSync(): void {
+    if (!this.isBrowser || this.queue.length === 0) {
+      return;
+    }
+
+    const payload = this.queue.slice();
+    this.queue = [];
+    this.persistQueue();
+
+    if ('sendBeacon' in navigator) {
+      const blob = new Blob([JSON.stringify(payload)], { type: 'application/json' });
+      navigator.sendBeacon(this.config.endpoint, blob);
+    }
+  }
+
+  private enqueue(record: SearchDiagRecord): void {
+    this.queue.push(record);
+    this.persistQueue();
+
+    if (this.queue.length >= this.config.batchSize) {
+      this.scheduleFlush(0);
+    } else {
+      this.scheduleFlush();
+    }
+  }
+
+  private scheduleFlush(delayMs = this.config.debounceMs): void {
+    if (!this.isBrowser) {
+      return;
+    }
+
+    if (this.flushTimer) {
+      return;
+    }
+
+    this.flushTimer = setTimeout(() => {
+      this.flushTimer = undefined;
+
+      if (typeof window !== 'undefined' && 'requestIdleCallback' in window) {
+        requestIdleCallback(() => this.processQueue());
+      } else {
+        this.processQueue();
+      }
+    }, delayMs);
+  }
+
+  private async processQueue(): Promise<void> {
+    if (!this.isBrowser) {
+      return;
+    }
+
+    if (this.isFlushing || this.queue.length === 0) {
+      return;
+    }
+
+    if (Date.now() < this.circuitBreakerOpenUntil) {
+      return;
+    }
+
+    if (typeof navigator !== 'undefined' && !navigator.onLine) {
+      return;
+    }
+
+    this.isFlushing = true;
+    const batch = [...this.queue];
+
+    try {
+      await this.sendWithRetry(batch);
+      this.queue = [];
+      this.consecutiveFailures = 0;
+      this.persistQueue();
+    } catch (error) {
+      this.consecutiveFailures++;
+
+      if (this.consecutiveFailures >= this.config.circuitBreakerThreshold) {
+        this.circuitBreakerOpenUntil = Date.now() + this.config.circuitBreakerResetMs;
+      }
+
+      console.warn('[SearchLogService] Failed to send diagnostics batch', error);
+    } finally {
+      this.isFlushing = false;
+    }
+  }
+
+  private async sendWithRetry(batch: SearchDiagRecord[]): Promise<void> {
+    let lastError: unknown;
+
+    for (let attempt = 0; attempt < this.config.maxRetries; attempt++) {
+      try {
+        await this.postBatch(batch);
+        return;
+      } catch (error) {
+        lastError = error;
+
+        if (attempt < this.config.maxRetries - 1) {
+          const delayMs = 500 * Math.pow(2, attempt);
+          await this.delay(delayMs);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  private async postBatch(batch: SearchDiagRecord[]): Promise<void> {
+    if (batch.length === 0) {
+      return;
+    }
+
+    const payload = batch.length === 1 ? batch[0] : batch;
+
+    const response = await fetch(this.config.endpoint, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload),
+      keepalive: true,
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+  }
+
+  private buildContext(): SearchDiagRecord['context'] {
+    const context: SearchDiagRecord['context'] = {
+      version: environment.appVersion,
+    };
+
+    if (!this.isBrowser) {
+      return context;
+    }
+
+    if (typeof window !== 'undefined' && window.location) {
+      context.route = window.location.pathname + window.location.search;
+    }
+
+    if (typeof document !== 'undefined' && document.documentElement) {
+      const theme = document.documentElement.getAttribute('data-theme');
+      if (theme === 'light' || theme === 'dark') {
+        context.theme = theme;
+      }
+    }
+
+    try {
+      const vault = localStorage.getItem('obsiviewer.vaultName');
+      if (vault) {
+        context.vault = vault;
+      }
+    } catch {
+      // ignore storage errors
+    }
+
+    return context;
+  }
+
+  private getUserAgent(): string | undefined {
+    if (!this.isBrowser || typeof navigator === 'undefined') {
+      return undefined;
+    }
+    return navigator.userAgent;
+  }
+
+  private prepareData(stage: SearchDiagEvent, data: Record<string, unknown>): Dictionary {
+    const cloned = this.cloneData(data);
+    delete cloned.queryRaw;
+    delete cloned['noteContent'];
+
+    const sanitized = this.sanitizeValue(cloned, [stage]) as Dictionary;
+    return this.enforceSizeLimit(sanitized);
+  }
+
+  private sanitizeValue(value: unknown, path: string[], seen = new WeakSet<object>()): SanitizedValue {
+    if (value === null || value === undefined) {
+      return value as SanitizedValue;
+    }
+
+    if (typeof value === 'number' || typeof value === 'boolean') {
+      return value;
+    }
+
+    if (typeof value === 'string') {
+      return this.sanitizeString(value, path);
+    }
+
+    if (Array.isArray(value)) {
+      if (seen.has(value)) {
+        return '[Circular]' as SanitizedValue;
+      }
+      seen.add(value);
+
+      const limited = value.slice(0, LARGE_ARRAY_LIMIT);
+      const sanitizedArray = limited.map((entry, index) => this.sanitizeValue(entry, [...path, String(index)], seen));
+
+      if (value.length > LARGE_ARRAY_LIMIT) {
+        sanitizedArray.push('[Truncated]' as unknown as SanitizedValue);
+      }
+
+      seen.delete(value);
+      return sanitizedArray;
+    }
+
+    if (typeof value === 'object') {
+      if (seen.has(value as object)) {
+        return '[Circular]' as SanitizedValue;
+      }
+
+      seen.add(value as object);
+      const result: Dictionary<SanitizedValue> = {};
+
+      for (const [key, entry] of Object.entries(value as Dictionary)) {
+        const lowerKey = key.toLowerCase();
+
+        if (this.isContentKey(lowerKey)) {
+          result[key] = '[REDACTED_CONTENT]';
+          continue;
+        }
+
+        if (this.isSensitivePathKey(lowerKey)) {
+          result[key] = this.maskPath(String(entry));
+          continue;
+        }
+
+        result[key] = this.sanitizeValue(entry, [...path, key], seen);
+      }
+
+      seen.delete(value as object);
+      return result;
+    }
+
+    return String(value);
+  }
+
+  private sanitizeString(value: string, path: string[]): string {
+    const lowerPath = path[path.length - 1]?.toLowerCase() ?? '';
+
+    if (this.isContentKey(lowerPath)) {
+      return '[REDACTED_CONTENT]';
+    }
+
+    if (this.isSensitivePathKey(lowerPath)) {
+      return this.maskPath(value);
+    }
+
+    if (value.length > MAX_DATA_SIZE) {
+      return `${value.slice(0, 512)}…[TRUNCATED_${value.length}]`;
+    }
+
+    return value;
+  }
+
+  private enforceSizeLimit<T>(value: T): T {
+    try {
+      const serialized = JSON.stringify(value);
+      if (serialized.length > MAX_DATA_SIZE) {
+        return {
+          _truncated: true,
+          _originalSize: serialized.length,
+          _message: 'Payload truncated due to size limit',
+        } as unknown as T;
+      }
+    } catch (error) {
+      return {
+        _error: 'Failed to serialize payload',
+        _message: String(error),
+      } as unknown as T;
+    }
+
+    return value;
+  }
+
+  private isContentKey(key: string): boolean {
+    return key.includes('content') || key.includes('raw') || key.includes('body') || key.includes('html');
+  }
+
+  private isSensitivePathKey(key: string): boolean {
+    return key.includes('path') || key.includes('filepath') || key.includes('folder');
+  }
+
+  private maskPath(value: string): string {
+    if (!value) {
+      return value;
+    }
+
+    const segments = value.split(/[\\/]+/).filter(Boolean);
+    if (segments.length === 0) {
+      return value;
+    }
+
+    const tail = segments.slice(-2).join('/');
+    return segments.length > 2 ? `…/${tail}` : tail;
+  }
+
+  private cloneData(data: Record<string, unknown>): Dictionary {
+    return JSON.parse(JSON.stringify(data ?? {})) as Dictionary;
+  }
+
+  private ensureSessionId(): string {
+    if (!this.isBrowser) {
+      return this.generateUUID();
+    }
+
+    try {
+      let id = sessionStorage.getItem(SESSION_STORAGE_KEY);
+      if (!id) {
+        id = this.generateUUID();
+        sessionStorage.setItem(SESSION_STORAGE_KEY, id);
+      }
+      return id;
+    } catch {
+      return this.generateUUID();
+    }
+  }
+
+  private shouldSample(correlationId: string): boolean {
+    const rate = this.config.sampleRateSearchDiag ?? 1;
+    if (rate >= 1) {
+      return true;
+    }
+
+    if (this.sampleDecisions.has(correlationId)) {
+      return this.sampleDecisions.get(correlationId) as boolean;
+    }
+
+    const decision = Math.random() < rate;
+    this.sampleDecisions.set(correlationId, decision);
+
+    if (this.sampleDecisions.size > SAMPLE_DECISIONS_LIMIT) {
+      const [firstKey] = this.sampleDecisions.keys();
+      if (firstKey) {
+        this.sampleDecisions.delete(firstKey);
+      }
+    }
+
+    return decision;
+  }
+
+  private extractQueryRaw(
+    correlationId: string,
+    stage: SearchDiagEvent,
+    data: Record<string, unknown>
+  ): string {
+    let query = '';
+    const maybeQuery = data['queryRaw'];
+    if (typeof maybeQuery === 'string') {
+      query = maybeQuery;
+    } else {
+      query = this.correlationQueries.get(correlationId) ?? '';
+    }
+
+    if (stage === 'SEARCH_DIAG_START' && query) {
+      this.correlationQueries.set(correlationId, query);
+    }
+
+    if (!query && typeof data['query'] === 'string') {
+      query = data['query'] as string;
+    }
+
+    return query;
+  }
+
+  private persistQueue(): void {
+    if (!this.isBrowser) {
+      return;
+    }
+
+    try {
+      localStorage.setItem(LOCAL_STORAGE_QUEUE_KEY, JSON.stringify(this.queue));
+    } catch {
+      // ignore
+    }
+  }
+
+  private loadQueueFromStorage(): void {
+    if (!this.isBrowser) {
+      return;
+    }
+
+    try {
+      const stored = localStorage.getItem(LOCAL_STORAGE_QUEUE_KEY);
+      if (!stored) {
+        return;
+      }
+      const parsed = JSON.parse(stored);
+      if (Array.isArray(parsed)) {
+        this.queue = parsed;
+        if (this.queue.length > 0) {
+          this.scheduleFlush();
+        }
+      }
+    } catch {
+      // ignore parse errors
+    }
+  }
+
+  private generateUUID(): string {
+    if (this.isBrowser && typeof window !== 'undefined' && 'crypto' in window && typeof crypto.randomUUID === 'function') {
+      return crypto.randomUUID();
+    }
+
+    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (char) => {
+      const random = (Math.random() * 16) | 0;
+      const value = char === 'x' ? random : (random & 0x3) | 0x8;
+      return value.toString(16);
+    });
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
@@ -0,0 +1,37 @@
+export type SearchDiagEvent =
+  | 'SEARCH_DIAG_START'
+  | 'SEARCH_DIAG_PARSE'
+  | 'SEARCH_DIAG_PLAN'
+  | 'SEARCH_DIAG_EXEC_PROVIDER'
+  | 'SEARCH_DIAG_RESULT_MAP'
+  | 'SEARCH_DIAG_SUMMARY'
+  | 'SEARCH_DIAG_ERROR';
+
+export interface SearchDiagRecord {
+  ts: string;
+  app: 'ObsiViewer';
+  sessionId: string;
+  level: 'info' | 'warn' | 'error';
+  correlationId: string;
+  queryRaw: string;
+  context: {
+    route?: string;
+    vault?: string;
+    theme?: 'dark' | 'light';
+    version?: string;
+  };
+  userAgent?: string;
+  stage: SearchDiagEvent;
+  data: Record<string, unknown>;
+}
+
+export interface SearchDiagContextMeta {
+  userAgent?: string;
+  sampleRate?: number;
+}
+
+export interface SearchDiagOptions {
+  correlationId?: string;
+  level?: 'info' | 'warn' | 'error';
+  queryRaw: string;
+}