diff --git a/backend/search.py b/backend/search.py index 61e4135..096b2c9 100644 --- a/backend/search.py +++ b/backend/search.py @@ -406,16 +406,17 @@ class InvertedIndex: for token in tokens: tf[token] += 1 # Also index stemmed forms (French stemming) - stems = stem_tokens(list(tf.keys())) + # Compute stem frequencies in one pass instead of O(S×T) + stem_freqs: Dict[str, int] = defaultdict(int) + for token, freq in tf.items(): + stemmed = stem_token(token) + if stemmed != token: # only index stem if different + stem_freqs[stemmed] += freq for token, freq in tf.items(): self.word_index[token][doc_key] = freq - for stem in stems: - # Accumulate frequency for stem (sum of all forms mapping to same stem) - stem_freq = self.word_index[stem].get(doc_key, 0) - for token, freq in tf.items(): - if stem_token(token) == stem: - stem_freq += freq - self.word_index[stem][doc_key] = stem_freq + for stem, freq in stem_freqs.items(): + existing = self.word_index[stem].get(doc_key, 0) + self.word_index[stem][doc_key] = existing + freq # --- Tag indexes --- for tag in vault_data.get("tags", {}): @@ -479,20 +480,21 @@ class InvertedIndex: for token in tokens: if token: tf[token] += 1 - # Also compute stems - stems = stem_tokens(list(tf.keys())) + # Also compute stems in one pass + stem_freqs: Dict[str, int] = defaultdict(int) + for token, freq in tf.items(): + stemmed = stem_token(token) + if stemmed != token: + stem_freqs[stemmed] += freq for token, freq in tf.items(): if not self.word_index.get(token): self._sorted_tokens.add(token) self.word_index[token][doc_key] = freq - for stem in stems: - stem_freq = self.word_index[stem].get(doc_key, 0) - for token, freq in tf.items(): - if stem_token(token) == stem: - stem_freq += freq + for stem, freq in stem_freqs.items(): if not self.word_index.get(stem): self._sorted_tokens.add(stem) - self.word_index[stem][doc_key] = stem_freq + existing = self.word_index[stem].get(doc_key, 0) + self.word_index[stem][doc_key] = existing + freq def remove_document(self, vault_name: str, path: str): """Remove a single document incrementally."""