perf: O(T) stemming instead of O(S×T) — fixes 15min index freeze
Replace double-nested stem loop (stems × tokens) with single-pass stem frequency map. For 100 unique tokens per file: 10,000 iterations → 100 iterations per file. Critical for large vaults.
This commit is contained in:
parent
25cfd7cc56
commit
1b9ba69c52
@ -406,16 +406,17 @@ class InvertedIndex:
|
||||
for token in tokens:
|
||||
tf[token] += 1
|
||||
# Also index stemmed forms (French stemming)
|
||||
stems = stem_tokens(list(tf.keys()))
|
||||
# Compute stem frequencies in one pass instead of O(S×T)
|
||||
stem_freqs: Dict[str, int] = defaultdict(int)
|
||||
for token, freq in tf.items():
|
||||
stemmed = stem_token(token)
|
||||
if stemmed != token: # only index stem if different
|
||||
stem_freqs[stemmed] += freq
|
||||
for token, freq in tf.items():
|
||||
self.word_index[token][doc_key] = freq
|
||||
for stem in stems:
|
||||
# Accumulate frequency for stem (sum of all forms mapping to same stem)
|
||||
stem_freq = self.word_index[stem].get(doc_key, 0)
|
||||
for token, freq in tf.items():
|
||||
if stem_token(token) == stem:
|
||||
stem_freq += freq
|
||||
self.word_index[stem][doc_key] = stem_freq
|
||||
for stem, freq in stem_freqs.items():
|
||||
existing = self.word_index[stem].get(doc_key, 0)
|
||||
self.word_index[stem][doc_key] = existing + freq
|
||||
|
||||
# --- Tag indexes ---
|
||||
for tag in vault_data.get("tags", {}):
|
||||
@ -479,20 +480,21 @@ class InvertedIndex:
|
||||
for token in tokens:
|
||||
if token:
|
||||
tf[token] += 1
|
||||
# Also compute stems
|
||||
stems = stem_tokens(list(tf.keys()))
|
||||
# Also compute stems in one pass
|
||||
stem_freqs: Dict[str, int] = defaultdict(int)
|
||||
for token, freq in tf.items():
|
||||
stemmed = stem_token(token)
|
||||
if stemmed != token:
|
||||
stem_freqs[stemmed] += freq
|
||||
for token, freq in tf.items():
|
||||
if not self.word_index.get(token):
|
||||
self._sorted_tokens.add(token)
|
||||
self.word_index[token][doc_key] = freq
|
||||
for stem in stems:
|
||||
stem_freq = self.word_index[stem].get(doc_key, 0)
|
||||
for token, freq in tf.items():
|
||||
if stem_token(token) == stem:
|
||||
stem_freq += freq
|
||||
for stem, freq in stem_freqs.items():
|
||||
if not self.word_index.get(stem):
|
||||
self._sorted_tokens.add(stem)
|
||||
self.word_index[stem][doc_key] = stem_freq
|
||||
existing = self.word_index[stem].get(doc_key, 0)
|
||||
self.word_index[stem][doc_key] = existing + freq
|
||||
|
||||
def remove_document(self, vault_name: str, path: str):
|
||||
"""Remove a single document incrementally."""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user