Projets/ObsiViewer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add non-markdown files to metadata endpoints

Browse Source 
- Added support for non-markdown files (images, PDFs, code, etc) in vault metadata endpoints
- Implemented filesystem scanning for non-markdown files with configurable extension whitelist
- Added deduplication by file path to prevent duplicates between Meilisearch and filesystem
- Enhanced folder filtering and kind filtering to properly handle non-markdown files
- Added validation to remove stale entries not present on disk
- Improved logging to track
This commit is contained in:
bruno
2025-10-31 23:20:36 -04:00
parent 545c07a4b3
commit b985a7cdbb
55 changed files with 8465 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/features/list/paginated-notes-list.component.ts
+147 -19
View File
@@ -46,6 +46,15 @@ import { takeUntil } from 'rxjs/operators';
<svg class="h-3.5 w-3.5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>
</button>
</div>
<div *ngIf="folderFilter() as f" class="flex items-center gap-2 text-xs">
<span class="inline-flex items-center gap-1 rounded-full bg-surface1 dark:bg-card text-main dark:text-main px-2 py-1">
<svg class="h-3.5 w-3.5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 7v10a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2h-6l-2-2H5a2 2 0 0 0-2 2z"/></svg>
{{ f }}
</span>
<button type="button" (click)="clearFolderFilter.emit()" class="rounded-full hover:bg-slate-500/10 dark:hover:bg-surface2/10 w-6 h-6 inline-flex items-center justify-center" title="Effacer le dossier">
<svg class="h-3.5 w-3.5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>
</button>
</div>
<input type="text"
[value]="query()"
(input)="onQuery($any($event.target).value)"
@@ -325,6 +334,7 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
@Output() openNote = new EventEmitter<string>();
@Output() queryChange = new EventEmitter<string>();
@Output() clearQuickLinkFilter = new EventEmitter<void>();
@Output() clearFolderFilter = new EventEmitter<void>();
// Local state
private q = signal('');
@@ -392,17 +402,50 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
});
}
// If Tag or Quick is active, use full vault notes (markdown) to ensure tags/frontmatter are present
const quickKey = String(this.quickLinkFilter() || '').toLowerCase();
const urlTag = (this.tagFilter() || '').toLowerCase();
const tagActive = !!urlTag || this.filter.tags().length > 0;
if (tagActive || !!quickKey) {
usedFallback = true;
const mdVaultNotes = vaultNotes.filter(v => this.fileTypes.getViewerType(v.filePath, v.rawContent ?? v.content ?? '') === 'markdown');
items = mdVaultNotes.map(n => ({
id: n.id,
title: n.title,
filePath: n.filePath,
createdAt: (n as any).createdAt,
updatedAt: (n as any).updatedAt || (n.mtime ? new Date(n.mtime).toISOString() : '')
}));
// Apply folder/trash constraint to the rebuilt list as well
if (folder) {
items = items.filter(n => {
const op = (n.filePath || '').toLowerCase().replace(/^\/+|\/+$/g, '');
const fp = (n.filePath || '').toLowerCase().replace(/\\/g, '/');
return folder === '.trash'
? (fp.startsWith('.trash/') || fp.includes('/.trash/'))
: (op === folder || op.startsWith(folder + '/'));
});
} else {
items = items.filter(n => {
const fp = (n.filePath || '').toLowerCase().replace(/\\/g, '/');
return !fp.startsWith('.trash/') && !fp.includes('/.trash/');
});
}
}
// Kind filters (FilterService multi-kinds first; fallback to single kindFilter)
const kinds = this.filter.kinds();
const urlKind = this.kindFilter();
let allowedKinds = new Set<string>(kinds.length > 0 ? kinds : (urlKind && urlKind !== 'all' ? [urlKind] : []));
// Folder/Trash views must show all types unless quick/tag constrain to markdown
// Folder/Trash views: if kinds are explicitly selected, honor them; otherwise default to 'all'
const folderActive = !!folder;
const quickActive = !!this.quickLinkFilter();
const tagActive = !!(this.tagFilter() || '').trim() || this.filter.tags().length > 0;
if (folderActive && !quickActive && !tagActive) {
allowedKinds = new Set<string>(); // no restriction in folder/trash
const quickActive = !!quickKey;
// tagActive already computed above
// Do not override allowedKinds in folder view; when none selected and no tag/quick, the filter below treats size 0 as 'all'
// IMPORTANT: Tags/Quick enforce markdown-only regardless of kind filters
if (tagActive || quickActive) {
allowedKinds = new Set<string>(['markdown']);
}
if (allowedKinds.size > 0) {
@@ -416,37 +459,41 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
}
// Tag and Quick Link filters using vault metadata when available
const urlTag = (this.tagFilter() || '').toLowerCase();
const urlTag2 = (this.tagFilter() || '').toLowerCase();
const localTags = this.filter.tags().map(t => (t || '').toLowerCase());
const quick = this.quickLinkFilter();
if (urlTag || localTags.length > 0) {
const quickKey2 = String(this.quickLinkFilter() || '').toLowerCase();
if (urlTag2 || localTags.length > 0) {
items = items.filter(n => {
const full = byId.get(n.id);
const ntags: string[] = Array.isArray(full?.tags) ? full.tags.map((t: string) => (t || '').toLowerCase()) : [];
if (urlTag && !ntags.includes(urlTag)) return false;
if (urlTag2 && !ntags.includes(urlTag2)) return false;
for (const t of localTags) { if (!ntags.includes(t)) return false; }
// Tags view must show markdown only
return this.matchesKind(n.filePath, 'markdown');
});
}
if (quick) {
if (quickKey2) {
items = items.filter(n => {
const full = byId.get(n.id);
const fm = full?.frontmatter || {};
return fm[quick] === true && this.matchesKind(n.filePath, 'markdown');
return fm[quickKey2] === true && this.matchesKind(n.filePath, 'markdown');
});
}
// If allowed kinds include any non-markdown type OR no kinds selected at all (default 'all'),
// ensure those files appear even if pagination didn't include them (server may return only markdown)
const needMergeForKinds = (allowedKinds.size > 0 && Array.from(allowedKinds).some(k => k !== 'markdown'))
|| (allowedKinds.size === 0 && !quick && !tagActive); // default 'all' and no quick/tag constraint
if (needMergeForKinds) {
const present = new Set(items.map(n => n.id));
|| (allowedKinds.size === 0 && !quickActive && !tagActive); // default 'all' and no quick/tag constraint
if (needMergeForKinds && !usedFallback) {
// de-duplicate by filePath (case-insensitive) to avoid duplicates between Meili and FS
const presentPath = new Set(items.map(n => String(n.filePath || '').toLowerCase().replace(/\\/g, '/')));
for (const full of vaultNotes) {
const t = this.fileTypes.getViewerType(full.filePath, full.rawContent ?? full.content ?? '');
// Only merge NON-markdown files to avoid duplicating markdown already provided by Meilisearch
if (t === 'markdown') continue;
const allowByKind = allowedKinds.size === 0 ? true : allowedKinds.has(t);
if (allowByKind && !present.has(full.id)) {
const fullPathLc = String(full.filePath || '').toLowerCase().replace(/\\/g, '/');
if (allowByKind && !presentPath.has(fullPathLc)) {
// Apply same folder filter and tag/quick constraints
const fp = (full.filePath || '').toLowerCase().replace(/\\/g, '/');
const op = (full.filePath || '').toLowerCase().replace(/^\/+|\/+$/g, '');
@@ -460,9 +507,9 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
if (urlTag && !ntags.includes(urlTag)) continue;
let okLocal = true; for (const t of localTags) { if (!ntags.includes(t)) { okLocal = false; break; } }
if (!okLocal) continue;
if (quick) {
if (quickKey2) {
const fm = full.frontmatter || {};
if (fm[quick] !== true) continue;
if (fm[quickKey2] !== true) continue;
}
if (q) {
const titleLc = (full.title || '').toLowerCase();
@@ -470,11 +517,20 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
if (!titleLc.includes(q) && !pathLc.includes(q)) continue;
}
items.push({ id: full.id, title: full.title, filePath: full.filePath, createdAt: (full as any).createdAt, updatedAt: (full as any).updatedAt || (full.mtime ? new Date(full.mtime).toISOString() : '') });
present.add(full.id);
presentPath.add(fullPathLc);
}
}
}
// Final de-duplication by filePath (case-insensitive) to avoid duplicates pointing to same file
const byPath = new Map<string, NoteMetadata>();
for (const it of items) {
const key = String(it.filePath || '').toLowerCase().replace(/\\/g, '/');
if (!key) continue;
if (!byPath.has(key)) byPath.set(key, it);
}
items = Array.from(byPath.values());
// Sorting (title/created/updated) like old list
const parseDate = (s?: string) => (s ? Date.parse(s) : 0) || 0;
const sortBy = this.state.sortBy();
@@ -498,6 +554,54 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
return items;
});
private prefetchScheduled = false;
private prefetched = new Set<string>();
private schedulePrefetch(): void {
if (this.prefetchScheduled) return;
this.prefetchScheduled = true;
setTimeout(() => {
this.prefetchScheduled = false;
const items = this.visibleNotes();
if (!items || items.length === 0) return;
const slice = items.slice(0, 40);
const paths: string[] = [];
for (const it of slice) {
const p = it.filePath || '';
if (!p) continue;
if (this.prefetched.has(p)) continue;
const kind = this.fileTypes.getViewerType(p, '');
if (kind === 'markdown' || kind === 'excalidraw') {
paths.push(p);
}
}
if (paths.length === 0) return;
(async () => {
// Limit concurrency
const concurrency = 6;
let i = 0;
const runNext = async (): Promise<void> => {
if (i >= paths.length) return;
const path = paths[i++];
try {
this.prefetched.add(path);
await this.vault.ensureNoteLoadedByPath(path);
} catch {}
await runNext();
};
const workers: Promise<void>[] = [];
for (let k = 0; k < Math.min(concurrency, paths.length); k++) {
workers.push(runNext());
}
await Promise.allSettled(workers);
})();
}, 50);
}
private prefetchEffect = effect(() => {
this.visibleNotes();
this.schedulePrefetch();
});
// Effects
private syncQuery = effect(() => {
this.q.set(this.query() || '');
@@ -539,8 +643,9 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
// Select a note
selectNote(note: NoteMetadata) {
try { console.debug('[NotesList] click', { id: note.id, path: note.filePath }); } catch {}
this.selectedNoteId.set(note.id);
this.openNote.emit(note.id);
this.openNote.emit(note.filePath || note.id);
}
// Search
@@ -603,10 +708,33 @@ export class PaginatedNotesListComponent implements OnInit, OnDestroy {
// Color and gradient
private getFullNoteById(id: string): any | null {
// 1) Direct by ID (already loaded)
try {
const n = (this.vault as any).getNoteById?.(id);
if (n) return n;
} catch {}
// 2) Fallback: find current metadata item to derive path -> slug ID
try {
const meta = (this.visibleNotes() || []).find(x => x.id === id);
const path = meta?.filePath || '';
if (path) {
const slug = (this.vault as any).buildSlugIdFromPath?.(path) || '';
if (slug) {
const loaded = (this.vault as any).getNoteById?.(slug);
if (loaded) return loaded;
// On-demand load for markdown/excalidraw to populate frontmatter (color)
const kind = this.fileTypes.getViewerType(path, '');
if (kind === 'markdown' || kind === 'excalidraw') {
try { (this.vault as any).ensureNoteLoadedByPath?.(path); } catch {}
const reloaded = (this.vault as any).getNoteById?.(slug);
if (reloaded) return reloaded;
}
}
}
} catch {}
// 3) Last resort: scan already-loaded notes by id
try {
const list = this.vault.allNotes() || [];
for (const n of list) if ((n as any).id === id) return n;
src/app/layout/app-shell-nimbus/app-shell-nimbus.component.ts
+32 -6
View File
@@ -180,6 +180,7 @@ import { InPageSearchOverlayComponent } from '../../shared/search/in-page-search
(openNote)="onOpenNote($event)"
(queryChange)="onQueryChange($event)"
(clearQuickLinkFilter)="onClearQuickLinkFilter()"
(clearFolderFilter)="onClearFolderFromList()"
/>
</div>
</section>
@@ -695,15 +696,40 @@ export class AppShellNimbusLayoutComponent implements AfterViewInit {
this.mobileNav.setActiveTab(prev);
}
onOpenNote(noteId: string) {
this.noteSelected.emit(noteId);
// Update URL with selected note path
const n = (this.vault.allNotes() || []).find(x => x.id === noteId);
if (n && n.filePath) {
this.urlState.openNote(n.filePath);
async onOpenNote(target: string) {
try { console.debug('[Nimbus] onOpenNote', target); } catch {}
let filePath: string | null = null;
let noteId: string = target;
try {
const looksLikePath = /[/\\]/.test(target) || /\.[a-z0-9]+$/i.test(target);
if (looksLikePath) {
filePath = target.replace(/\\/g, '/');
try { await this.vault.ensureNoteLoadedByPath(filePath); } catch {}
try { noteId = this.vault.buildSlugIdFromPath(filePath); } catch {}
} else {
await this.vault.ensureNoteLoadedById(target);
const n = (this.vault.allNotes() || []).find(x => x.id === target);
filePath = n?.filePath || this.vault.getFastMetaById(target)?.path || null;
}
if (filePath) {
try { console.debug('[Nimbus] opening note path', filePath); } catch {}
this.urlState.openNote(filePath);
} else {
try { console.warn('[Nimbus] onOpenNote: no filePath resolved for', target); } catch {}
}
} finally {
try { console.debug('[Nimbus] emitting noteSelected', noteId); } catch {}
this.noteSelected.emit(noteId);
}
}
onClearFolderFromList() {
this.folderFilter = null;
this.urlState.clearFolderFilter();
}
onAboutSelected(): void {
this.showAboutPanel = true;
}
src/components/file-explorer/file-explorer.component.ts
+43
View File
@@ -3,6 +3,8 @@ import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { VaultNode, VaultFile, VaultFolder } from '../../types';
import { VaultService } from '../../services/vault.service';
import { FileTypeDetectorService } from '../../services/file-type-detector.service';
import { FilterService } from '../../app/services/filter.service';
import { NoteCreationService } from '../../app/services/note-creation.service';
import { UrlStateService } from '../../app/services/url-state.service';
import { NotesListFocusService } from '../../app/services/notes-list-focus.service';
@@ -186,6 +188,8 @@ export class FileExplorerComponent {
private notesListFocus = inject(NotesListFocusService);
private folderFilter = inject(FolderFilterService);
private urlState = inject(UrlStateService);
private types = inject(FileTypeDetectorService);
private filterKinds = inject(FilterService);
// Computed filtered nodes based on folder filter settings
filteredNodes = computed(() => {
@@ -240,6 +244,15 @@ export class FileExplorerComponent {
return filteredNotes.length;
}
// If kinds filters are active, compute counts from the fast file tree with kind filtering
const kinds = this.filterKinds.kinds();
if (kinds && kinds.length > 0) {
const nodes = this.vaultService.fastFileTree();
const target = this.findFolderNode(nodes, path);
if (!target) return 0;
return this.countFilesByKinds(target, new Set(kinds));
}
if (this.useTrashCounts()) {
const counts = this.vaultService.trashFolderCounts();
const raw = (path || '').replace(/\\/g, '/');
@@ -251,6 +264,36 @@ export class FileExplorerComponent {
}
}
private findFolderNode(nodes: VaultNode[], path: string): VaultFolder | null {
const norm = (path || '').replace(/\\/g, '/');
for (const n of nodes) {
if (n.type === 'folder') {
const f = n as VaultFolder;
if (f.path === norm) return f;
const child = this.findFolderNode(f.children, path);
if (child) return child;
}
}
return null;
}
private countFilesByKinds(folder: VaultFolder, kinds: Set<string>): number {
let total = 0;
const visit = (f: VaultFolder) => {
for (const child of f.children) {
if (child.type === 'folder') {
visit(child as VaultFolder);
} else {
const file = child as VaultFile;
const t = this.types.getViewerType(file.path || file.name, '');
if (kinds.has(t)) total++;
}
}
};
visit(folder);
return total;
}
/**
* Get the color for a folder badge, synchronized with folder color
*/
src/services/vault.service.ts
+3 -3
View File
@@ -270,7 +270,7 @@ export class VaultService implements OnDestroy {
const existing = this.notesMap().get(id);
if (existing) return existing;
// Fallback: synthesize a virtual note from fast metadata (for non-markdown files)
const path = this.idToPathIndex.get(id) || this.slugIdToPathIndex.get(id);
const path = this.slugIdToPathIndex.get(id) || this.idToPathIndex.get(id);
const meta = path ? this.metaByPathIndex.get(path) : undefined;
if (!meta || !path) return undefined;
@@ -359,7 +359,7 @@ export class VaultService implements OnDestroy {
async ensureNoteLoadedById(id: string): Promise<boolean> {
if (!id || this.getNoteById(id)) return !!id;
const path = this.idToPathIndex.get(id) || this.slugIdToPathIndex.get(id);
const path = this.slugIdToPathIndex.get(id) || this.idToPathIndex.get(id);
if (!path) return false;
// Try to load real markdown content; if not possible, synthesize virtual note
const ok = await this.ensureNoteLoadedByPath(path);
@@ -454,7 +454,7 @@ export class VaultService implements OnDestroy {
}
getFastMetaById(id: string): FileMetadata | undefined {
const path = this.idToPathIndex.get(id) || this.slugIdToPathIndex.get(id);
const path = this.slugIdToPathIndex.get(id) || this.idToPathIndex.get(id);
return path ? this.metaByPathIndex.get(path) : undefined;
}