feat(import): v5.6.0 unified data import (phases 0-5)

- unified importer framework (app/services/importers/): normalized model,
  registry, common pipeline (hierarchy, attachments, collections, dedup),
  async jobs, dry-run preview, column->type mapping
- Phase 1: Obsidian, Notion, Logseq/Roam, HTML (Apple Notes/Bear/Ulysses/
  OneNote), Google Keep, generic Markdown
- Phase 2: typed CSV/TSV, Excel (openpyxl), generic JSON
- Phase 3: Word .docx (python-docx), PDF (pypdf), HTML folders
- Phase 4: Raindrop, Pocket, Readwise, Shaarli, Netscape bookmarks, .ics,
  OPML, Standard Notes, Gitea/GitHub issues (+labels/milestones)
- Phase 5: incremental re-sync (skip/update/duplicate), partial-error resume,
  forge repo files, URL web clipper (SSRF guard), batch multi-file + UI queue,
  Notion relation resolution, exportable JSON reports
- /import wizard, API /api/import/*, migration 9 (import_items, import_jobs)
- fix: property values stored by property id (correct DB view rendering)
- deps: openpyxl, beautifulsoup4, PyYAML, python-docx, pypdf
- 43 import tests; full suite 491 green; ruff clean
- bump version 5.11.5
This commit is contained in:
2026-09-13 10:43:20 -04:00
parent d986959927
commit 3b00cbc371
32 changed files with 5275 additions and 58 deletions
+44
View File
@@ -375,3 +375,47 @@ def _migration_db_templates_validation(conn: sqlite3.Connection) -> None:
(tpl["name"], tpl.get("icon", "📋"), tpl.get("description", ""),
__import__("json").dumps(tpl.get("schema", []))),
)
@register(9, "v5.6.0: import items (dedup) + import jobs")
def _migration_import_framework(conn: sqlite3.Connection) -> None:
"""Unified import framework (Phase 0).
``import_items`` — one row per imported page, keyed by workspace + source +
external id, so re-importing the same vault is idempotent.
``import_jobs`` — background import job status/history for UI polling.
"""
conn.execute(
"""
CREATE TABLE IF NOT EXISTS import_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
workspace_id INTEGER,
source TEXT NOT NULL DEFAULT '',
external_id TEXT NOT NULL DEFAULT '',
page_id INTEGER,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(workspace_id, source, external_id)
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_import_items_lookup "
"ON import_items(workspace_id, source, external_id)"
)
conn.execute(
"""
CREATE TABLE IF NOT EXISTS import_jobs (
id TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT '',
filename TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'queued',
error TEXT NOT NULL DEFAULT '',
report_json TEXT NOT NULL DEFAULT '{}',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_import_jobs_created ON import_jobs(created_at)"
)