feat(import): v5.6.0 unified data import (phases 0-5)
- unified importer framework (app/services/importers/): normalized model, registry, common pipeline (hierarchy, attachments, collections, dedup), async jobs, dry-run preview, column->type mapping - Phase 1: Obsidian, Notion, Logseq/Roam, HTML (Apple Notes/Bear/Ulysses/ OneNote), Google Keep, generic Markdown - Phase 2: typed CSV/TSV, Excel (openpyxl), generic JSON - Phase 3: Word .docx (python-docx), PDF (pypdf), HTML folders - Phase 4: Raindrop, Pocket, Readwise, Shaarli, Netscape bookmarks, .ics, OPML, Standard Notes, Gitea/GitHub issues (+labels/milestones) - Phase 5: incremental re-sync (skip/update/duplicate), partial-error resume, forge repo files, URL web clipper (SSRF guard), batch multi-file + UI queue, Notion relation resolution, exportable JSON reports - /import wizard, API /api/import/*, migration 9 (import_items, import_jobs) - fix: property values stored by property id (correct DB view rendering) - deps: openpyxl, beautifulsoup4, PyYAML, python-docx, pypdf - 43 import tests; full suite 491 green; ruff clean - bump version 5.11.5
This commit is contained in:
@@ -375,3 +375,47 @@ def _migration_db_templates_validation(conn: sqlite3.Connection) -> None:
|
||||
(tpl["name"], tpl.get("icon", "📋"), tpl.get("description", ""),
|
||||
__import__("json").dumps(tpl.get("schema", []))),
|
||||
)
|
||||
|
||||
|
||||
@register(9, "v5.6.0: import items (dedup) + import jobs")
|
||||
def _migration_import_framework(conn: sqlite3.Connection) -> None:
|
||||
"""Unified import framework (Phase 0).
|
||||
|
||||
``import_items`` — one row per imported page, keyed by workspace + source +
|
||||
external id, so re-importing the same vault is idempotent.
|
||||
``import_jobs`` — background import job status/history for UI polling.
|
||||
"""
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS import_items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workspace_id INTEGER,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
external_id TEXT NOT NULL DEFAULT '',
|
||||
page_id INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(workspace_id, source, external_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_import_items_lookup "
|
||||
"ON import_items(workspace_id, source, external_id)"
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS import_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
report_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_import_jobs_created ON import_jobs(created_at)"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user