"""FlowDeck — versioned schema migrations (lightweight, no Alembic). This replaces the previous "ad-hoc" approach where every new schema change was appended directly to `app/db.py::init_db()` with no tracking. A `schema_version` table now records the highest applied migration; the full baseline schema (created idempotently by `init_db`) is treated as version 1, and any incremental change is expressed as an ordered, versioned step below and applied exactly once. Each migration function receives a raw ``sqlite3.Connection`` (WAL + foreign keys already enabled) and must be written idempotently (``IF NOT EXISTS`` / guarded ``ALTER TABLE``) so it is safe even if partially re-run. """ from __future__ import annotations import logging import sqlite3 from typing import Callable logger = logging.getLogger(__name__) # The full baseline schema created by `app.db::init_db()` is "version 1". BASELINE_VERSION = 1 # (version, name, apply_fn). Kept sorted by version at registration time. MIGRATIONS: list[tuple[int, str, Callable[[sqlite3.Connection], None]]] = [] def register(version: int, name: str) -> Callable: """Decorator registering a migration in the ordered registry.""" if any(v == version for v, _, _ in MIGRATIONS): raise ValueError(f"Duplicate migration version {version}") def decorator(fn: Callable[[sqlite3.Connection], None]): MIGRATIONS.append((version, name, fn)) MIGRATIONS.sort(key=lambda item: item[0]) return fn return decorator def _ensure_table(conn: sqlite3.Connection) -> None: conn.execute( """ CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) def columns(conn: sqlite3.Connection, table: str) -> set[str]: """Colonnes d'une table — A31 : l'unique helper qui remplace les 24 copies de `{r[1] for r in conn.execute("PRAGMA table_info(...)")}`. ``table_exists``/``column_exists`` (préconisés par l'audit) ne sont pas livrés : aucune migration n'interroge ``sqlite_master``, et un contrôle unitaire se lit déjà dans le set. """ if not table.replace("_", "").isalnum(): raise ValueError(f"nom de table invalide: {table!r}") return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()} def current_version(conn: sqlite3.Connection) -> int: _ensure_table(conn) row = conn.execute( "SELECT COALESCE(MAX(version), 0) AS v FROM schema_version" ).fetchone() return int(row[0]) def fts5_available() -> bool: """True when the bundled SQLite ships the FTS5 extension.""" probe = sqlite3.connect(":memory:") try: probe.execute("CREATE VIRTUAL TABLE _fts5_probe USING fts5(x)") return True except sqlite3.OperationalError: return False finally: probe.close() def apply_migrations(conn: sqlite3.Connection) -> int: """Seal the baseline schema (version 1) and apply pending migrations. Returns the resulting schema version. """ _ensure_table(conn) applied = current_version(conn) if applied < BASELINE_VERSION: # The pre-existing schema (already created by init_db) is our baseline. conn.execute( "INSERT OR IGNORE INTO schema_version (version, name) VALUES (?, ?)", (BASELINE_VERSION, "baseline"), ) conn.commit() applied = BASELINE_VERSION for version, name, fn in MIGRATIONS: if version <= applied: continue _apply_one(conn, version, name, fn) applied = version logger.info("Applied migration %d: %s", version, name) return applied def _apply_one(conn: sqlite3.Connection, version: int, name: str, fn: Callable) -> None: """A31 : une migration = une transaction (DDL tout-ou-rien). Avant : le DDL sortait en autocommit (isolation_level legacy) — un échec au milieu laissait un schéma partiel commité ET pas de ligne schema_version : la reprise rejouait un DDL déjà appliqué. Maintenant : BEGIN explicite, rollback complet à l'échec, donc la prochaine exécution retente proprement. """ if conn.in_transaction: # transaction résiduelle du caller (init_db commit juste avant) — on # part d'un état propre plutôt que d'englober son travail. conn.commit() conn.execute("BEGIN") try: fn(conn) conn.execute( "INSERT INTO schema_version (version, name) VALUES (?, ?)", (version, name), ) conn.commit() except BaseException: conn.rollback() raise # ═══════════════════════════════════════════════════════════════════════════ # Migrations # ═══════════════════════════════════════════════════════════════════════════ @register(2, "missing indexes") def _migration_missing_indexes(conn: sqlite3.Connection) -> None: """Add the indexes flagged in the roadmap (fast lookups by email, forge user).""" for ddl in ( "CREATE INDEX IF NOT EXISTS idx_users_email ON users(email)", "CREATE INDEX IF NOT EXISTS idx_user_oauth_tokens_user ON user_oauth_tokens(user_id, provider)", "CREATE INDEX IF NOT EXISTS idx_collections_workspace ON collections(workspace_id)", "CREATE INDEX IF NOT EXISTS idx_pages_workspace ON pages(workspace_id)", "CREATE INDEX IF NOT EXISTS idx_pages_deleted ON pages(deleted_at)", ): conn.execute(ddl) @register(3, "full-text search (FTS5)") def _migration_fts5(conn: sqlite3.Connection) -> None: """Create a full-text index over pages (title + content) for the command palette. Kept in sync via row-level triggers on the ``pages`` table so page insert/update/delete are reflected immediately. Skips gracefully if the bundled SQLite lacks FTS5 (search then falls back to LIKE). """ if not fts5_available(): logger.warning("FTS5 unavailable — skipping full-text index (LIKE fallback active)") return conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(title, body)") conn.execute( """ CREATE TRIGGER IF NOT EXISTS pages_fts_ai AFTER INSERT ON pages BEGIN INSERT INTO pages_fts(rowid, title, body) VALUES (new.id, COALESCE(new.title, ''), COALESCE(new.content, '')); END """ ) # `pages_fts` is a standalone FTS5 table (it stores its own content), so deletes # use a plain DELETE by rowid (NOT the special 'delete' insert that only applies # to external-content/contentless FTS5 tables). conn.execute( """ CREATE TRIGGER IF NOT EXISTS pages_fts_ad AFTER DELETE ON pages BEGIN DELETE FROM pages_fts WHERE rowid = old.id; END """ ) conn.execute( """ CREATE TRIGGER IF NOT EXISTS pages_fts_au AFTER UPDATE ON pages BEGIN DELETE FROM pages_fts WHERE rowid = old.id; INSERT INTO pages_fts(rowid, title, body) VALUES (new.id, COALESCE(new.title, ''), COALESCE(new.content, '')); END """ ) # Backfill the index from any rows that already exist. conn.execute( """ INSERT INTO pages_fts(rowid, title, body) SELECT id, COALESCE(title, ''), COALESCE(content, '') FROM pages WHERE deleted_at IS NULL """ ) @register(5, "automations (v5.1.0 rules engine)") def _migration_automations(conn: sqlite3.Connection) -> None: """v5.1.0: database automations — if-this-then-that rule engine (trigger + condition + action) and clickable buttons that trigger actions. ``automations`` — the rules (event/cron/button trigger, optional condition JSON, actions JSON, run counters). ``automation_runs`` — execution history for auditing and the Settings UI. """ conn.execute( """ CREATE TABLE IF NOT EXISTS automations ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, trigger_type TEXT NOT NULL DEFAULT 'event', -- event | cron | button event TEXT NOT NULL DEFAULT 'page.created', -- for trigger_type='event' cron_expression TEXT NOT NULL DEFAULT '', -- for trigger_type='cron' collection_id INTEGER, -- optional scope (event triggers) condition_json TEXT NOT NULL DEFAULT '[]', -- list of condition clauses actions_json TEXT NOT NULL DEFAULT '[]', -- list of action descriptors enabled BOOLEAN NOT NULL DEFAULT 1, created_by INTEGER, last_run_at TIMESTAMP, run_count INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_automations_trigger ON automations(trigger_type, event, enabled)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS automation_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, automation_id INTEGER NOT NULL REFERENCES automations(id) ON DELETE CASCADE, trigger_source TEXT NOT NULL DEFAULT 'event', status TEXT NOT NULL DEFAULT 'fired', -- fired | skipped | error detail TEXT NOT NULL DEFAULT '', collection_id INTEGER, page_id INTEGER, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_automation_runs_auto ON automation_runs(automation_id, created_at)" ) @register(6, "v5.2.0: api tokens, user sessions, projects") def _migration_v520_security_projects(conn: sqlite3.Connection) -> None: """v5.2.0 (Security & Forge): per-user API tokens, revocable sessions and the forge-agnostic ``projects`` table. ``api_tokens`` — per-user bearer tokens (sha256-stored), revocable, powering the public API (/api/v1) and Settings UI. ``user_sessions`` — one row per signed session cookie; revocation here instantly kills the corresponding cookie. ``projects`` — normalized project list across forges (builtin/gitea/ github) + last sync timestamp for the periodic cron. """ _pcols = columns(conn, "api_tokens") if "id" not in _pcols: conn.execute( """ CREATE TABLE api_tokens ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, name TEXT NOT NULL DEFAULT 'API token', token_hash TEXT NOT NULL UNIQUE, token_prefix TEXT NOT NULL DEFAULT '', last_used_at TIMESTAMP, revoked INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_api_tokens_user ON api_tokens(user_id, revoked)" ) _scols = columns(conn, "user_sessions") if "id" not in _scols: conn.execute( """ CREATE TABLE user_sessions ( id TEXT PRIMARY KEY, -- session id (cookie payload) user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, ip_address TEXT DEFAULT '', user_agent TEXT DEFAULT '', last_seen_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, revoked INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_user_sessions_user ON user_sessions(user_id, revoked)" ) _projcols = columns(conn, "projects") if "id" not in _projcols: conn.execute( """ CREATE TABLE projects ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, proj_type TEXT NOT NULL DEFAULT 'builtin', -- builtin | gitea | github owner TEXT NOT NULL DEFAULT '', forge_id TEXT DEFAULT '', clone_url TEXT DEFAULT '', default_branch TEXT DEFAULT '', language TEXT DEFAULT '', description TEXT DEFAULT '', last_synced_at TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(proj_type, owner, name) ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_projects_type ON projects(proj_type, last_synced_at)" ) @register(7, "v5.4.0/v5.5.0: page versions, cover + icon") def _migration_v54_page_versions_cover(conn: sqlite3.Connection) -> None: """v5.4.0 (version history + page duplication) & v5.5.0 (cover & icon). ``page_versions`` — undoable version snapshots for block-editor pages (NOT tied to ``collection_pages`` like the legacy ``page_history`` table). One row per save with the full block list + title so the UI can browse/restore. ``pages.cover_url`` — image cover shown above the page title. ``pages.page_icon`` — emoji / icon label shown next to the title. """ conn.execute( """ CREATE TABLE IF NOT EXISTS page_versions ( id INTEGER PRIMARY KEY AUTOINCREMENT, page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, user_id INTEGER REFERENCES users(id), title TEXT NOT NULL DEFAULT '', blocks_json TEXT NOT NULL DEFAULT '[]', note TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_page_versions_page ON page_versions(page_id, created_at)" ) _pcols = columns(conn, "pages") if "cover_url" not in _pcols: conn.execute("ALTER TABLE pages ADD COLUMN cover_url TEXT DEFAULT ''") if "page_icon" not in _pcols: conn.execute("ALTER TABLE pages ADD COLUMN page_icon TEXT DEFAULT ''") @register(8, "v5.6.0: custom workspace emojis") def _migration_custom_emojis(conn: sqlite3.Connection) -> None: """Workspace-wide custom emojis (uploaded images) used as page icons.""" conn.execute( """ CREATE TABLE IF NOT EXISTS custom_emojis ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace_id INTEGER NOT NULL DEFAULT 1, name TEXT NOT NULL DEFAULT '', url TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_custom_emojis_ws ON custom_emojis(workspace_id, created_at)" ) @register(4, "database templates (icon) + property validation") def _migration_db_templates_validation(conn: sqlite3.Connection) -> None: """v5.3.0: database templates get an icon, properties a validation config, and the built-in database templates are seeded (idempotently).""" _cols = columns(conn, "database_templates") if "icon" not in _cols: conn.execute("ALTER TABLE database_templates ADD COLUMN icon TEXT NOT NULL DEFAULT '📋'") _pcols = columns(conn, "collection_properties") if "validation_json" not in _pcols: conn.execute("ALTER TABLE collection_properties ADD COLUMN validation_json TEXT NOT NULL DEFAULT '{}'") # Seed built-in templates (idempotent: only missing names are inserted). from app.services.db_templates import SEED_TEMPLATES for tpl in SEED_TEMPLATES: conn.execute( """INSERT OR IGNORE INTO database_templates (name, icon, description, schema_json) VALUES (?, ?, ?, ?)""", (tpl["name"], tpl.get("icon", "📋"), tpl.get("description", ""), __import__("json").dumps(tpl.get("schema", []))), ) @register(9, "v5.6.0: import items (dedup) + import jobs") def _migration_import_framework(conn: sqlite3.Connection) -> None: """Unified import framework (Phase 0). ``import_items`` — one row per imported page, keyed by workspace + source + external id, so re-importing the same vault is idempotent. ``import_jobs`` — background import job status/history for UI polling. """ conn.execute( """ CREATE TABLE IF NOT EXISTS import_items ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace_id INTEGER, source TEXT NOT NULL DEFAULT '', external_id TEXT NOT NULL DEFAULT '', page_id INTEGER, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(workspace_id, source, external_id) ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_import_items_lookup " "ON import_items(workspace_id, source, external_id)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS import_jobs ( id TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT '', filename TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'queued', error TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_import_jobs_created ON import_jobs(created_at)" ) @register(10, "v5.7.0: property groups, per-user views, row covers") def _migration_v57_db_advanced(conn: sqlite3.Connection) -> None: """v5.7.0 — Database Avancée (Pt. 2). ``collection_properties.group_name`` — groups properties into collapsible sections in the table header (Notion property groups). ``collection_views.created_by`` — owner of a saved view; ``NULL`` means a shared/legacy view visible to everyone, otherwise it is personal to a user. ``collection_pages.cover_url`` — per-row cover image (gallery/board cards), independent from the block-page ``pages.cover_url``. """ _pcols = columns(conn, "collection_properties") if "group_name" not in _pcols: conn.execute( "ALTER TABLE collection_properties ADD COLUMN group_name TEXT NOT NULL DEFAULT ''" ) _vcols = columns(conn, "collection_views") if "created_by" not in _vcols: conn.execute("ALTER TABLE collection_views ADD COLUMN created_by INTEGER") if "updated_at" not in _vcols: conn.execute("ALTER TABLE collection_views ADD COLUMN updated_at TIMESTAMP") _cpcols = columns(conn, "collection_pages") if "cover_url" not in _cpcols: conn.execute("ALTER TABLE collection_pages ADD COLUMN cover_url TEXT DEFAULT ''") @register(11, "v5.8.0: reminder log + user timezones") def _migration_v58_calendar_reminders(conn: sqlite3.Connection) -> None: """v5.8.0 — Calendrier & Rappels. ``reminder_log`` — dedup ledger: one row per (page, occurrence date) so a reminder fires exactly once even across restarts. ``users.timezone`` — personal IANA timezone used for "today" in calendar views and reminder firing (empty = UTC). """ conn.execute( """CREATE TABLE IF NOT EXISTS reminder_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, page_id INTEGER NOT NULL REFERENCES collection_pages(id) ON DELETE CASCADE, occurrence_date TEXT NOT NULL, fired_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(page_id, occurrence_date) )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_remlog_page ON reminder_log(page_id)" ) _ucols = columns(conn, "users") if "timezone" not in _ucols: conn.execute("ALTER TABLE users ADD COLUMN timezone TEXT NOT NULL DEFAULT ''") @register(12, "v5.8.0: enrich Meeting notes template") def _migration_v58_meeting_template(conn: sqlite3.Connection) -> None: """v5.8.0 — the seeded 'Meeting notes' database template gains Agenda and Notes text properties. Only refreshed when the row still matches the old built-in schema (user edits are never clobbered).""" import json as _json row = conn.execute( "SELECT schema_json FROM database_templates WHERE name='Meeting notes'" ).fetchone() if not row: return try: schema = _json.loads(row[0] or "[]") except (ValueError, TypeError): return names = [p.get("name") for p in schema] if "Agenda" in names or "Notes" in names: return if names != ["Title", "Date", "Attendees", "Status", "Action items"]: return # customised — leave alone idx = names.index("Action items") schema[idx:idx] = [ {"name": "Agenda", "type": "text"}, {"name": "Notes", "type": "text"}, ] conn.execute( "UPDATE database_templates SET schema_json=? WHERE name='Meeting notes'", (_json.dumps(schema),), ) conn.execute( "UPDATE database_templates SET description=? WHERE name='Meeting notes'", ("Notes de réunion avec participants, agenda, notes et actions.",), ) @register(13, "v5.11.0/v5.12.0: page lock, user typo prefs, global page templates") def _migration_v511_wiki_v512_templates(conn: sqlite3.Connection) -> None: """v5.11.0 Wiki-links + v5.12.0 Templates & verrouillage. ``pages.is_locked`` — read-only page (locker/admin can unlock). ``pages.locked_by`` — user that locked the page. ``pages.full_width`` — per-page full-width layout toggle. ``pages.font_small`` — per-page compact typography toggle. ``page_global_templates`` — user-created global page templates (blocks_json = same format as the block editor saves). """ _pcols = columns(conn, "pages") if "is_locked" not in _pcols: conn.execute("ALTER TABLE pages ADD COLUMN is_locked INTEGER NOT NULL DEFAULT 0") if "locked_by" not in _pcols: conn.execute("ALTER TABLE pages ADD COLUMN locked_by INTEGER REFERENCES users(id) ON DELETE SET NULL") if "full_width" not in _pcols: conn.execute("ALTER TABLE pages ADD COLUMN full_width INTEGER NOT NULL DEFAULT 0") if "font_small" not in _pcols: conn.execute("ALTER TABLE pages ADD COLUMN font_small INTEGER NOT NULL DEFAULT 0") conn.execute( """CREATE TABLE IF NOT EXISTS page_global_templates ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, icon TEXT NOT NULL DEFAULT '📄', description TEXT NOT NULL DEFAULT '', blocks_json TEXT NOT NULL DEFAULT '[]', created_by INTEGER REFERENCES users(id) ON DELETE SET NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_pgt_creator ON page_global_templates(created_by)" ) @register(14, "v5.13.0: fix stale LLM provider api_base values") def _migration_fix_llm_api_bases(conn: sqlite3.Connection) -> None: """Clear `api_base` values that freeze a provider URL to a wrong/default value. The Agent "Test connection" uses the per-user (or global) stored `api_base` when present, so an old/incorrect value (e.g. Mistral `…/v2`, Cohere `…/v2`, Google native `/v1beta`) keeps failing even after `PROVIDERS` is corrected. Two kinds of rows are reset to the provider default: * known-wrong legacy bases from earlier releases; * a stored base identical to the current provider default (a no-op override that would block future default changes). """ from app.services.llm_client import PROVIDERS legacy: dict[str, set[str]] = { "mistral": {"https://api.mistral.ai/v2"}, "cohere": {"https://api.cohere.com/v2", "https://api.cohere.com/v1", "https://api.cohere.ai/v2"}, "google": {"https://generativelanguage.googleapis.com/v1beta"}, "perplexity": {"https://api.perplexity.ai/v1"}, "chutes": {"https://api.chutes.ai/v1"}, "sensenova": {"https://token.sensenova.cn/v1"}, "ltx": {"https://api.ltx.io/v1"}, "memtensor": {"https://memos.memtensor.cn/api/openmem/v1"}, } for provider, (base, _) in PROVIDERS.items(): if base: legacy.setdefault(provider, set()).update({base, base.rstrip("/")}) for provider, bases in legacy.items(): variants = {b for b in bases if b} variants |= {b.rstrip("/") for b in bases if b} for table in ("user_llm_keys", "llm_config"): for value in variants: conn.execute( f"UPDATE {table} SET api_base='' WHERE provider=? AND api_base=?", (provider, value), ) @register(15, "v5.14.0: synced blocks") def _migration_synced_blocks(conn: sqlite3.Connection) -> None: """v5.14.0 — Synced blocks: a block created once, displayed & edited across multiple pages. ``synced_blocks`` — source-of-truth content for synced blocks. ``page_blocks`` — per-page reference to a synced block (so each page can independently decide to use/unsync). """ conn.execute( """CREATE TABLE IF NOT EXISTS synced_blocks ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL DEFAULT '', content TEXT NOT NULL DEFAULT '[]', created_by INTEGER REFERENCES users(id), workspace TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_synced_blocks_ws ON synced_blocks(workspace)" ) conn.execute( """CREATE TABLE IF NOT EXISTS page_synced_blocks ( id INTEGER PRIMARY KEY AUTOINCREMENT, page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, synced_block_id INTEGER NOT NULL REFERENCES synced_blocks(id) ON DELETE CASCADE, block_index INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(page_id, synced_block_id) )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_psb_page ON page_synced_blocks(page_id)" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_psb_synced ON page_synced_blocks(synced_block_id)" ) @register(16, "v6.0.0: offline sync queue") def _migration_offline_sync_queue(conn: sqlite3.Connection) -> None: """v6.0.0 — PWA offline support. ``offline_sync_queue`` persists server-side the mutations received from offline clients (``/api/v2/sync/batch``) so work is not lost and can be audited/replayed per device. """ conn.execute( """CREATE TABLE IF NOT EXISTS offline_sync_queue ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, device_id TEXT NOT NULL, type TEXT NOT NULL, -- 'page_create', 'page_update', 'page_delete', 'page_move', -- 'collection_create', 'collection_update', 'collection_delete' payload TEXT NOT NULL, client_timestamp REAL NOT NULL, server_version INTEGER DEFAULT 0, status TEXT NOT NULL DEFAULT 'pending', -- 'pending', 'syncing', 'synced', 'failed' retries INTEGER NOT NULL DEFAULT 0, error TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_syncqueue_user ON offline_sync_queue(user_id, status)" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_syncqueue_device ON offline_sync_queue(device_id, status)" ) @register(18, "v6.0.0: granular permissions (page/collection/property ACL + groups)") def _migration_v600_granular_permissions(conn: sqlite3.Connection) -> None: """v6.0.0 — Granular permissions (page-level, collection-level, property-level access control + reusable user groups). ``user_groups`` — named groups scoped to a workspace. ``group_members`` — users inside a group (N-ary join). ``page_permissions`` — explicit grants for block-editor pages (``pages`` table). user_id XOR group_id. ``collection_permissions`` — explicit grants for databases. ``property_permissions`` — explicit viewer/editor grants per property. ``permission_audit_log`` — immutable trail of every grant/revoke. ``pages.permission_type`` / ``collections.permission_type`` — access mode: 'inherit' (default, follows the workspace/collection chain) | 'restricted' | 'private' (explicit grants only). """ conn.execute( """CREATE TABLE IF NOT EXISTS user_groups ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace_id INTEGER REFERENCES workspaces(id) ON DELETE CASCADE, name TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', created_by INTEGER REFERENCES users(id) ON DELETE SET NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(workspace_id, name) )""" ) conn.execute( """CREATE TABLE IF NOT EXISTS group_members ( id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER NOT NULL REFERENCES user_groups(id) ON DELETE CASCADE, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, joined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(group_id, user_id) )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_gm_group ON group_members(group_id)") conn.execute("CREATE INDEX IF NOT EXISTS idx_gm_user ON group_members(user_id)") conn.execute( """CREATE TABLE IF NOT EXISTS page_permissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, group_id INTEGER REFERENCES user_groups(id) ON DELETE CASCADE, role TEXT NOT NULL, -- viewer | commenter | editor | owner grant_type TEXT NOT NULL DEFAULT 'explicit', -- explicit | group granted_by INTEGER REFERENCES users(id) ON DELETE SET NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, CHECK (user_id IS NOT NULL OR group_id IS NOT NULL), UNIQUE(page_id, user_id, group_id) )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_pp_page ON page_permissions(page_id, role)") conn.execute("CREATE INDEX IF NOT EXISTS idx_pp_user ON page_permissions(user_id)") conn.execute( """CREATE TABLE IF NOT EXISTS collection_permissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE, user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, group_id INTEGER REFERENCES user_groups(id) ON DELETE CASCADE, role TEXT NOT NULL, -- viewer | commenter | editor | owner grant_type TEXT NOT NULL DEFAULT 'explicit', granted_by INTEGER REFERENCES users(id) ON DELETE SET NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, CHECK (user_id IS NOT NULL OR group_id IS NOT NULL), UNIQUE(collection_id, user_id, group_id) )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_cp_collection ON collection_permissions(collection_id, role)") conn.execute( """CREATE TABLE IF NOT EXISTS property_permissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE, property_id INTEGER NOT NULL REFERENCES collection_properties(id) ON DELETE CASCADE, user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, group_id INTEGER REFERENCES user_groups(id) ON DELETE CASCADE, role TEXT NOT NULL, -- viewer | editor grant_type TEXT NOT NULL DEFAULT 'explicit', granted_by INTEGER REFERENCES users(id) ON DELETE SET NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, CHECK (user_id IS NOT NULL OR group_id IS NOT NULL), UNIQUE(collection_id, property_id, user_id, group_id) )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_propp_prop ON property_permissions(property_id, role)") conn.execute( """CREATE TABLE IF NOT EXISTS permission_audit_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, resource_type TEXT NOT NULL, -- page | collection | property | group resource_id INTEGER NOT NULL, action TEXT NOT NULL, -- grant | revoke | type_change | group_create | group_delete | member_add | member_remove target_user_id INTEGER, target_group_id INTEGER, old_role TEXT, new_role TEXT, performed_by INTEGER REFERENCES users(id) ON DELETE SET NULL, ip_address TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_perm_audit_res " "ON permission_audit_log(resource_type, resource_id, created_at)" ) for table in ("pages", "collection_pages"): cols = columns(conn, table) if "permission_type" not in cols: conn.execute( f"ALTER TABLE {table} ADD COLUMN permission_type TEXT NOT NULL DEFAULT 'inherit'" ) _ccols = columns(conn, "collections") if "permission_type" not in _ccols: conn.execute( "ALTER TABLE collections ADD COLUMN permission_type TEXT NOT NULL DEFAULT 'inherit'" ) def _add_sync_version(conn: sqlite3.Connection, table: str) -> None: """Add ``sync_version`` to ``table`` if it is not already present.""" cols = columns(conn, table) if "sync_version" not in cols: conn.execute(f"ALTER TABLE {table} ADD COLUMN sync_version INTEGER NOT NULL DEFAULT 1") @register(19, "v6.0.0: web clipper — extension devices & clips") def _migration_web_clipper(conn: sqlite3.Connection) -> None: """v6.0.0 — Web Clipper: extension browser + capture.""" conn.execute( """CREATE TABLE IF NOT EXISTS extension_devices ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, extension_name TEXT NOT NULL DEFAULT 'clipper', device_id TEXT NOT NULL, device_name TEXT DEFAULT '', token_hash TEXT NOT NULL DEFAULT '', scopes TEXT NOT NULL DEFAULT 'read,write', last_used_at TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, revoked INTEGER NOT NULL DEFAULT 0, UNIQUE(user_id, extension_name, device_id) )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_ext_devices_user ON extension_devices(user_id, revoked)" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_ext_devices_device ON extension_devices(device_id)" ) conn.execute( """CREATE TABLE IF NOT EXISTS extension_clips ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, device_id TEXT NOT NULL DEFAULT '', clip_type TEXT NOT NULL DEFAULT 'article', source_url TEXT NOT NULL DEFAULT '', target_page_id INTEGER REFERENCES pages(id) ON DELETE SET NULL, target_workspace_id INTEGER REFERENCES workspaces(id) ON DELETE SET NULL, title TEXT DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_ext_clips_user ON extension_clips(user_id, created_at)" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_ext_clips_device ON extension_clips(device_id)" ) @register(20, "v6.3.0: api v2 — scopes, expires_at, audit, webhooks, idempotency") def _migration_v630_api_v2(conn: sqlite3.Connection) -> None: """v6.3.0 — API publique complète v2. ``api_tokens`` — adds ``scopes`` + ``expires_at`` (idempotent ALTER). ``webhook_deliveries`` — delivery log for outbound webhooks (CRUD simple phase 1). ``api_audit_log`` — immutable audit trail for v2 mutations. ``idempotency_keys`` — Idempotency-Key support for POST creations. """ # api_tokens extra columns _cols = columns(conn, "api_tokens") if "scopes" not in _cols: conn.execute("ALTER TABLE api_tokens ADD COLUMN scopes TEXT NOT NULL DEFAULT 'read,write'") if "expires_at" not in _cols: conn.execute("ALTER TABLE api_tokens ADD COLUMN expires_at TIMESTAMP") # Backfill existing tokens without scopes try: conn.execute("UPDATE api_tokens SET scopes='read,write' WHERE scopes='' OR scopes IS NULL") except Exception: logger.exception("_migration_v630_api_v2") conn.execute( """CREATE TABLE IF NOT EXISTS api_audit_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, token_id INTEGER REFERENCES api_tokens(id) ON DELETE SET NULL, action TEXT NOT NULL, resource_type TEXT NOT NULL DEFAULT '', resource_id TEXT NOT NULL DEFAULT '', ip_address TEXT NOT NULL DEFAULT '', detail TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_api_audit_user ON api_audit_log(user_id, created_at)") conn.execute("CREATE INDEX IF NOT EXISTS idx_api_audit_resource ON api_audit_log(resource_type, resource_id)") conn.execute( """CREATE TABLE IF NOT EXISTS webhook_deliveries ( id INTEGER PRIMARY KEY AUTOINCREMENT, webhook_id INTEGER NOT NULL REFERENCES webhook_subscriptions(id) ON DELETE CASCADE, status TEXT NOT NULL DEFAULT 'pending', http_code INTEGER, error TEXT NOT NULL DEFAULT '', duration_ms INTEGER NOT NULL DEFAULT 0, attempt INTEGER NOT NULL DEFAULT 0, payload TEXT NOT NULL DEFAULT '{}', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_wd_webhook ON webhook_deliveries(webhook_id, created_at)") conn.execute( """CREATE TABLE IF NOT EXISTS idempotency_keys ( key TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, response_json TEXT NOT NULL DEFAULT '{}', status_code INTEGER NOT NULL DEFAULT 200, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) conn.execute("CREATE INDEX IF NOT EXISTS idx_idemp_user ON idempotency_keys(user_id, created_at)") @register(21, "v6.4.0: webhooks prod — event, retry ledger") def _migration_v640_webhooks_prod(conn: sqlite3.Connection) -> None: """v6.4.0 — Webhooks v2 production. ``webhook_deliveries`` gains ``event`` (which event was delivered) and ``next_retry_at`` (epoch seconds; picked up by the retry scheduler). New statuses: ``retrying`` (a later attempt is scheduled) and ``superseded`` (a retry row replaced this attempt). """ _cols = columns(conn, "webhook_deliveries") if "event" not in _cols: conn.execute("ALTER TABLE webhook_deliveries ADD COLUMN event TEXT NOT NULL DEFAULT ''") if "next_retry_at" not in _cols: conn.execute("ALTER TABLE webhook_deliveries ADD COLUMN next_retry_at REAL") conn.execute( "CREATE INDEX IF NOT EXISTS idx_wd_retry ON webhook_deliveries(status, next_retry_at)" ) @register(17, "v6.0.0: sync_version columns") def _migration_sync_version_columns(conn: sqlite3.Connection) -> None: """v6.0.0 — optimistic-concurrency version counters for offline sync. Every write on a page / collection increments its ``sync_version`` so a reconnecting client can detect edit-edit conflicts via version mismatch. A ``BEFORE UPDATE`` trigger performs the increment automatically on every write path (no need to patch dozens of ``UPDATE`` call-sites). """ for table in ("pages", "collection_pages", "collections"): _add_sync_version(conn, table) # AFTER UPDATE + inner UPDATE: bumps sync_version on every write path. # `recursive_triggers` is OFF by default, so the inner UPDATE never # re-fires the trigger (no infinite loop), incl. the page FTS triggers. conn.execute( """CREATE TRIGGER IF NOT EXISTS pages_sync_version_bu AFTER UPDATE ON pages FOR EACH ROW BEGIN UPDATE pages SET sync_version = sync_version + 1 WHERE id = NEW.id; END""" ) conn.execute( """CREATE TRIGGER IF NOT EXISTS collection_pages_sync_version_bu AFTER UPDATE ON collection_pages FOR EACH ROW BEGIN UPDATE collection_pages SET sync_version = sync_version + 1 WHERE id = NEW.id; END""" ) conn.execute( """CREATE TRIGGER IF NOT EXISTS collections_sync_version_bu AFTER UPDATE ON collections FOR EACH ROW BEGIN UPDATE collections SET sync_version = sync_version + 1 WHERE id = NEW.id; END""" ) @register(22, "v6.5.0: database row content pages") def _migration_row_content_pages(conn: sqlite3.Connection) -> None: """v6.5.0 — Synced blocks production: content for database rows. A database row (``collection_pages``) gains a shadow ``pages`` row (``pages.collection_row_id``) that carries the Notion-style block content of the row: the full page editor, synced blocks, versions and realtime all work on it unchanged. ``ON DELETE CASCADE``: deleting a database row deletes its content page (and ``page_synced_blocks`` cascades from ``pages``). """ cols = columns(conn, "pages") if "collection_row_id" not in cols: conn.execute( "ALTER TABLE pages ADD COLUMN collection_row_id INTEGER " "REFERENCES collection_pages(id) ON DELETE CASCADE" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_pages_row " "ON pages(collection_row_id) WHERE collection_row_id IS NOT NULL" ) @register(24, "v6.8.0: Sites & public Forms") def _migration_sites_forms(conn: sqlite3.Connection) -> None: """v6.8.0 — Notion Sites + Forms publics (voir docs/V68_Sites_Forms.md). ``sites`` — mini-site multi-pages (slug, root_page, thème, domaine custom, password hash, expiry, noindex). ``site_pages`` — arbre public ordonné (site_id, page_id, position). ``site_views`` — compteur de vues jour/site (upsert, pas d'IP brute). ``form_responses`` — log des soumissions anonymes (ip_hash jour, pas d'IP). ``collections.form_config_json`` — config du formulaire public par DB. """ conn.execute( """ CREATE TABLE IF NOT EXISTS sites ( id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, root_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, title TEXT NOT NULL DEFAULT '', theme TEXT NOT NULL DEFAULT 'dark', custom_domain TEXT UNIQUE, password_hash TEXT DEFAULT '', expires_at TIMESTAMP, noindex INTEGER NOT NULL DEFAULT 0, analytics_id TEXT DEFAULT '', created_by INTEGER REFERENCES users(id), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS site_pages ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER NOT NULL REFERENCES sites(id) ON DELETE CASCADE, page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, position INTEGER NOT NULL DEFAULT 0, UNIQUE(site_id, page_id) ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_site_pages_site ON site_pages(site_id, position)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS site_views ( site_id INTEGER NOT NULL REFERENCES sites(id) ON DELETE CASCADE, day TEXT NOT NULL, views INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (site_id, day) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS form_responses ( id INTEGER PRIMARY KEY AUTOINCREMENT, collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE, row_id INTEGER REFERENCES collection_pages(id) ON DELETE SET NULL, ip_hash TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_form_responses_col ON form_responses(collection_id, created_at)" ) cols = columns(conn, "collections") if "form_config_json" not in cols: conn.execute( "ALTER TABLE collections ADD COLUMN form_config_json TEXT NOT NULL DEFAULT '{}'" ) @register(25, "v6.9.0: semantic search + Ask AI") def _migration_semantic_search(conn: sqlite3.Connection) -> None: """v6.9.0 — hybrid lexical+vector search and RAG Ask AI (docs/V69_* md). ``semantic_embeddings`` — hashed-TF chunk vectors (no external dep): keyed by (resource_type, resource_id, chunk_id) so both ``page`` and ``collection`` resources are indexed. (Design doc names a ``page_embeddings`` table; the generic key covers collections too.) ``semantic_index_state`` — last indexed timestamp per resource for the incremental background job. ``pages.search_excluded`` — opt-out flag respected by indexer + search. """ conn.execute( """ CREATE TABLE IF NOT EXISTS semantic_embeddings ( resource_type TEXT NOT NULL, resource_id INTEGER NOT NULL, chunk_id INTEGER NOT NULL, chunk_text TEXT NOT NULL DEFAULT '', embedding BLOB NOT NULL, model TEXT NOT NULL DEFAULT 'hash-256', updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (resource_type, resource_id, chunk_id) ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_sem_emb_res " "ON semantic_embeddings(resource_type, resource_id)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS semantic_index_state ( resource_type TEXT NOT NULL, resource_id INTEGER NOT NULL, indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (resource_type, resource_id) ) """ ) cols = columns(conn, "pages") if "search_excluded" not in cols: conn.execute( "ALTER TABLE pages ADD COLUMN search_excluded INTEGER NOT NULL DEFAULT 0" ) @register(26, "v7.0.0: automations v2 (steps) + workers") def _migration_automations_v2_workers(conn: sqlite3.Connection) -> None: """v7.0.0 — multi-step automations + sandboxed workers (docs/V70_* md). ``automation_steps`` — ordered trigger/condition/delay/action chain per automation. Legacy single trigger+actions columns keep working (engine falls back when an automation has no steps). ``automations.trigger_mode`` — ``any`` (default) or ``all`` (every trigger event must arrive within a 5-minute window). ``workers`` / ``worker_runs`` — custom Python snippets (cron/manual), shareable across the team, with execution logs + daily budget. ``collection_properties.button_automation_id`` — native DB button cells. """ conn.execute( """ CREATE TABLE IF NOT EXISTS automation_steps ( id INTEGER PRIMARY KEY AUTOINCREMENT, automation_id INTEGER NOT NULL REFERENCES automations(id) ON DELETE CASCADE, kind TEXT NOT NULL, position INTEGER NOT NULL DEFAULT 0, config_json TEXT NOT NULL DEFAULT '{}', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_asteps_auto " "ON automation_steps(automation_id, position)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS workers ( id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, workspace_id INTEGER REFERENCES workspaces(id) ON DELETE CASCADE, name TEXT NOT NULL DEFAULT '', code_py TEXT NOT NULL DEFAULT '', schedule_cron TEXT DEFAULT '', shared INTEGER NOT NULL DEFAULT 0, daily_budget_s INTEGER NOT NULL DEFAULT 60, created_by INTEGER REFERENCES users(id), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS worker_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, worker_id INTEGER NOT NULL REFERENCES workers(id) ON DELETE CASCADE, status TEXT NOT NULL, logs TEXT NOT NULL DEFAULT '', duration_ms INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_worker_runs_worker " "ON worker_runs(worker_id, created_at)" ) auto_cols = columns(conn, "automations") if "trigger_mode" not in auto_cols: conn.execute( "ALTER TABLE automations ADD COLUMN trigger_mode TEXT NOT NULL DEFAULT 'any'" ) prop_cols = columns(conn, "collection_properties") if "button_automation_id" not in prop_cols: conn.execute( "ALTER TABLE collection_properties ADD COLUMN button_automation_id " "INTEGER REFERENCES automations(id) ON DELETE SET NULL" ) @register(27, "v7.1.0: calendar sync + meeting transcripts") def _migration_calendar_meetings(conn: sqlite3.Connection) -> None: """v7.1.0 — external calendar sync + AI meeting notes (docs/V71_* md). ``calendar_links`` — per-user link between a collection and an external calendar (google REST / generic caldav), tokens Fernet-encrypted. ``meeting_transcripts`` — uploaded audio + transcript + AI summary per page. ``collection_pages.external_event_id`` — remote event id for push/pull matching and conflict detection. """ conn.execute( """ CREATE TABLE IF NOT EXISTS calendar_links ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, provider TEXT NOT NULL, tokens_enc TEXT NOT NULL DEFAULT '', calendar_id TEXT NOT NULL DEFAULT 'primary', collection_id INTEGER REFERENCES collections(id) ON DELETE CASCADE, date_property TEXT DEFAULT '', sync_token TEXT DEFAULT '', last_sync TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(user_id, provider, calendar_id) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS meeting_transcripts ( id INTEGER PRIMARY KEY AUTOINCREMENT, page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, audio_path TEXT NOT NULL DEFAULT '', transcript TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', language TEXT NOT NULL DEFAULT 'fr', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_meeting_transcripts_page " "ON meeting_transcripts(page_id)" ) cols = columns(conn, "collection_pages") if "external_event_id" not in cols: conn.execute( "ALTER TABLE collection_pages ADD COLUMN external_event_id TEXT DEFAULT ''" ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_cp_external " "ON collection_pages(collection_id, external_event_id)" ) @register(28, "v7.2.0: SCIM + 2FA + audit UI + agent governance") def _migration_enterprise_admin(conn: sqlite3.Connection) -> None: """v7.2.0 — enterprise admin (docs/V72_* md). ``scim_tokens`` — Bearer tokens for SCIM provisioning (admin-managed). ``domain_claims`` — DNS/well-known verified domains + SSO enforcement. ``webauthn_credentials`` — passkeys (credential_id, COSE public key). ``agent_policies`` — per-workspace tool scope + approval gate. ``agent_approvals`` — approval queue for gated write actions. ``users.totp_secret_enc`` / ``totp_backup_hashes`` — TOTP 2FA. (``users.is_active`` already exists — used by SCIM suspend.) """ conn.execute( """ CREATE TABLE IF NOT EXISTS scim_tokens ( id INTEGER PRIMARY KEY AUTOINCREMENT, token_hash TEXT NOT NULL UNIQUE, name TEXT NOT NULL DEFAULT '', created_by INTEGER REFERENCES users(id), revoked INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS domain_claims ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT NOT NULL UNIQUE, txt_token TEXT NOT NULL DEFAULT '', verified INTEGER NOT NULL DEFAULT 0, auto_join_role TEXT NOT NULL DEFAULT 'viewer', enforce_sso INTEGER NOT NULL DEFAULT 0, workspace_id INTEGER REFERENCES workspaces(id) ON DELETE SET NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS webauthn_credentials ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, credential_id TEXT NOT NULL UNIQUE, public_key TEXT NOT NULL DEFAULT '', sign_count INTEGER NOT NULL DEFAULT 0, name TEXT NOT NULL DEFAULT '', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_webauthn_user ON webauthn_credentials(user_id)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS agent_policies ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace_id INTEGER REFERENCES workspaces(id) ON DELETE CASCADE, allowed_tools_json TEXT, max_steps INTEGER NOT NULL DEFAULT 12, require_approval INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(workspace_id) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS agent_approvals ( id INTEGER PRIMARY KEY AUTOINCREMENT, conversation_id INTEGER NOT NULL DEFAULT 0, tool TEXT NOT NULL DEFAULT '', args_json TEXT NOT NULL DEFAULT '{}', status TEXT NOT NULL DEFAULT 'pending', requester_id INTEGER REFERENCES users(id), approver_id INTEGER REFERENCES users(id), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_agent_approvals_status " "ON agent_approvals(status, created_at)" ) user_cols = columns(conn, "users") if "totp_secret_enc" not in user_cols: conn.execute("ALTER TABLE users ADD COLUMN totp_secret_enc TEXT DEFAULT ''") if "totp_backup_hashes" not in user_cols: conn.execute("ALTER TABLE users ADD COLUMN totp_backup_hashes TEXT DEFAULT '[]'") @register(29, "v7.3.0: teamspaces + verified pages + collab polish") def _migration_wiki_teamspaces(conn: sqlite3.Connection) -> None: """v7.3.0 — teamspaces, verified pages, collab polish (docs/V73_*.md). ``teamspaces`` / ``teamspace_members`` — namespaces for pages + databases; ``private=1`` hides a teamspace from non-members (404, like restricted collections). ``page_verifications`` — ✅ badge with expiry. ``comment_reactions`` / ``page_follows`` — collab polish. ``guest_shares`` — account-less page access via ``/g/``. ``page_views`` — daily counters, same pattern as ``site_views``. """ conn.execute( """ CREATE TABLE IF NOT EXISTS teamspaces ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace_id INTEGER NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, name TEXT NOT NULL, description TEXT DEFAULT '', private INTEGER NOT NULL DEFAULT 0, created_by INTEGER REFERENCES users(id), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(workspace_id, name) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS teamspace_members ( id INTEGER PRIMARY KEY AUTOINCREMENT, teamspace_id INTEGER NOT NULL REFERENCES teamspaces(id) ON DELETE CASCADE, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, role TEXT NOT NULL DEFAULT 'editor', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(teamspace_id, user_id) ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_teamspace_members_user " "ON teamspace_members(user_id)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS page_verifications ( page_id INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE, verified_by INTEGER REFERENCES users(id), note TEXT NOT NULL DEFAULT '', verified_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, expires_at TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_page_verifications_expiry " "ON page_verifications(expires_at)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS comment_reactions ( id INTEGER PRIMARY KEY AUTOINCREMENT, comment_id INTEGER NOT NULL REFERENCES comments(id) ON DELETE CASCADE, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, emoji TEXT NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(comment_id, user_id, emoji) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS page_follows ( page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (page_id, user_id) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS guest_shares ( id INTEGER PRIMARY KEY AUTOINCREMENT, page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, email TEXT NOT NULL DEFAULT '', token TEXT NOT NULL UNIQUE, role TEXT NOT NULL DEFAULT 'viewer', created_by INTEGER REFERENCES users(id), expires_at TIMESTAMP, revoked INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS page_views ( page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, day TEXT NOT NULL, views INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (page_id, day) ) """ ) for table in ("pages", "collections"): cols = columns(conn, table) if "teamspace_id" not in cols: conn.execute(f"ALTER TABLE {table} ADD COLUMN teamspace_id INTEGER") @register(23, "v6.7.0: SSO/SAML enterprise auth") def _migration_sso_enterprise_auth(conn: sqlite3.Connection) -> None: """v6.7.0 — SSO/SAML 2.0 + OIDC enterprise authentication. ``sso_config`` — single active SSO provider (SAML or OIDC), managed from Settings → Admin → SSO / Enterprise. Secrets (``client_secret``, SP private key) are encrypted at rest by ``app.services.sso_provisioning``. ``sso_login_history`` — audit trail of every SSO login attempt (successes AND rejections — signature failure, replay, no local account…). ``sso_requests`` — single-use anti-replay store: AuthnRequest ids and OIDC states, CSRF relay tokens, PKCE verifiers and the post-login redirect target. One row is consumed by exactly one callback. """ conn.execute( """ CREATE TABLE IF NOT EXISTS sso_config ( id INTEGER PRIMARY KEY AUTOINCREMENT, workspace_id INTEGER REFERENCES workspaces(id) ON DELETE CASCADE, provider_type TEXT NOT NULL DEFAULT 'saml', name TEXT NOT NULL DEFAULT 'Company SSO', entity_id TEXT NOT NULL DEFAULT '', sso_url TEXT NOT NULL DEFAULT '', slo_url TEXT DEFAULT '', x509_certificate TEXT NOT NULL DEFAULT '', issuer_url TEXT DEFAULT '', client_id TEXT DEFAULT '', client_secret TEXT DEFAULT '', scope TEXT DEFAULT 'openid profile email', attribute_mapping TEXT NOT NULL DEFAULT '{}', groups_mapping TEXT NOT NULL DEFAULT '[]', auto_provision INTEGER NOT NULL DEFAULT 1, sso_only INTEGER NOT NULL DEFAULT 0, sign_requests INTEGER NOT NULL DEFAULT 0, default_workspace_id INTEGER REFERENCES workspaces(id) ON DELETE SET NULL, sp_private_key TEXT DEFAULT '', sp_certificate TEXT DEFAULT '', active INTEGER NOT NULL DEFAULT 1, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, created_by INTEGER REFERENCES users(id) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS sso_login_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, provider_type TEXT NOT NULL, provider_name TEXT NOT NULL DEFAULT 'SSO', sso_identifier TEXT, ip_address TEXT DEFAULT '', user_agent TEXT DEFAULT '', success INTEGER NOT NULL DEFAULT 0, error_message TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_sso_history_user " "ON sso_login_history(user_id, created_at)" ) conn.execute( """ CREATE TABLE IF NOT EXISTS sso_requests ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, relay_state TEXT NOT NULL DEFAULT '', code_verifier TEXT NOT NULL DEFAULT '', next_path TEXT NOT NULL DEFAULT '/workspaces', used INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """ ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_sso_requests_created ON sso_requests(created_at)" ) @register(30, "v7.47.0: My Tasks — mapping des propriétés de base de tâches") def _migration_my_tasks_mapping(conn: sqlite3.Connection) -> None: """My Tasks façon Notion : une base ne diffuse ses tâches qu'après conversion explicite, et les trois propriétés requises sont **nommées** (pas devinées d'après leur type). ``collections.is_task`` existait déjà mais ne mémorisait rien : My Tasks devait deviner « la colonne personne = Assigné à » et n'avait aucun moyen de distinguer deux bases connectées. Trois colonnes nullable REFERENCES fixent le mapping ; suppression de la propriété → `SET NULL`, et la base redevient « non configurée » au lieu de pointer sur du vide. """ cols = columns(conn, "collections") for name, target in ( ("task_assignee_prop", "collection_properties"), ("task_status_prop", "collection_properties"), ("task_due_prop", "collection_properties"), ): if name in cols: continue conn.execute( f"ALTER TABLE collections ADD COLUMN {name} INTEGER " f"REFERENCES {target}(id) ON DELETE SET NULL" ) # Index de lecture : « les bases de tâches de cet utilisateur ». conn.execute( "CREATE INDEX IF NOT EXISTS idx_collections_task " "ON collections(is_task, workspace_id)" ) @register(35, "v7.58.0: registre des plugins (on/off à effet réel)") def _migration_plugins(conn: sqlite3.Connection) -> None: """Catalogue de l'instance : 3 modules câblés (routes/UI/outils réels).""" conn.execute( """CREATE TABLE IF NOT EXISTS plugins ( slug TEXT PRIMARY KEY, name TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', enabled INTEGER NOT NULL DEFAULT 1 )""" ) conn.executemany( "INSERT OR IGNORE INTO plugins (slug, name, description, enabled) VALUES (?,?,?,1)", [ ("web-tools", "Outils web de l'agent", "Retire web_search et fetch_url du registre d'outils de l'agent"), ("web-clipper", "Web Clipper", "Coupe la page /extensions et l'API /api/v2/web-clipper/*"), ("automations", "Automations", "Coupe /workspace/automations* et le scheduler en arrière-plan"), ], ) @register(34, "v7.57.0: connecteurs — kind/auth/tools_json (Discord, Telegram, MCP)") def _migration_connector_kinds(conn: sqlite3.Connection) -> None: """Discord (auth « Bot »), Telegram (jeton dans l'URL via `{secret}`) et serveurs MCP (kind='mcp', cache d'outils) réutilisent la même table.""" cols = columns(conn, "agent_connectors") if "kind" not in cols: conn.execute("ALTER TABLE agent_connectors ADD COLUMN kind TEXT NOT NULL DEFAULT 'custom'") if "auth" not in cols: conn.execute("ALTER TABLE agent_connectors ADD COLUMN auth TEXT NOT NULL DEFAULT 'bearer'") if "tools_json" not in cols: conn.execute("ALTER TABLE agent_connectors ADD COLUMN tools_json TEXT NOT NULL DEFAULT '[]'") @register(33, "v7.56.0: tokens OAuth des connecteurs (Google / M365)") def _migration_connector_tokens(conn: sqlite3.Connection) -> None: """Tokens OAuth par (kind, utilisateur), chiffrés Fernet côté service.""" conn.execute( """CREATE TABLE IF NOT EXISTS connector_tokens ( kind TEXT NOT NULL, user_id INTEGER NOT NULL, tokens_enc TEXT NOT NULL DEFAULT '', updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (kind, user_id) )""" ) @register(32, "v7.55.0: connecteurs de l'agent (socle)") def _migration_agent_connectors(conn: sqlite3.Connection) -> None: """Catalogue de connecteurs : 3 natifs servis à la volée + personnels ici. Colonne ``url`` plate (pas de ``config_json``) : les scopes/OAuth des phases 6-7 ajouteront leur propre colonne le moment venu. """ conn.execute( """CREATE TABLE IF NOT EXISTS agent_connectors ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, url TEXT NOT NULL, secret_encrypted TEXT NOT NULL DEFAULT '', enabled INTEGER NOT NULL DEFAULT 1, status TEXT NOT NULL DEFAULT 'unknown', detail TEXT NOT NULL DEFAULT '', created_by INTEGER )""" ) @register(31, "v7.54.0: agent memory (mémoire de l'agent)") def _migration_agent_memory(conn: sqlite3.Connection) -> None: """Toggle par conversation + table des résumés mémorisés. Une ligne résumé par conversation (upsert) — voir `app/services/agent_memory.py`. """ if "memory_enabled" not in columns(conn, "agent_conversations"): conn.execute( "ALTER TABLE agent_conversations " "ADD COLUMN memory_enabled INTEGER NOT NULL DEFAULT 1" ) conn.execute( """CREATE TABLE IF NOT EXISTS agent_memory ( conversation_id INTEGER PRIMARY KEY REFERENCES agent_conversations(id) ON DELETE CASCADE, kind TEXT NOT NULL DEFAULT 'summary', content TEXT NOT NULL DEFAULT '', updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )""" ) @register(36, "v7.59.1: mistral — modèles par défaut hors plan basique") def _migration_mistral_default_model(conn: sqlite3.Connection) -> None: """mistral-large-latest / pixtral-large-latest renvoient 403 « tier_not_allowed » sur les plans d'abonnement basiques : la clé est valide mais le test de connexion et l'agent échouaient. Reset du default_model stocké → vide = nouveau défaut du provider (small-latest, servi par tous les plans). """ blocked = ("mistral-large-latest", "pixtral-large-latest") conn.execute( "UPDATE user_llm_keys SET default_model='' " "WHERE provider='mistral' AND default_model IN (?,?)", blocked, ) conn.execute( "UPDATE llm_config SET model='mistral-small-latest' " "WHERE provider='mistral' AND model IN (?,?)", blocked, )