Files
flowdeck/app/migrations.py
T
bruno 9dfc38706c
FlowDeck CI / lint (push) Successful in 50s
FlowDeck CI / test (push) Successful in 5m41s
FlowDeck CI / docker (push) Successful in 45s
feat(wiki,templates): v5.11.0 wiki-links & mentions + v5.12.0 templates & page lock (release 5.12.0)
v5.11.0 Wiki-links & mentions de page :
- tokens [[fdpage:ID]] / [[fddate:ISO]] dans le texte des blocs,
  service app/services/wiki_links.py (labels, rendu HTML, extraction)
- taper [[ ouvre le picker de pages (recherche floue, clavier) ;
  le menu @ gagne les sections Pages et Date (today/tomorrow/YYYY-MM-DD)
- chips atomiques contenteditable=false relues en tokens par gtTok()
  (autosave/drag/undo preservent les liens) ; renommage propage via
  GET /board/api/wiki/titles ; backlinks reconnaissent les tokens ;
  page publique rend les chips (echopee)

v5.12.0 Templates & verrouillage :
- template picker global sur + New page : 5 built-in
  (app/services/block_templates.py) + templates perso
  (table page_global_templates, migration 13)
- POST /board/api/page-templates (save current page) + /{id}/use
  (instantiate, id 0 = built-in par cle)
- page lock : POST /api/pages/{id}/lock, garde _ensure_page_editable
  -> 423 en ecriture pour les non-privileged, deblocage par
  locked_by ou admin seulement (403 sinon), banniere + read-only UI
- full-width / small text par page (pages.full_width/font_small,
  POST /api/pages/{id}/options, classes CSS)
- migration 13 : is_locked, locked_by, full_width, font_small,
  page_global_templates

Tests : tests/test_v511_v512_wiki_templates.py (15) ; suite complete
538 verte ; ruff OK ; node --check des templates JS OK.
2026-09-14 06:38:09 -04:00

550 lines
22 KiB
Python

"""FlowDeck — versioned schema migrations (lightweight, no Alembic).
This replaces the previous "ad-hoc" approach where every new schema change was
appended directly to `app/db.py::init_db()` with no tracking. A `schema_version`
table now records the highest applied migration; the full baseline schema
(created idempotently by `init_db`) is treated as version 1, and any incremental
change is expressed as an ordered, versioned step below and applied exactly once.
Each migration function receives a raw ``sqlite3.Connection`` (WAL + foreign keys
already enabled) and must be written idempotently (``IF NOT EXISTS`` / guarded
``ALTER TABLE``) so it is safe even if partially re-run.
"""
from __future__ import annotations
import logging
import sqlite3
from typing import Callable
logger = logging.getLogger(__name__)
# The full baseline schema created by `app.db::init_db()` is "version 1".
BASELINE_VERSION = 1
# (version, name, apply_fn). Kept sorted by version at registration time.
MIGRATIONS: list[tuple[int, str, Callable[[sqlite3.Connection], None]]] = []
def register(version: int, name: str) -> Callable:
"""Decorator registering a migration in the ordered registry."""
if any(v == version for v, _, _ in MIGRATIONS):
raise ValueError(f"Duplicate migration version {version}")
def decorator(fn: Callable[[sqlite3.Connection], None]):
MIGRATIONS.append((version, name, fn))
MIGRATIONS.sort(key=lambda item: item[0])
return fn
return decorator
def _ensure_table(conn: sqlite3.Connection) -> None:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_version (
version INTEGER PRIMARY KEY,
name TEXT NOT NULL,
applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
def current_version(conn: sqlite3.Connection) -> int:
_ensure_table(conn)
row = conn.execute(
"SELECT COALESCE(MAX(version), 0) AS v FROM schema_version"
).fetchone()
return int(row[0])
def fts5_available() -> bool:
"""True when the bundled SQLite ships the FTS5 extension."""
probe = sqlite3.connect(":memory:")
try:
probe.execute("CREATE VIRTUAL TABLE _fts5_probe USING fts5(x)")
return True
except sqlite3.OperationalError:
return False
finally:
probe.close()
def apply_migrations(conn: sqlite3.Connection) -> int:
"""Seal the baseline schema (version 1) and apply pending migrations.
Returns the resulting schema version.
"""
_ensure_table(conn)
applied = current_version(conn)
if applied < BASELINE_VERSION:
# The pre-existing schema (already created by init_db) is our baseline.
conn.execute(
"INSERT OR IGNORE INTO schema_version (version, name) VALUES (?, ?)",
(BASELINE_VERSION, "baseline"),
)
conn.commit()
applied = BASELINE_VERSION
for version, name, fn in MIGRATIONS:
if version <= applied:
continue
fn(conn)
conn.execute(
"INSERT INTO schema_version (version, name) VALUES (?, ?)",
(version, name),
)
conn.commit()
applied = version
logger.info("Applied migration %d: %s", version, name)
return applied
# ═══════════════════════════════════════════════════════════════════════════
# Migrations
# ═══════════════════════════════════════════════════════════════════════════
@register(2, "missing indexes")
def _migration_missing_indexes(conn: sqlite3.Connection) -> None:
"""Add the indexes flagged in the roadmap (fast lookups by email, forge user)."""
for ddl in (
"CREATE INDEX IF NOT EXISTS idx_users_email ON users(email)",
"CREATE INDEX IF NOT EXISTS idx_user_oauth_tokens_user ON user_oauth_tokens(user_id, provider)",
"CREATE INDEX IF NOT EXISTS idx_collections_workspace ON collections(workspace_id)",
"CREATE INDEX IF NOT EXISTS idx_pages_workspace ON pages(workspace_id)",
"CREATE INDEX IF NOT EXISTS idx_pages_deleted ON pages(deleted_at)",
):
conn.execute(ddl)
@register(3, "full-text search (FTS5)")
def _migration_fts5(conn: sqlite3.Connection) -> None:
"""Create a full-text index over pages (title + content) for the command palette.
Kept in sync via row-level triggers on the ``pages`` table so page
insert/update/delete are reflected immediately. Skips gracefully if the
bundled SQLite lacks FTS5 (search then falls back to LIKE).
"""
if not fts5_available():
logger.warning("FTS5 unavailable — skipping full-text index (LIKE fallback active)")
return
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(title, body)")
conn.execute(
"""
CREATE TRIGGER IF NOT EXISTS pages_fts_ai AFTER INSERT ON pages BEGIN
INSERT INTO pages_fts(rowid, title, body)
VALUES (new.id, COALESCE(new.title, ''), COALESCE(new.content, ''));
END
"""
)
# `pages_fts` is a standalone FTS5 table (it stores its own content), so deletes
# use a plain DELETE by rowid (NOT the special 'delete' insert that only applies
# to external-content/contentless FTS5 tables).
conn.execute(
"""
CREATE TRIGGER IF NOT EXISTS pages_fts_ad AFTER DELETE ON pages BEGIN
DELETE FROM pages_fts WHERE rowid = old.id;
END
"""
)
conn.execute(
"""
CREATE TRIGGER IF NOT EXISTS pages_fts_au AFTER UPDATE ON pages BEGIN
DELETE FROM pages_fts WHERE rowid = old.id;
INSERT INTO pages_fts(rowid, title, body)
VALUES (new.id, COALESCE(new.title, ''), COALESCE(new.content, ''));
END
"""
)
# Backfill the index from any rows that already exist.
conn.execute(
"""
INSERT INTO pages_fts(rowid, title, body)
SELECT id, COALESCE(title, ''), COALESCE(content, '') FROM pages
WHERE deleted_at IS NULL
"""
)
@register(5, "automations (v5.1.0 rules engine)")
def _migration_automations(conn: sqlite3.Connection) -> None:
"""v5.1.0: database automations — if-this-then-that rule engine (trigger +
condition + action) and clickable buttons that trigger actions.
``automations`` — the rules (event/cron/button trigger, optional
condition JSON, actions JSON, run counters).
``automation_runs`` — execution history for auditing and the Settings UI.
"""
conn.execute(
"""
CREATE TABLE IF NOT EXISTS automations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
workspace TEXT NOT NULL DEFAULT '',
name TEXT NOT NULL,
trigger_type TEXT NOT NULL DEFAULT 'event', -- event | cron | button
event TEXT NOT NULL DEFAULT 'page.created', -- for trigger_type='event'
cron_expression TEXT NOT NULL DEFAULT '', -- for trigger_type='cron'
collection_id INTEGER, -- optional scope (event triggers)
condition_json TEXT NOT NULL DEFAULT '[]', -- list of condition clauses
actions_json TEXT NOT NULL DEFAULT '[]', -- list of action descriptors
enabled BOOLEAN NOT NULL DEFAULT 1,
created_by INTEGER,
last_run_at TIMESTAMP,
run_count INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_automations_trigger ON automations(trigger_type, event, enabled)"
)
conn.execute(
"""
CREATE TABLE IF NOT EXISTS automation_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
automation_id INTEGER NOT NULL REFERENCES automations(id) ON DELETE CASCADE,
trigger_source TEXT NOT NULL DEFAULT 'event',
status TEXT NOT NULL DEFAULT 'fired', -- fired | skipped | error
detail TEXT NOT NULL DEFAULT '',
collection_id INTEGER,
page_id INTEGER,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_automation_runs_auto ON automation_runs(automation_id, created_at)"
)
@register(6, "v5.2.0: api tokens, user sessions, projects")
def _migration_v520_security_projects(conn: sqlite3.Connection) -> None:
"""v5.2.0 (Security & Forge): per-user API tokens, revocable sessions and
the forge-agnostic ``projects`` table.
``api_tokens`` — per-user bearer tokens (sha256-stored), revocable,
powering the public API (/api/v1) and Settings UI.
``user_sessions`` — one row per signed session cookie; revocation here
instantly kills the corresponding cookie.
``projects`` — normalized project list across forges (builtin/gitea/
github) + last sync timestamp for the periodic cron.
"""
_pcols = {r[1] for r in conn.execute("PRAGMA table_info(api_tokens)").fetchall()}
if "id" not in _pcols:
conn.execute(
"""
CREATE TABLE api_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name TEXT NOT NULL DEFAULT 'API token',
token_hash TEXT NOT NULL UNIQUE,
token_prefix TEXT NOT NULL DEFAULT '',
last_used_at TIMESTAMP,
revoked INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_api_tokens_user ON api_tokens(user_id, revoked)"
)
_scols = {r[1] for r in conn.execute("PRAGMA table_info(user_sessions)").fetchall()}
if "id" not in _scols:
conn.execute(
"""
CREATE TABLE user_sessions (
id TEXT PRIMARY KEY, -- session id (cookie payload)
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
ip_address TEXT DEFAULT '',
user_agent TEXT DEFAULT '',
last_seen_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
revoked INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_user_sessions_user ON user_sessions(user_id, revoked)"
)
_projcols = {r[1] for r in conn.execute("PRAGMA table_info(projects)").fetchall()}
if "id" not in _projcols:
conn.execute(
"""
CREATE TABLE projects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
proj_type TEXT NOT NULL DEFAULT 'builtin', -- builtin | gitea | github
owner TEXT NOT NULL DEFAULT '',
forge_id TEXT DEFAULT '',
clone_url TEXT DEFAULT '',
default_branch TEXT DEFAULT '',
language TEXT DEFAULT '',
description TEXT DEFAULT '',
last_synced_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(proj_type, owner, name)
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_projects_type ON projects(proj_type, last_synced_at)"
)
@register(7, "v5.4.0/v5.5.0: page versions, cover + icon")
def _migration_v54_page_versions_cover(conn: sqlite3.Connection) -> None:
"""v5.4.0 (version history + page duplication) & v5.5.0 (cover & icon).
``page_versions`` — undoable version snapshots for block-editor pages
(NOT tied to ``collection_pages`` like the legacy
``page_history`` table). One row per save with the
full block list + title so the UI can browse/restore.
``pages.cover_url`` — image cover shown above the page title.
``pages.page_icon`` — emoji / icon label shown next to the title.
"""
conn.execute(
"""
CREATE TABLE IF NOT EXISTS page_versions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
user_id INTEGER REFERENCES users(id),
title TEXT NOT NULL DEFAULT '',
blocks_json TEXT NOT NULL DEFAULT '[]',
note TEXT NOT NULL DEFAULT '',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_page_versions_page ON page_versions(page_id, created_at)"
)
_pcols = {r[1] for r in conn.execute("PRAGMA table_info(pages)").fetchall()}
if "cover_url" not in _pcols:
conn.execute("ALTER TABLE pages ADD COLUMN cover_url TEXT DEFAULT ''")
if "page_icon" not in _pcols:
conn.execute("ALTER TABLE pages ADD COLUMN page_icon TEXT DEFAULT ''")
@register(8, "v5.6.0: custom workspace emojis")
def _migration_custom_emojis(conn: sqlite3.Connection) -> None:
"""Workspace-wide custom emojis (uploaded images) used as page icons."""
conn.execute(
"""
CREATE TABLE IF NOT EXISTS custom_emojis (
id INTEGER PRIMARY KEY AUTOINCREMENT,
workspace_id INTEGER NOT NULL DEFAULT 1,
name TEXT NOT NULL DEFAULT '',
url TEXT NOT NULL DEFAULT '',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_custom_emojis_ws ON custom_emojis(workspace_id, created_at)"
)
@register(4, "database templates (icon) + property validation")
def _migration_db_templates_validation(conn: sqlite3.Connection) -> None:
"""v5.3.0: database templates get an icon, properties a validation config,
and the built-in database templates are seeded (idempotently)."""
_cols = {r[1] for r in conn.execute("PRAGMA table_info(database_templates)").fetchall()}
if "icon" not in _cols:
conn.execute("ALTER TABLE database_templates ADD COLUMN icon TEXT NOT NULL DEFAULT '📋'")
_pcols = {r[1] for r in conn.execute("PRAGMA table_info(collection_properties)").fetchall()}
if "validation_json" not in _pcols:
conn.execute("ALTER TABLE collection_properties ADD COLUMN validation_json TEXT NOT NULL DEFAULT '{}'")
# Seed built-in templates (idempotent: only missing names are inserted).
from app.services.db_templates import SEED_TEMPLATES
for tpl in SEED_TEMPLATES:
conn.execute(
"""INSERT OR IGNORE INTO database_templates (name, icon, description, schema_json)
VALUES (?, ?, ?, ?)""",
(tpl["name"], tpl.get("icon", "📋"), tpl.get("description", ""),
__import__("json").dumps(tpl.get("schema", []))),
)
@register(9, "v5.6.0: import items (dedup) + import jobs")
def _migration_import_framework(conn: sqlite3.Connection) -> None:
"""Unified import framework (Phase 0).
``import_items`` — one row per imported page, keyed by workspace + source +
external id, so re-importing the same vault is idempotent.
``import_jobs`` — background import job status/history for UI polling.
"""
conn.execute(
"""
CREATE TABLE IF NOT EXISTS import_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
workspace_id INTEGER,
source TEXT NOT NULL DEFAULT '',
external_id TEXT NOT NULL DEFAULT '',
page_id INTEGER,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(workspace_id, source, external_id)
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_import_items_lookup "
"ON import_items(workspace_id, source, external_id)"
)
conn.execute(
"""
CREATE TABLE IF NOT EXISTS import_jobs (
id TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT '',
filename TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'queued',
error TEXT NOT NULL DEFAULT '',
report_json TEXT NOT NULL DEFAULT '{}',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_import_jobs_created ON import_jobs(created_at)"
)
@register(10, "v5.7.0: property groups, per-user views, row covers")
def _migration_v57_db_advanced(conn: sqlite3.Connection) -> None:
"""v5.7.0 — Database Avancée (Pt. 2).
``collection_properties.group_name`` — groups properties into collapsible
sections in the table header (Notion property groups).
``collection_views.created_by`` — owner of a saved view; ``NULL`` means
a shared/legacy view visible to everyone, otherwise it is personal to a user.
``collection_pages.cover_url`` — per-row cover image (gallery/board
cards), independent from the block-page ``pages.cover_url``.
"""
_pcols = {r[1] for r in conn.execute("PRAGMA table_info(collection_properties)").fetchall()}
if "group_name" not in _pcols:
conn.execute(
"ALTER TABLE collection_properties ADD COLUMN group_name TEXT NOT NULL DEFAULT ''"
)
_vcols = {r[1] for r in conn.execute("PRAGMA table_info(collection_views)").fetchall()}
if "created_by" not in _vcols:
conn.execute("ALTER TABLE collection_views ADD COLUMN created_by INTEGER")
if "updated_at" not in _vcols:
conn.execute("ALTER TABLE collection_views ADD COLUMN updated_at TIMESTAMP")
_cpcols = {r[1] for r in conn.execute("PRAGMA table_info(collection_pages)").fetchall()}
if "cover_url" not in _cpcols:
conn.execute("ALTER TABLE collection_pages ADD COLUMN cover_url TEXT DEFAULT ''")
@register(11, "v5.8.0: reminder log + user timezones")
def _migration_v58_calendar_reminders(conn: sqlite3.Connection) -> None:
"""v5.8.0 — Calendrier & Rappels.
``reminder_log`` — dedup ledger: one row per (page, occurrence date) so
a reminder fires exactly once even across restarts.
``users.timezone`` — personal IANA timezone used for "today" in calendar
views and reminder firing (empty = UTC).
"""
conn.execute(
"""CREATE TABLE IF NOT EXISTS reminder_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
page_id INTEGER NOT NULL REFERENCES collection_pages(id) ON DELETE CASCADE,
occurrence_date TEXT NOT NULL,
fired_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(page_id, occurrence_date)
)"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_remlog_page ON reminder_log(page_id)"
)
_ucols = {r[1] for r in conn.execute("PRAGMA table_info(users)").fetchall()}
if "timezone" not in _ucols:
conn.execute("ALTER TABLE users ADD COLUMN timezone TEXT NOT NULL DEFAULT ''")
@register(12, "v5.8.0: enrich Meeting notes template")
def _migration_v58_meeting_template(conn: sqlite3.Connection) -> None:
"""v5.8.0 — the seeded 'Meeting notes' database template gains Agenda and
Notes text properties. Only refreshed when the row still matches the old
built-in schema (user edits are never clobbered)."""
import json as _json
row = conn.execute(
"SELECT schema_json FROM database_templates WHERE name='Meeting notes'"
).fetchone()
if not row:
return
try:
schema = _json.loads(row[0] or "[]")
except (ValueError, TypeError):
return
names = [p.get("name") for p in schema]
if "Agenda" in names or "Notes" in names:
return
if names != ["Title", "Date", "Attendees", "Status", "Action items"]:
return # customised — leave alone
idx = names.index("Action items")
schema[idx:idx] = [
{"name": "Agenda", "type": "text"},
{"name": "Notes", "type": "text"},
]
conn.execute(
"UPDATE database_templates SET schema_json=? WHERE name='Meeting notes'",
(_json.dumps(schema),),
)
conn.execute(
"UPDATE database_templates SET description=? WHERE name='Meeting notes'",
("Notes de réunion avec participants, agenda, notes et actions.",),
)
@register(13, "v5.11.0/v5.12.0: page lock, user typo prefs, global page templates")
def _migration_v511_wiki_v512_templates(conn: sqlite3.Connection) -> None:
"""v5.11.0 Wiki-links + v5.12.0 Templates & verrouillage.
``pages.is_locked`` — read-only page (locker/admin can unlock).
``pages.locked_by`` — user that locked the page.
``pages.full_width`` — per-page full-width layout toggle.
``pages.font_small`` — per-page compact typography toggle.
``page_global_templates`` — user-created global page templates
(blocks_json = same format as the block editor saves).
"""
_pcols = {r[1] for r in conn.execute("PRAGMA table_info(pages)").fetchall()}
if "is_locked" not in _pcols:
conn.execute("ALTER TABLE pages ADD COLUMN is_locked INTEGER NOT NULL DEFAULT 0")
if "locked_by" not in _pcols:
conn.execute("ALTER TABLE pages ADD COLUMN locked_by INTEGER REFERENCES users(id) ON DELETE SET NULL")
if "full_width" not in _pcols:
conn.execute("ALTER TABLE pages ADD COLUMN full_width INTEGER NOT NULL DEFAULT 0")
if "font_small" not in _pcols:
conn.execute("ALTER TABLE pages ADD COLUMN font_small INTEGER NOT NULL DEFAULT 0")
conn.execute(
"""CREATE TABLE IF NOT EXISTS page_global_templates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
icon TEXT NOT NULL DEFAULT '📄',
description TEXT NOT NULL DEFAULT '',
blocks_json TEXT NOT NULL DEFAULT '[]',
created_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_pgt_creator ON page_global_templates(created_by)"
)