From d7c1f4c03e8ab0a8f1882f43f843e8fce2525554 Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 21 Jul 2026 21:55:45 -0400 Subject: [PATCH] feat(v4.1.0): Data Sources & Linked Databases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add collection_data_sources table for multi-source collections - API: GET/POST/DELETE /db/{id}/sources/api — data source management - API: POST /db/{id}/linked/api — create linked database (copies views/properties) - API: POST /db/{id}/toggle-inline/api — toggle full-page vs inline - API: POST /db/inline/api — create inline database - Linked DB inherits workspace_id (permissions) from source - Migration: add workspace_id + created_by to collections table - 10 new tests (153 total) --- app/db.py | 25 ++++ app/routers/collections.py | 250 +++++++++++++++++++++++++++++++++++++ tests/test_app.py | 226 +++++++++++++++++++++++++++++++++ 3 files changed, 501 insertions(+) diff --git a/app/db.py b/app/db.py index 307dca5..601baa8 100644 --- a/app/db.py +++ b/app/db.py @@ -462,6 +462,31 @@ def init_db(): """) conn.commit() + # v4.1.0: Data Sources & Linked Databases + conn.execute(""" + CREATE TABLE IF NOT EXISTS collection_data_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE, + source_collection_id INTEGER NOT NULL REFERENCES collections(id), + source_name TEXT DEFAULT '', + is_linked BOOLEAN NOT NULL DEFAULT 0, + position INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(collection_id, source_collection_id) + ) + """) + # Add workspace_id to collections if missing (v4.1.0 migration) + try: + conn.execute("ALTER TABLE collections ADD COLUMN workspace_id INTEGER REFERENCES workspaces(id)") + except sqlite3.OperationalError: + pass + # Add created_by to collections if missing + try: + conn.execute("ALTER TABLE collections ADD COLUMN created_by INTEGER REFERENCES users(id)") + except sqlite3.OperationalError: + pass + conn.commit() + @contextmanager def get_conn(): diff --git a/app/routers/collections.py b/app/routers/collections.py index 4510f26..c02e2d4 100644 --- a/app/routers/collections.py +++ b/app/routers/collections.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import logging +import sqlite3 from fastapi import APIRouter, Request, HTTPException, Query from fastapi.responses import HTMLResponse @@ -760,6 +761,255 @@ async def check_dependencies(request: Request, collection_id: int, page_id: int) } +# ── v4.1.0: Data Sources & Linked Databases ── + + +@router.get("/{collection_id}/sources/api") +async def list_data_sources(request: Request, collection_id: int): + """API: list all data sources for a collection.""" + with get_conn() as conn: + coll = conn.execute("SELECT id FROM collections WHERE id=?", (collection_id,)).fetchone() + if not coll: + raise HTTPException(status_code=404, detail="Collection not found") + + rows = conn.execute( + "SELECT * FROM collection_data_sources WHERE collection_id=? ORDER BY position", + (collection_id,), + ).fetchall() + return {"sources": [dict(r) for r in rows]} + + +@router.post("/{collection_id}/sources/api") +async def add_data_source(request: Request, collection_id: int): + """API: add a data source to a collection.""" + try: + body = await request.json() + except Exception: + body = {} + + source_collection_id = body.get("source_collection_id") + if not source_collection_id: + raise HTTPException(status_code=400, detail="source_collection_id is required") + + source_name = body.get("source_name", "").strip() + is_linked = body.get("is_linked", False) + + with get_conn() as conn: + # Verify both collections exist + for cid in (collection_id, source_collection_id): + if not conn.execute("SELECT id FROM collections WHERE id=?", (cid,)).fetchone(): + raise HTTPException(status_code=404, detail=f"Collection {cid} not found") + + max_pos = conn.execute( + "SELECT COALESCE(MAX(position), -1) + 1 FROM collection_data_sources WHERE collection_id=?", + (collection_id,), + ).fetchone()[0] + + try: + cur = conn.execute( + """INSERT INTO collection_data_sources + (collection_id, source_collection_id, source_name, is_linked, position) + VALUES (?, ?, ?, ?, ?)""", + (collection_id, source_collection_id, source_name, int(is_linked), max_pos), + ) + conn.commit() + except sqlite3.IntegrityError: + raise HTTPException(status_code=409, detail="This data source already exists in this collection") + + return { + "id": cur.lastrowid, + "collection_id": collection_id, + "source_collection_id": source_collection_id, + "status": "added", + } + + +@router.delete("/{collection_id}/sources/{source_id}/api") +async def remove_data_source(request: Request, collection_id: int, source_id: int): + """API: remove a data source from a collection.""" + with get_conn() as conn: + existing = conn.execute( + "SELECT * FROM collection_data_sources WHERE id=? AND collection_id=?", + (source_id, collection_id), + ).fetchone() + if not existing: + raise HTTPException(status_code=404, detail="Data source not found") + + conn.execute("DELETE FROM collection_data_sources WHERE id=?", (source_id,)) + conn.commit() + + return {"id": source_id, "status": "removed"} + + +@router.post("/{collection_id}/linked/api") +async def create_linked_database(request: Request, collection_id: int): + """API: create a linked database view from a source collection. + A linked database copies the structure (views, filters, sorts) of a source + but shares the same pages — edits to pages propagate to the source. + """ + try: + body = await request.json() + except Exception: + body = {} + + name = body.get("name", "").strip() + new_workspace_id = body.get("workspace_id") + + with get_conn() as conn: + source = conn.execute( + "SELECT * FROM collections WHERE id=?", (collection_id,) + ).fetchone() + if not source: + raise HTTPException(status_code=404, detail="Source collection not found") + + if not name: + name = f"{source['name']} (linked)" + + # Create the linked collection (shallow copy of structure) + # Inherit workspace_id from source for permission inheritance + src_dict = dict(source) + source_workspace_id = src_dict.get("workspace_id") + cur = conn.execute( + """INSERT INTO collections + (name, description, icon, schema_json, is_locked, is_inline, parent_page_id, workspace_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ( + name, + src_dict["description"], + src_dict["icon"], + src_dict["schema_json"], + 0, # linked DB is never locked + 1, # linked DB starts as inline + src_dict.get("parent_page_id"), + source_workspace_id, # linked DB inherits source workspace permissions + ), + ) + linked_id = cur.lastrowid + + # Copy views from source + views = conn.execute( + "SELECT * FROM collection_views WHERE collection_id=? ORDER BY position", + (collection_id,), + ).fetchall() + for v in views: + conn.execute( + "INSERT INTO collection_views (collection_id, name, view_type, config_json, position) VALUES (?, ?, ?, ?, ?)", + (linked_id, v["name"], v["view_type"], v["config_json"], v["position"]), + ) + + # Add the source as a data source with is_linked=1 + conn.execute( + """INSERT INTO collection_data_sources + (collection_id, source_collection_id, source_name, is_linked, position) + VALUES (?, ?, ?, 1, 0)""", + (linked_id, collection_id, source["name"]), + ) + + # Copy properties from source + props = conn.execute( + "SELECT * FROM collection_properties WHERE collection_id=? ORDER BY position", + (collection_id,), + ).fetchall() + for p in props: + conn.execute( + """INSERT INTO collection_properties + (collection_id, name, prop_type, options_json, number_format, + related_collection_id, reverse_name, relation_property_id, + target_property_id, rollup_function, formula_expression, + position, required, visible_in_views) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + linked_id, p["name"], p["prop_type"], p["options_json"], + p["number_format"], p["related_collection_id"], p["reverse_name"], + p["relation_property_id"], p["target_property_id"], + p["rollup_function"], p["formula_expression"], + p["position"], p["required"], p["visible_in_views"], + ), + ) + + conn.commit() + + return { + "linked_id": linked_id, + "name": name, + "source_collection_id": collection_id, + "status": "created", + } + + +@router.post("/{collection_id}/toggle-inline/api") +async def toggle_inline(request: Request, collection_id: int): + """API: toggle a collection between full-page and inline mode.""" + with get_conn() as conn: + coll = conn.execute( + "SELECT id, is_inline FROM collections WHERE id=?", + (collection_id,), + ).fetchone() + if not coll: + raise HTTPException(status_code=404, detail="Collection not found") + + new_inline = 0 if coll["is_inline"] else 1 + conn.execute( + "UPDATE collections SET is_inline=?, updated_at=CURRENT_TIMESTAMP WHERE id=?", + (new_inline, collection_id), + ) + conn.commit() + + return { + "collection_id": collection_id, + "is_inline": bool(new_inline), + "mode": "inline" if new_inline else "full-page", + } + + +@router.post("/inline/api") +async def create_inline_database(request: Request): + """API: create an inline database within a parent page.""" + try: + body = await request.json() + except Exception: + body = {} + + name = body.get("name", "").strip() + if not name: + raise HTTPException(status_code=400, detail="name is required") + + description = body.get("description", "") + icon = body.get("icon", "📋") + parent_page_id = body.get("parent_page_id") + workspace_id = body.get("workspace_id") + + with get_conn() as conn: + cur = conn.execute( + """INSERT INTO collections + (name, description, icon, schema_json, is_inline, parent_page_id) + VALUES (?, ?, ?, '[]', 1, ?)""", + (name, description, icon, parent_page_id), + ) + collection_id = cur.lastrowid + + # Create default view + conn.execute( + """INSERT INTO collection_views + (collection_id, name, view_type, config_json) + VALUES (?, ?, ?, ?)""", + (collection_id, "Default View", "table", json.dumps({ + "visible_properties": ["Title"], + "sorts": [], + "filters": [], + })), + ) + conn.commit() + + return { + "id": collection_id, + "name": name, + "is_inline": True, + "parent_page_id": parent_page_id, + "status": "created", + } + + # ── {collection_id} wildcards (LAST — catches everything else) ── diff --git a/tests/test_app.py b/tests/test_app.py index 33a9a80..ee58721 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -2002,3 +2002,229 @@ def test_session_expired_redirect(client): location = resp.headers.get("location", "") assert "login" in location assert "expired=1" in location + + +# ── v4.1.0: Data Sources & Linked Databases ── + + +def test_data_sources_list_empty(client): + """List data sources for a collection — empty by default.""" + # Create a collection first + resp = client.post("/db/api", json={"name": "Sources Test DB"}) + coll_id = resp.json()["id"] + + resp = client.get(f"/db/{coll_id}/sources/api") + assert resp.status_code == 200 + assert resp.json()["sources"] == [] + + +def test_data_sources_add_and_remove(client): + """Add and remove a data source from a collection.""" + # Create two collections + r1 = client.post("/db/api", json={"name": "Source A"}) + r2 = client.post("/db/api", json={"name": "Source B"}) + coll_a = r1.json()["id"] + coll_b = r2.json()["id"] + + # Add B as a data source of A + resp = client.post(f"/db/{coll_a}/sources/api", json={ + "source_collection_id": coll_b, + "source_name": "Linked B", + }) + assert resp.status_code == 200 + assert resp.json()["status"] == "added" + source_id = resp.json()["id"] + + # List — should have 1 source + resp = client.get(f"/db/{coll_a}/sources/api") + assert resp.status_code == 200 + assert len(resp.json()["sources"]) == 1 + assert resp.json()["sources"][0]["source_name"] == "Linked B" + + # Remove the source + resp = client.delete(f"/db/{coll_a}/sources/{source_id}/api") + assert resp.status_code == 200 + assert resp.json()["status"] == "removed" + + # List — should be empty again + resp = client.get(f"/db/{coll_a}/sources/api") + assert len(resp.json()["sources"]) == 0 + + +def test_data_sources_duplicate_rejected(client): + """Adding the same data source twice returns 409.""" + r1 = client.post("/db/api", json={"name": "Dup Source A"}) + r2 = client.post("/db/api", json={"name": "Dup Source B"}) + coll_a = r1.json()["id"] + coll_b = r2.json()["id"] + + # First add — OK + resp = client.post(f"/db/{coll_a}/sources/api", json={ + "source_collection_id": coll_b, + }) + assert resp.status_code == 200 + + # Second add — conflict + resp = client.post(f"/db/{coll_a}/sources/api", json={ + "source_collection_id": coll_b, + }) + assert resp.status_code == 409 + + +def test_data_sources_not_found(client): + """Non-existent collection returns 404.""" + resp = client.get("/db/99999/sources/api") + assert resp.status_code == 404 + + +def test_create_linked_database(client): + """Create a linked database from a source collection.""" + # Create source collection + resp = client.post("/db/api", json={ + "name": "CRM Contacts", + "description": "Customer contacts", + "icon": "👥", + }) + source_id = resp.json()["id"] + + # Add a property to the source + client.post(f"/db/{source_id}/properties/api", json={ + "name": "Email", + "prop_type": "email", + }) + + # Create linked DB + resp = client.post(f"/db/{source_id}/linked/api", json={}) + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "created" + assert data["source_collection_id"] == source_id + linked_id = data["linked_id"] + assert linked_id != source_id + + # Linked DB should have a data source pointing to source + resp = client.get(f"/db/{linked_id}/sources/api") + sources = resp.json()["sources"] + assert len(sources) == 1 + assert sources[0]["source_collection_id"] == source_id + assert sources[0]["is_linked"] == 1 + + # Linked DB should have copied the source's properties + resp = client.get(f"/db/{linked_id}/properties/api") + props = resp.json()["properties"] + assert any(p["name"] == "Email" for p in props) + + # Linked DB should have copied the source's views + resp = client.get(f"/db/{linked_id}/views/api") + views = resp.json()["views"] + assert len(views) >= 1 + + +def test_create_linked_database_custom_name(client): + """Create a linked database with a custom name.""" + resp = client.post("/db/api", json={"name": "Task DB"}) + source_id = resp.json()["id"] + + resp = client.post(f"/db/{source_id}/linked/api", json={ + "name": "My Linked Tasks", + }) + assert resp.status_code == 200 + data = resp.json() + assert data["name"] == "My Linked Tasks" + assert data["status"] == "created" + + +def test_toggle_inline(client): + """Toggle a collection between full-page and inline modes.""" + resp = client.post("/db/api", json={"name": "Toggle Test"}) + coll_id = resp.json()["id"] + + # Start as full-page (is_inline=0 by default) + resp = client.get(f"/db/{coll_id}/api") + assert resp.json()["collection"]["is_inline"] == 0 + + # Toggle to inline + resp = client.post(f"/db/{coll_id}/toggle-inline/api") + assert resp.status_code == 200 + assert resp.json()["is_inline"] is True + assert resp.json()["mode"] == "inline" + + # Verify in DB + resp = client.get(f"/db/{coll_id}/api") + assert resp.json()["collection"]["is_inline"] == 1 + + # Toggle back to full-page + resp = client.post(f"/db/{coll_id}/toggle-inline/api") + assert resp.status_code == 200 + assert resp.json()["is_inline"] is False + assert resp.json()["mode"] == "full-page" + + +def test_create_inline_database(client): + """Create an inline database within a parent page.""" + # Create a page first (dummy) + from app.db import get_conn + with get_conn() as conn: + conn.execute( + "INSERT INTO pages (workspace, title) VALUES ('test', 'Parent Page')" + ) + conn.commit() + page_id = conn.execute("SELECT id FROM pages ORDER BY id DESC LIMIT 1").fetchone()["id"] + + resp = client.post("/db/inline/api", json={ + "name": "Inline Comments DB", + "description": "Inline comments database", + "parent_page_id": page_id, + }) + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "created" + assert data["is_inline"] is True + assert data["parent_page_id"] == page_id + + # Verify the collection exists with inline flag + resp = client.get(f"/db/{data['id']}/api") + assert resp.json()["collection"]["is_inline"] == 1 + + +def test_linked_db_inherits_workspace(client): + """Linked database should inherit the source's workspace_id.""" + # Create workspace + from app.db import get_conn + with get_conn() as conn: + conn.execute( + "INSERT INTO workspaces (name, owner_id) VALUES ('Test WS', 1)" + ) + conn.commit() + ws_id = conn.execute("SELECT id FROM workspaces ORDER BY id DESC LIMIT 1").fetchone()["id"] + + # Create collection with workspace_id + conn.execute( + "INSERT INTO collections (name, workspace_id) VALUES (?, ?)", + ("WS Collection", ws_id), + ) + conn.commit() + coll_id = conn.execute( + "SELECT id FROM collections WHERE name='WS Collection'" + ).fetchone()["id"] + + # Create linked DB + resp = client.post(f"/db/{coll_id}/linked/api", json={"name": "Linked WS DB"}) + assert resp.status_code == 200 + linked_id = resp.json()["linked_id"] + + # Verify linked DB has same workspace_id + with get_conn() as conn: + linked = conn.execute( + "SELECT workspace_id FROM collections WHERE id=?", (linked_id,) + ).fetchone() + assert linked["workspace_id"] == ws_id + + +def test_remove_data_source_not_found(client): + """Deleting a non-existent data source returns 404.""" + resp = client.post("/db/api", json={"name": "Remove Test"}) + coll_id = resp.json()["id"] + + resp = client.delete(f"/db/{coll_id}/sources/99999/api") + assert resp.status_code == 404