feat(import): v5.6.0 unified data import (phases 0-5)
- unified importer framework (app/services/importers/): normalized model, registry, common pipeline (hierarchy, attachments, collections, dedup), async jobs, dry-run preview, column->type mapping - Phase 1: Obsidian, Notion, Logseq/Roam, HTML (Apple Notes/Bear/Ulysses/ OneNote), Google Keep, generic Markdown - Phase 2: typed CSV/TSV, Excel (openpyxl), generic JSON - Phase 3: Word .docx (python-docx), PDF (pypdf), HTML folders - Phase 4: Raindrop, Pocket, Readwise, Shaarli, Netscape bookmarks, .ics, OPML, Standard Notes, Gitea/GitHub issues (+labels/milestones) - Phase 5: incremental re-sync (skip/update/duplicate), partial-error resume, forge repo files, URL web clipper (SSRF guard), batch multi-file + UI queue, Notion relation resolution, exportable JSON reports - /import wizard, API /api/import/*, migration 9 (import_items, import_jobs) - fix: property values stored by property id (correct DB view rendering) - deps: openpyxl, beautifulsoup4, PyYAML, python-docx, pypdf - 43 import tests; full suite 491 green; ruff clean - bump version 5.11.5
This commit is contained in:
+6
-2
@@ -41,6 +41,8 @@ from app.routers.collaboration import router as collaboration_router
|
||||
from app.routers.emoji import router as emoji_router
|
||||
from app.routers.gitea import router as gitea_router
|
||||
from app.routers.github_routes import router as github_router
|
||||
from app.routers.imports import page_router as import_page_router
|
||||
from app.routers.imports import router as imports_router
|
||||
from app.routers.notifications import router as notifications_router
|
||||
from app.routers.realtime import router as realtime_router
|
||||
from app.services.webhook_outbound import init_webhook_tables
|
||||
@@ -86,7 +88,7 @@ async def lifespan(_app: FastAPI):
|
||||
from app.services.trash import trash_purge_scheduler
|
||||
trash_task = asyncio.create_task(trash_purge_scheduler())
|
||||
|
||||
logger.info("FlowDeck v5.11.2 started on port %d", settings.app_port)
|
||||
logger.info("FlowDeck v5.11.5 started on port %d", settings.app_port)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@@ -101,7 +103,7 @@ async def lifespan(_app: FastAPI):
|
||||
|
||||
app = FastAPI(
|
||||
title="FlowDeck",
|
||||
version="5.11.2",
|
||||
version="5.11.5",
|
||||
docs_url="/docs" if settings.log_level == "DEBUG" else None,
|
||||
redoc_url=None,
|
||||
lifespan=lifespan,
|
||||
@@ -141,6 +143,8 @@ app.include_router(agent.router)
|
||||
app.include_router(search.router)
|
||||
app.include_router(security.router)
|
||||
app.include_router(onboarding.router)
|
||||
app.include_router(imports_router)
|
||||
app.include_router(import_page_router)
|
||||
|
||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||
|
||||
|
||||
@@ -375,3 +375,47 @@ def _migration_db_templates_validation(conn: sqlite3.Connection) -> None:
|
||||
(tpl["name"], tpl.get("icon", "📋"), tpl.get("description", ""),
|
||||
__import__("json").dumps(tpl.get("schema", []))),
|
||||
)
|
||||
|
||||
|
||||
@register(9, "v5.6.0: import items (dedup) + import jobs")
|
||||
def _migration_import_framework(conn: sqlite3.Connection) -> None:
|
||||
"""Unified import framework (Phase 0).
|
||||
|
||||
``import_items`` — one row per imported page, keyed by workspace + source +
|
||||
external id, so re-importing the same vault is idempotent.
|
||||
``import_jobs`` — background import job status/history for UI polling.
|
||||
"""
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS import_items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workspace_id INTEGER,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
external_id TEXT NOT NULL DEFAULT '',
|
||||
page_id INTEGER,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(workspace_id, source, external_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_import_items_lookup "
|
||||
"ON import_items(workspace_id, source, external_id)"
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS import_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
report_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_import_jobs_created ON import_jobs(created_at)"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""FlowDeck — unified import API (v5.6.0, Phase 0/1/2).
|
||||
|
||||
Exposes the importer registry, a dry-run preview, a synchronous run and an
|
||||
optional background job with polling. Works with the existing workspace cookie
|
||||
(``flowdeck_workspace``) and session.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||
|
||||
from app.auth.session import SessionManager
|
||||
from app.db import get_conn
|
||||
from app.services.automations import fire_event
|
||||
from app.services.importers import (
|
||||
get_job,
|
||||
list_jobs,
|
||||
list_sources,
|
||||
parse_upload,
|
||||
preview_result,
|
||||
resolve_relations,
|
||||
run_import,
|
||||
start_import_job,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/import", tags=["import"])
|
||||
page_router = APIRouter(tags=["import"])
|
||||
|
||||
MAX_UPLOAD_BYTES = 200 * 1024 * 1024
|
||||
|
||||
|
||||
def _current_user(request: Request) -> dict:
|
||||
return SessionManager.decode_session(request.cookies.get("flowdeck_session", "")) or {}
|
||||
|
||||
|
||||
@page_router.get("/import", response_class=HTMLResponse)
|
||||
async def import_page(request: Request):
|
||||
"""Standalone import wizard (source picker, dry-run, mapping, progress)."""
|
||||
user = _current_user(request)
|
||||
if not user:
|
||||
return RedirectResponse("/auth/login?provider=local", status_code=302)
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
env = Environment(loader=FileSystemLoader("app/templates"))
|
||||
return HTMLResponse(content=env.get_template("import.html").render(user=user))
|
||||
|
||||
|
||||
def _workspace(request: Request) -> tuple[int | None, str]:
|
||||
"""Resolve (workspace_id, login) from the workspace cookie + session."""
|
||||
ws_id: int | None = None
|
||||
cookie = request.cookies.get("flowdeck_workspace", "")
|
||||
try:
|
||||
value = int(cookie)
|
||||
if value > 0:
|
||||
ws_id = value
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
user = _current_user(request)
|
||||
login = user.get("login", "") if user else ""
|
||||
return ws_id, login
|
||||
|
||||
|
||||
async def _read_upload(request: Request) -> tuple[str, bytes, str | None]:
|
||||
form = await request.form()
|
||||
upload = form.get("file")
|
||||
if upload is None or not hasattr(upload, "filename"):
|
||||
raise HTTPException(400, "file field required")
|
||||
filename = (upload.filename or "import").replace("\\", "/").rsplit("/", 1)[-1]
|
||||
data = await upload.read()
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "File too large (max 200 MB)")
|
||||
source_id = form.get("source") or None
|
||||
return filename, data, source_id
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def import_sources(request: Request):
|
||||
"""List every available importer for the UI source picker."""
|
||||
return {"sources": list_sources()}
|
||||
|
||||
|
||||
@router.post("/preview")
|
||||
async def import_preview(request: Request):
|
||||
"""Dry-run: parse the upload and describe what would be created."""
|
||||
filename, data, source_id = await _read_upload(request)
|
||||
imp, result = parse_upload(filename, data, source_id)
|
||||
if imp is None:
|
||||
raise HTTPException(400, "Format non reconnu — choisissez une source")
|
||||
out = preview_result(result)
|
||||
out["detected_source"] = imp.source_id
|
||||
out["source_label"] = imp.label
|
||||
return out
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
async def import_run(request: Request):
|
||||
"""Import an upload (synchronously, or as a background job when async=true)."""
|
||||
form = await request.form()
|
||||
upload = form.get("file")
|
||||
if upload is None or not hasattr(upload, "filename"):
|
||||
raise HTTPException(400, "file field required")
|
||||
filename = (upload.filename or "import").replace("\\", "/").rsplit("/", 1)[-1]
|
||||
data = await upload.read()
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise HTTPException(413, "File too large (max 200 MB)")
|
||||
|
||||
source_id = form.get("source") or None
|
||||
parent_id = _int_or_none(form.get("parent_id"))
|
||||
target = _int_or_none(form.get("target_collection_id"))
|
||||
dedup = str(form.get("dedup", "true")).lower() not in ("false", "0", "no")
|
||||
async_mode = str(form.get("async", "false")).lower() in ("true", "1", "yes")
|
||||
mapping = _parse_mapping(form.get("mapping"))
|
||||
mode = _parse_mode(form.get("mode"))
|
||||
ws_id, login = _workspace(request)
|
||||
|
||||
if async_mode:
|
||||
job = start_import_job(
|
||||
filename=filename, data=data, source_id=source_id,
|
||||
workspace_id=ws_id, workspace_name=login, user_login=login,
|
||||
parent_page_id=parent_id, target_collection_id=target, dedup=dedup,
|
||||
mapping=mapping, mode=mode,
|
||||
)
|
||||
return {"status": "queued", "job_id": job["id"]}
|
||||
|
||||
imp, result = parse_upload(filename, data, source_id)
|
||||
if imp is None:
|
||||
raise HTTPException(400, "Format non reconnu — choisissez une source")
|
||||
report = run_import(
|
||||
result, workspace_id=ws_id, workspace_name=login, user_login=login,
|
||||
parent_page_id=parent_id, target_collection_id=target, dedup=dedup,
|
||||
mapping=mapping, mode=mode,
|
||||
)
|
||||
report["detected_source"] = imp.source_id
|
||||
if imp.source_id == "notion":
|
||||
with get_conn() as conn:
|
||||
report["relations"] = resolve_relations(conn, ws_id)
|
||||
for page_id in report.get("page_ids", [])[:100]:
|
||||
try:
|
||||
await fire_event("page.created", {"page_id": page_id, "title": "", "workspace": login})
|
||||
except Exception: # noqa: BLE001 - events are best-effort
|
||||
pass
|
||||
return report
|
||||
|
||||
|
||||
@router.post("/forge")
|
||||
async def import_forge(request: Request):
|
||||
"""Import a forge repo's issues (+ labels/milestones) into collections."""
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
raise HTTPException(400, "Invalid JSON body") from None
|
||||
provider = str(body.get("provider") or "gitea").lower()
|
||||
owner = str(body.get("owner") or "").strip()
|
||||
repo = str(body.get("repo") or "").strip()
|
||||
if not owner or not repo:
|
||||
raise HTTPException(400, "owner and repo are required")
|
||||
state = str(body.get("state") or "all")
|
||||
include_labels = bool(body.get("include_labels", True))
|
||||
include_milestones = bool(body.get("include_milestones", True))
|
||||
ws_id, login = _workspace(request)
|
||||
|
||||
if provider == "gitea":
|
||||
from app.services.gitea_client import get_user_gitea_client
|
||||
from app.services.importers.forge import GiteaForgeAdapter
|
||||
client = get_user_gitea_client(request)
|
||||
if client is None:
|
||||
raise HTTPException(400, "Gitea non connecté")
|
||||
adapter = GiteaForgeAdapter(client)
|
||||
elif provider == "github":
|
||||
from app.services.github_adapter import GitHubAdapter
|
||||
token = _user_oauth_token(request, "github")
|
||||
if not token:
|
||||
raise HTTPException(400, "GitHub non connecté")
|
||||
adapter = GitHubAdapter(token)
|
||||
else:
|
||||
raise HTTPException(400, "provider must be 'gitea' or 'github'")
|
||||
|
||||
from app.services.importers.forge import fetch_forge_issues
|
||||
result = await fetch_forge_issues(
|
||||
adapter, owner, repo, provider=provider, state=state,
|
||||
include_labels=include_labels, include_milestones=include_milestones,
|
||||
)
|
||||
report = run_import(
|
||||
result, workspace_id=ws_id, workspace_name=login, user_login=login,
|
||||
)
|
||||
report["detected_source"] = f"forge:{provider}"
|
||||
return report
|
||||
|
||||
|
||||
@router.post("/forge-repo")
|
||||
async def import_forge_repo(request: Request):
|
||||
"""Import a forge repo's text files as pages (folder hierarchy preserved)."""
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
raise HTTPException(400, "Invalid JSON body") from None
|
||||
provider = str(body.get("provider") or "gitea").lower()
|
||||
owner = str(body.get("owner") or "").strip()
|
||||
repo = str(body.get("repo") or "").strip()
|
||||
if not owner or not repo:
|
||||
raise HTTPException(400, "owner and repo are required")
|
||||
path = str(body.get("path") or "")
|
||||
max_files = min(int(body.get("max_files") or 200), 1000)
|
||||
ws_id, login = _workspace(request)
|
||||
|
||||
adapter = _forge_adapter(request, provider)
|
||||
|
||||
from app.services.importers.forge_repo import fetch_forge_repo
|
||||
result = await fetch_forge_repo(
|
||||
adapter, owner, repo, provider=provider, path=path, max_files=max_files,
|
||||
)
|
||||
report = run_import(result, workspace_id=ws_id, workspace_name=login, user_login=login)
|
||||
report["detected_source"] = f"forge-repo:{provider}"
|
||||
return report
|
||||
|
||||
|
||||
@router.post("/url")
|
||||
async def import_url(request: Request):
|
||||
"""Web clipper: fetch a URL and create a page (bookmark card + content)."""
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
raise HTTPException(400, "Invalid JSON body") from None
|
||||
url = str(body.get("url") or "").strip()
|
||||
if not url:
|
||||
raise HTTPException(400, "url is required")
|
||||
ws_id, login = _workspace(request)
|
||||
|
||||
from app.services.importers.url_fetch import fetch_url_result
|
||||
try:
|
||||
result = await fetch_url_result(url)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from None
|
||||
if not result.pages:
|
||||
raise HTTPException(422, "; ".join(result.warnings) or "Page introuvable")
|
||||
report = run_import(result, workspace_id=ws_id, workspace_name=login, user_login=login)
|
||||
report["detected_source"] = "url"
|
||||
return report
|
||||
|
||||
|
||||
@router.post("/run-batch")
|
||||
async def import_run_batch(request: Request):
|
||||
"""Import several uploaded files sequentially, returning one report each."""
|
||||
form = await request.form()
|
||||
uploads = form.getlist("file")
|
||||
if not uploads:
|
||||
raise HTTPException(400, "file field required")
|
||||
source_id = form.get("source") or None
|
||||
parent_id = _int_or_none(form.get("parent_id"))
|
||||
target = _int_or_none(form.get("target_collection_id"))
|
||||
dedup = str(form.get("dedup", "true")).lower() not in ("false", "0", "no")
|
||||
mode = _parse_mode(form.get("mode"))
|
||||
mapping = _parse_mapping(form.get("mapping"))
|
||||
ws_id, login = _workspace(request)
|
||||
|
||||
results: list[dict] = []
|
||||
summary = {"files": 0, "pages_created": 0, "rows_created": 0, "errors": 0}
|
||||
for upload in uploads:
|
||||
filename = (getattr(upload, "filename", "") or "import").replace("\\", "/").rsplit("/", 1)[-1]
|
||||
data = await upload.read()
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
results.append({"filename": filename, "report": {"status": "error",
|
||||
"errors": [{"title": filename, "error": "File too large"}]}})
|
||||
summary["errors"] += 1
|
||||
continue
|
||||
imp, result = parse_upload(filename, data, source_id)
|
||||
if imp is None:
|
||||
results.append({"filename": filename, "report": {"status": "error",
|
||||
"errors": [{"title": filename, "error": "Format non reconnu"}]}})
|
||||
summary["errors"] += 1
|
||||
continue
|
||||
report = run_import(
|
||||
result, workspace_id=ws_id, workspace_name=login, user_login=login,
|
||||
parent_page_id=parent_id, target_collection_id=target, dedup=dedup,
|
||||
mapping=mapping, mode=mode,
|
||||
)
|
||||
report["detected_source"] = imp.source_id
|
||||
results.append({"filename": filename, "report": report})
|
||||
summary["files"] += 1
|
||||
summary["pages_created"] += report.get("pages_created", 0)
|
||||
summary["rows_created"] += report.get("rows_created", 0)
|
||||
summary["errors"] += len(report.get("errors", []))
|
||||
return {"status": "ok", "summary": summary, "results": results}
|
||||
|
||||
|
||||
@router.post("/relations/resolve")
|
||||
async def import_resolve_relations(request: Request):
|
||||
"""Convert text columns referencing another collection into relation props."""
|
||||
ws_id, _ = _workspace(request)
|
||||
with get_conn() as conn:
|
||||
return resolve_relations(conn, ws_id)
|
||||
|
||||
|
||||
@router.get("/jobs")
|
||||
async def import_jobs(request: Request):
|
||||
return {"jobs": list_jobs()}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
async def import_job(job_id: str):
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, "Job not found")
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/report")
|
||||
async def import_job_report(job_id: str):
|
||||
"""Download a job's import report as JSON."""
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, "Job not found")
|
||||
payload = json.dumps(job.get("report") or {}, ensure_ascii=False, indent=2)
|
||||
return Response(
|
||||
content=payload,
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="import-{job_id}.json"'},
|
||||
)
|
||||
|
||||
|
||||
def _forge_adapter(request: Request, provider: str):
|
||||
if provider == "gitea":
|
||||
from app.services.gitea_client import get_user_gitea_client
|
||||
from app.services.importers.forge import GiteaForgeAdapter
|
||||
client = get_user_gitea_client(request)
|
||||
if client is None:
|
||||
raise HTTPException(400, "Gitea non connecté")
|
||||
return GiteaForgeAdapter(client)
|
||||
if provider == "github":
|
||||
from app.services.github_adapter import GitHubAdapter
|
||||
token = _user_oauth_token(request, "github")
|
||||
if not token:
|
||||
raise HTTPException(400, "GitHub non connecté")
|
||||
return GitHubAdapter(token)
|
||||
raise HTTPException(400, "provider must be 'gitea' or 'github'")
|
||||
|
||||
|
||||
def _int_or_none(value) -> int | None:
|
||||
try:
|
||||
ivalue = int(value)
|
||||
return ivalue if ivalue > 0 else None
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_mapping(value) -> dict[str, str] | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if isinstance(parsed, dict):
|
||||
return {str(k): str(v) for k, v in parsed.items() if v}
|
||||
return None
|
||||
|
||||
|
||||
def _parse_mode(value) -> str | None:
|
||||
mode = str(value or "").strip().lower()
|
||||
return mode if mode in ("skip", "update", "duplicate") else None
|
||||
|
||||
|
||||
def _user_oauth_token(request: Request, provider: str) -> str:
|
||||
user = _current_user(request)
|
||||
if not user or not user.get("id"):
|
||||
return ""
|
||||
with get_conn() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT access_token FROM user_oauth_tokens WHERE user_id=? AND provider=? "
|
||||
"ORDER BY updated_at DESC LIMIT 1",
|
||||
(user["id"], provider),
|
||||
).fetchone()
|
||||
return row["access_token"] if row else ""
|
||||
+14
-15
@@ -390,6 +390,20 @@ def _md_to_blocks(md: str) -> list:
|
||||
blocks.append({"type": "divider", "content": ""})
|
||||
i += 1
|
||||
continue
|
||||
if re.match(r"^\s*[-*+]\s+\[[ xX]\]\s+", line):
|
||||
flush_para()
|
||||
while i < n:
|
||||
s = lines[i].strip()
|
||||
m2 = re.match(r"^[-*+]\s+\[([ xX])\]\s+(.*)$", s)
|
||||
if not m2:
|
||||
break
|
||||
blocks.append({
|
||||
"type": "to_do",
|
||||
"content": m2.group(2).strip(),
|
||||
"checked": m2.group(1).lower() == "x",
|
||||
})
|
||||
i += 1
|
||||
continue
|
||||
if re.match(r"^\s*[-*+]\s+", line):
|
||||
flush_para()
|
||||
while i < n:
|
||||
@@ -410,21 +424,6 @@ def _md_to_blocks(md: str) -> list:
|
||||
blocks.append({"type": "numbered_list", "content": m2.group(1).strip()})
|
||||
i += 1
|
||||
continue
|
||||
mtodo = re.match(r"^\s*[-*+]\s+\[([ xX])\]\s+(.*)$", stripped)
|
||||
if mtodo:
|
||||
flush_para()
|
||||
while i < n:
|
||||
s = lines[i].strip()
|
||||
m2 = re.match(r"^[-*+]\s+\[([ xX])\]\s+(.*)$", s)
|
||||
if not m2:
|
||||
break
|
||||
blocks.append({
|
||||
"type": "to_do",
|
||||
"content": m2.group(2).strip(),
|
||||
"checked": m2.group(1).lower() == "x",
|
||||
})
|
||||
i += 1
|
||||
continue
|
||||
mq = re.match(r"^>\s?(.*)$", stripped)
|
||||
if mq and line == stripped:
|
||||
flush_para()
|
||||
|
||||
@@ -209,6 +209,60 @@ class GitHubAdapter(ForgeAdapter):
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
# ── issues / labels / milestones ──
|
||||
|
||||
async def list_issues(self, owner: str, repo: str, state: str = "all") -> list[dict]:
|
||||
"""List issues (pull requests are filtered out)."""
|
||||
issues: list[dict] = []
|
||||
for page in range(1, 6):
|
||||
async with self._client() as client:
|
||||
resp = await client.get(
|
||||
f"{self._base}/repos/{owner}/{repo}/issues",
|
||||
headers=self._headers,
|
||||
params={"state": state, "per_page": 100, "page": page},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
batch = resp.json()
|
||||
if not batch:
|
||||
break
|
||||
issues.extend(i for i in batch if "pull_request" not in i)
|
||||
if len(batch) < 100:
|
||||
break
|
||||
return issues
|
||||
|
||||
async def list_labels(self, owner: str, repo: str) -> list[dict]:
|
||||
async with self._client() as client:
|
||||
resp = await client.get(
|
||||
f"{self._base}/repos/{owner}/{repo}/labels",
|
||||
headers=self._headers,
|
||||
params={"per_page": 100},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def list_milestones(self, owner: str, repo: str, state: str = "all") -> list[dict]:
|
||||
async with self._client() as client:
|
||||
resp = await client.get(
|
||||
f"{self._base}/repos/{owner}/{repo}/milestones",
|
||||
headers=self._headers,
|
||||
params={"state": state, "per_page": 100},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def list_repo_files(self, owner: str, repo: str, path: str = "") -> list[dict]:
|
||||
"""Flatten the repo tree into file entries (``path``, ``size``)."""
|
||||
tree = await self.get_repo_tree(owner, repo)
|
||||
prefix = path.strip("/")
|
||||
files = [
|
||||
{"path": item["path"], "size": item.get("size", 0)}
|
||||
for item in tree
|
||||
if item.get("type") == "blob" and item.get("path")
|
||||
]
|
||||
if prefix:
|
||||
files = [f for f in files if f["path"].startswith(prefix + "/") or f["path"] == prefix]
|
||||
return files
|
||||
|
||||
# ── token validation ──
|
||||
|
||||
async def validate_token(self) -> bool:
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""FlowDeck — importers package (v5.6.0).
|
||||
|
||||
Importing this package registers every built-in importer. Use
|
||||
:func:`parse_upload` to detect a source and :func:`run_import` to persist it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services.importers import ( # noqa: F401 - registration side effects
|
||||
bookmarks,
|
||||
calendar,
|
||||
docx,
|
||||
html_notes,
|
||||
markdown,
|
||||
notion,
|
||||
obsidian,
|
||||
opml,
|
||||
outline,
|
||||
pdf,
|
||||
standard_notes,
|
||||
tabular,
|
||||
)
|
||||
from app.services.importers.base import ( # noqa: F401
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
all_importers,
|
||||
detect_importer,
|
||||
get_importer,
|
||||
list_sources,
|
||||
)
|
||||
from app.services.importers.jobs import ( # noqa: F401
|
||||
create_job,
|
||||
get_job,
|
||||
list_jobs,
|
||||
parse_upload,
|
||||
start_import_job,
|
||||
)
|
||||
from app.services.importers.pipeline import preview_result, resolve_relations, run_import # noqa: F401
|
||||
|
||||
__all__ = [
|
||||
"ImportAttachment",
|
||||
"ImportPage",
|
||||
"ImportResult",
|
||||
"Importer",
|
||||
"all_importers",
|
||||
"detect_importer",
|
||||
"get_importer",
|
||||
"list_sources",
|
||||
"create_job",
|
||||
"get_job",
|
||||
"list_jobs",
|
||||
"parse_upload",
|
||||
"start_import_job",
|
||||
"preview_result",
|
||||
"run_import",
|
||||
"resolve_relations",
|
||||
]
|
||||
@@ -0,0 +1,106 @@
|
||||
"""FlowDeck — shared helpers for note importers (frontmatter, wikilinks)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
try: # PyYAML ships transitively via uvicorn[standard]
|
||||
import yaml
|
||||
except Exception: # pragma: no cover - fallback parser below
|
||||
yaml = None
|
||||
|
||||
|
||||
_FRONTMATTER_RE = re.compile(r"^\ufeff?---\s*\n(.*?)\n---\s*\n?", re.DOTALL)
|
||||
_WIKILINK_RE = re.compile(r"(!?)\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|([^\]]+))?\]\]")
|
||||
|
||||
|
||||
def split_frontmatter(text: str) -> tuple[dict[str, Any], str]:
|
||||
"""Split YAML frontmatter from the body. Returns ``(metadata, body)``."""
|
||||
m = _FRONTMATTER_RE.match(text)
|
||||
if not m:
|
||||
return {}, text
|
||||
raw = m.group(1)
|
||||
body = text[m.end():]
|
||||
if yaml is not None:
|
||||
try:
|
||||
meta = yaml.safe_load(raw)
|
||||
if isinstance(meta, dict):
|
||||
return meta, body
|
||||
except Exception: # noqa: BLE001 - fall back to the simple parser
|
||||
pass
|
||||
return _simple_yaml(raw), body
|
||||
|
||||
|
||||
def _simple_yaml(raw: str) -> dict[str, Any]:
|
||||
"""Minimal YAML subset parser (scalars, inline lists, block lists)."""
|
||||
meta: dict[str, Any] = {}
|
||||
current: str | None = None
|
||||
for line in raw.splitlines():
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
if line.lstrip().startswith("- ") and current:
|
||||
meta.setdefault(current, [])
|
||||
if isinstance(meta[current], list):
|
||||
meta[current].append(_scalar(line.lstrip()[2:].strip()))
|
||||
continue
|
||||
if ":" in line:
|
||||
key, _, value = line.partition(":")
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
current = key
|
||||
if not value:
|
||||
meta[key] = []
|
||||
elif value.startswith("[") and value.endswith("]"):
|
||||
inner = value[1:-1].strip()
|
||||
meta[key] = [_scalar(v.strip()) for v in inner.split(",") if v.strip()] if inner else []
|
||||
else:
|
||||
meta[key] = _scalar(value)
|
||||
return meta
|
||||
|
||||
|
||||
def _scalar(value: str) -> Any:
|
||||
v = value.strip().strip('"').strip("'")
|
||||
if v.lower() in ("true", "false"):
|
||||
return v.lower() == "true"
|
||||
if re.fullmatch(r"-?\d+", v):
|
||||
return int(v)
|
||||
if re.fullmatch(r"-?\d+\.\d+", v):
|
||||
return float(v)
|
||||
return v
|
||||
|
||||
|
||||
def convert_wikilinks(text: str, *, embeds: bool = True) -> str:
|
||||
"""Turn Obsidian/Logseq ``[[link]]`` into Markdown links and ``![[img]]``
|
||||
into Markdown images so the block converter can render them."""
|
||||
|
||||
def repl(m: re.Match) -> str:
|
||||
bang, target, alias = m.group(1), m.group(2).strip(), m.group(3)
|
||||
label = (alias or target).strip()
|
||||
if bang == "!":
|
||||
return f"" if embeds else label
|
||||
return f"[{label}]({target})"
|
||||
|
||||
return _WIKILINK_RE.sub(repl, text)
|
||||
|
||||
|
||||
def normalize_title(value: Any) -> str:
|
||||
return str(value).strip() if value is not None else ""
|
||||
|
||||
|
||||
def coerce_tags(value: Any) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [str(v).strip().lstrip("#") for v in value if str(v).strip()]
|
||||
if isinstance(value, str):
|
||||
parts = re.split(r"[,\s]+", value)
|
||||
return [p.strip().lstrip("#") for p in parts if p.strip()]
|
||||
return [str(value)]
|
||||
|
||||
|
||||
def strip_markdown(text: str) -> str:
|
||||
text = re.sub(r"`{1,3}([^`]*)`{1,3}", r"\1", text)
|
||||
text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", text)
|
||||
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
|
||||
text = re.sub(r"[*_~#>]+", "", text)
|
||||
return text.strip()
|
||||
@@ -0,0 +1,147 @@
|
||||
"""FlowDeck — unified import framework (v5.6.0, Phase 0).
|
||||
|
||||
Defines the normalized data model shared by every importer and the registry
|
||||
used to auto-detect a source. An :class:`Importer` turns an uploaded file into
|
||||
an :class:`ImportResult` (pages, attachments, warnings, stats) which the
|
||||
pipeline (:mod:`app.services.importers.pipeline`) persists into FlowDeck.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
def decode_text(data: bytes) -> str:
|
||||
"""Best-effort decode of uploaded bytes (BOM aware, latin-1 fallback)."""
|
||||
for enc in ("utf-8-sig", "utf-8", "utf-16", "latin-1"):
|
||||
try:
|
||||
return data.decode(enc)
|
||||
except (UnicodeDecodeError, UnicodeError):
|
||||
continue
|
||||
return data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportAttachment:
|
||||
"""A binary asset extracted from an archive/vault."""
|
||||
|
||||
source_path: str
|
||||
filename: str
|
||||
data: bytes = b""
|
||||
mime: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportPage:
|
||||
"""One page to create. ``markdown`` is converted to blocks by the pipeline
|
||||
unless ``blocks`` is already provided. ``collection`` marks a database
|
||||
(Notion database, Excel sheet…) whose rows become ``collection_pages``."""
|
||||
|
||||
title: str = "Untitled"
|
||||
markdown: str = ""
|
||||
blocks: list[dict] = field(default_factory=list)
|
||||
source_path: str = ""
|
||||
parent_path: str = ""
|
||||
properties: dict[str, Any] = field(default_factory=dict)
|
||||
collection: dict[str, Any] | None = None
|
||||
external_id: str = ""
|
||||
page_id: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportResult:
|
||||
"""Normalized output of any importer."""
|
||||
|
||||
source: str = ""
|
||||
pages: list[ImportPage] = field(default_factory=list)
|
||||
attachments: list[ImportAttachment] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
stats: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def warn(self, message: str) -> None:
|
||||
if message and message not in self.warnings:
|
||||
self.warnings.append(message)
|
||||
|
||||
def finalize(self) -> ImportResult:
|
||||
self.stats.setdefault("pages", len(self.pages))
|
||||
self.stats.setdefault("collections", sum(1 for p in self.pages if p.collection))
|
||||
self.stats.setdefault("attachments", len(self.attachments))
|
||||
self.stats.setdefault("warnings", len(self.warnings))
|
||||
return self
|
||||
|
||||
|
||||
class Importer(ABC):
|
||||
"""Base class for a source importer."""
|
||||
|
||||
source_id: str = ""
|
||||
label: str = ""
|
||||
description: str = ""
|
||||
extensions: tuple[str, ...] = ()
|
||||
order: int = 100
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
"""Return True when this importer recognizes the uploaded file."""
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
"""Parse the upload into a normalized :class:`ImportResult`."""
|
||||
|
||||
def info(self) -> dict[str, Any]:
|
||||
return {
|
||||
"source_id": self.source_id,
|
||||
"label": self.label,
|
||||
"description": self.description,
|
||||
"extensions": list(self.extensions),
|
||||
}
|
||||
|
||||
|
||||
REGISTRY: list[Importer] = []
|
||||
|
||||
|
||||
def register_importer(cls: type[Importer]) -> type[Importer]:
|
||||
"""Class decorator registering an importer instance."""
|
||||
REGISTRY.append(cls())
|
||||
REGISTRY.sort(key=lambda i: i.order)
|
||||
return cls
|
||||
|
||||
|
||||
def all_importers() -> list[Importer]:
|
||||
return list(REGISTRY)
|
||||
|
||||
|
||||
def get_importer(source_id: str) -> Importer | None:
|
||||
for imp in REGISTRY:
|
||||
if imp.source_id == source_id:
|
||||
return imp
|
||||
return None
|
||||
|
||||
|
||||
def detect_importer(filename: str, data: bytes) -> Importer | None:
|
||||
"""First importer that recognizes the file, else None."""
|
||||
for imp in REGISTRY:
|
||||
try:
|
||||
if imp.detect(filename, data):
|
||||
return imp
|
||||
except Exception: # noqa: BLE001 - a broken detector must not break detection
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def list_sources() -> list[dict[str, Any]]:
|
||||
return [imp.info() for imp in REGISTRY]
|
||||
|
||||
|
||||
def make_collection(name: str, schema: list[dict], rows: list[dict],
|
||||
*, source_path: str = "", external_id: str = "") -> ImportPage:
|
||||
"""Build an ImportPage carrying a collection spec (database import).
|
||||
|
||||
``rows`` entries are ``{"title": str, "properties": {name: value}}``.
|
||||
"""
|
||||
return ImportPage(
|
||||
title=name or "Imported database",
|
||||
collection={"name": name or "Imported database", "schema": schema, "rows": rows},
|
||||
source_path=source_path or name,
|
||||
external_id=external_id or source_path or name,
|
||||
)
|
||||
@@ -0,0 +1,284 @@
|
||||
"""FlowDeck — bookmark importers (v5.6.0, Phase 4).
|
||||
|
||||
Raindrop.io, Pocket, Readwise, Shaarli and generic Netscape bookmark files are
|
||||
normalized into a FlowDeck collection (URL, description, tags, created date).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.services.importers._common import coerce_tags
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
make_collection,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_SCHEMA = [
|
||||
{"name": "Title", "type": "title"},
|
||||
{"name": "URL", "type": "url"},
|
||||
{"name": "Description", "type": "text"},
|
||||
{"name": "Tags", "type": "multi_select"},
|
||||
{"name": "Created", "type": "date"},
|
||||
]
|
||||
|
||||
_TAG_RE = re.compile(
|
||||
r"<h3[^>]*>(?P<folder>.*?)</h3>|<a\s+(?P<attrs>[^>]*?)>(?P<title>.*?)</a>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ATTR_RE = re.compile(r'([a-zA-Z_:-]+)\s*=\s*"([^"]*)"')
|
||||
|
||||
|
||||
def _strip_tags(value: str) -> str:
|
||||
return re.sub(r"<[^>]+>", "", value or "").strip()
|
||||
|
||||
|
||||
def _iso_from_epoch(value: Any) -> str:
|
||||
try:
|
||||
return datetime.fromtimestamp(int(str(value)[:10]), tz=UTC).date().isoformat()
|
||||
except (ValueError, TypeError, OSError, OverflowError):
|
||||
return ""
|
||||
|
||||
|
||||
def _iso(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if re.fullmatch(r"\d{10}", text):
|
||||
return _iso_from_epoch(text)
|
||||
return text[:10] if re.match(r"^\d{4}-\d{2}-\d{2}", text) else text
|
||||
|
||||
|
||||
def _row(title: str, url: str, description: str = "", tags=None, created: str = "") -> dict:
|
||||
props: dict[str, Any] = {}
|
||||
if url:
|
||||
props["URL"] = url
|
||||
if description:
|
||||
props["Description"] = description
|
||||
tag_list = coerce_tags(tags)
|
||||
if tag_list:
|
||||
props["Tags"] = tag_list
|
||||
if created:
|
||||
props["Created"] = created
|
||||
return {"title": (title or url or "Bookmark").strip()[:200], "properties": props}
|
||||
|
||||
|
||||
def parse_netscape(html: str) -> list[dict]:
|
||||
"""Parse a Netscape bookmark file (browser / Pocket / Raindrop HTML)."""
|
||||
rows: list[dict] = []
|
||||
folder = ""
|
||||
for match in _TAG_RE.finditer(html):
|
||||
if match.group("folder") is not None:
|
||||
folder = _strip_tags(match.group("folder"))
|
||||
continue
|
||||
attrs = dict(_ATTR_RE.findall(match.group("attrs") or ""))
|
||||
url = attrs.get("href") or attrs.get("HREF") or ""
|
||||
if not url:
|
||||
continue
|
||||
title = _strip_tags(match.group("title"))
|
||||
tags = attrs.get("tags") or attrs.get("TAGS") or folder
|
||||
rows.append(_row(title, url, tags=tags, created=_iso_from_epoch(attrs.get("add_date", ""))))
|
||||
return rows
|
||||
|
||||
|
||||
def _csv_rows(text: str) -> list[dict]:
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
return [{(k or "").strip().lower(): v for k, v in row.items()} for row in reader]
|
||||
|
||||
|
||||
class _BookmarkBase(Importer):
|
||||
source_id = "bookmarks"
|
||||
label = "Signets"
|
||||
description = ""
|
||||
order = 44
|
||||
keywords: tuple[str, ...] = ()
|
||||
|
||||
def _hint(self, filename: str, text: str) -> bool:
|
||||
low = filename.lower()
|
||||
return any(k in low or k in text.lower() for k in self.keywords)
|
||||
|
||||
|
||||
@register_importer
|
||||
class RaindropImporter(_BookmarkBase):
|
||||
source_id = "raindrop"
|
||||
label = "Raindrop.io"
|
||||
description = "Export Raindrop.io (CSV ou HTML) → collection de signets."
|
||||
extensions = (".csv", ".html", ".htm")
|
||||
order = 46
|
||||
keywords = ("raindrop",)
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
text = decode_text(data)
|
||||
if not self._hint(filename, text):
|
||||
return False
|
||||
return filename.lower().endswith((".csv", ".html", ".htm"))
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
text = decode_text(data)
|
||||
rows: list[dict] = []
|
||||
if filename.lower().endswith(".csv"):
|
||||
for r in _csv_rows(text):
|
||||
rows.append(_row(
|
||||
r.get("title", ""), r.get("url", ""),
|
||||
r.get("note") or r.get("excerpt") or "",
|
||||
r.get("tags", ""), _iso(r.get("created", "")),
|
||||
))
|
||||
else:
|
||||
rows = parse_netscape(text)
|
||||
result.pages.append(make_collection("Raindrop", _SCHEMA, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class PocketImporter(_BookmarkBase):
|
||||
source_id = "pocket"
|
||||
label = "Pocket"
|
||||
description = "Export Pocket (CSV ou HTML) → collection de signets."
|
||||
extensions = (".csv", ".html", ".htm")
|
||||
order = 45
|
||||
keywords = ("pocket",)
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
text = decode_text(data)
|
||||
if not self._hint(filename, text):
|
||||
return False
|
||||
return filename.lower().endswith((".csv", ".html", ".htm"))
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
text = decode_text(data)
|
||||
rows: list[dict] = []
|
||||
if filename.lower().endswith(".csv"):
|
||||
for r in _csv_rows(text):
|
||||
rows.append(_row(
|
||||
r.get("title", ""), r.get("url", ""),
|
||||
"", r.get("tags", ""), _iso(r.get("time_added", "")),
|
||||
))
|
||||
else:
|
||||
rows = parse_netscape(text)
|
||||
result.pages.append(make_collection("Pocket", _SCHEMA, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class ReadwiseImporter(_BookmarkBase):
|
||||
source_id = "readwise"
|
||||
label = "Readwise"
|
||||
description = "Export Readwise (highlights CSV) → collection de surlignages."
|
||||
extensions = (".csv", ".md", ".markdown")
|
||||
order = 47
|
||||
keywords = ("readwise",)
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
text = decode_text(data)
|
||||
if not self._hint(filename, text):
|
||||
return False
|
||||
if filename.lower().endswith(".csv"):
|
||||
header = text.splitlines()[0].lower() if text.strip() else ""
|
||||
return "highlight" in header or "book title" in header
|
||||
return True
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
text = decode_text(data)
|
||||
schema = [
|
||||
{"name": "Highlight", "type": "title"},
|
||||
{"name": "Book", "type": "text"},
|
||||
{"name": "Author", "type": "text"},
|
||||
{"name": "Note", "type": "text"},
|
||||
{"name": "Tags", "type": "multi_select"},
|
||||
{"name": "Highlighted at", "type": "date"},
|
||||
]
|
||||
rows: list[dict] = []
|
||||
if filename.lower().endswith(".csv"):
|
||||
for r in _csv_rows(text):
|
||||
props: dict[str, Any] = {}
|
||||
if r.get("book title"):
|
||||
props["Book"] = r["book title"]
|
||||
if r.get("book author"):
|
||||
props["Author"] = r["book author"]
|
||||
if r.get("note"):
|
||||
props["Note"] = r["note"]
|
||||
tags = coerce_tags(r.get("document tags") or r.get("tags"))
|
||||
if tags:
|
||||
props["Tags"] = tags
|
||||
created = _iso(r.get("highlighted at", ""))
|
||||
if created:
|
||||
props["Highlighted at"] = created
|
||||
rows.append({"title": (r.get("highlight") or "Highlight").strip()[:200], "properties": props})
|
||||
else:
|
||||
rows = [{"title": ln.lstrip("-* ").strip()[:200], "properties": {}} for ln in text.splitlines() if ln.strip()]
|
||||
result.pages.append(make_collection("Readwise", schema, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class ShaarliImporter(_BookmarkBase):
|
||||
source_id = "shaarli"
|
||||
label = "Shaarli"
|
||||
description = "Export Shaarli (JSON) → collection de signets."
|
||||
extensions = (".json",)
|
||||
order = 48
|
||||
keywords = ("shaarli",)
|
||||
|
||||
def _records(self, data: bytes) -> list[dict] | None:
|
||||
try:
|
||||
obj = json.loads(decode_text(data))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if isinstance(obj, dict) and isinstance(obj.get("links"), list):
|
||||
obj = obj["links"]
|
||||
if isinstance(obj, list) and obj and all(isinstance(x, dict) and x.get("url") for x in obj):
|
||||
return obj
|
||||
return None
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if not filename.lower().endswith(".json"):
|
||||
return False
|
||||
return self._records(data) is not None
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
records = self._records(data) or []
|
||||
rows = [
|
||||
_row(r.get("title", ""), r.get("url", ""), r.get("description", ""),
|
||||
r.get("tags", ""), _iso(r.get("created", "")))
|
||||
for r in records
|
||||
]
|
||||
result.pages.append(make_collection("Shaarli", _SCHEMA, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class BookmarksImporter(_BookmarkBase):
|
||||
source_id = "bookmarks"
|
||||
label = "Signets HTML (navigateur)"
|
||||
description = "Fichier de signets Netscape HTML (Chrome/Firefox/Edge…)."
|
||||
extensions = (".html", ".htm")
|
||||
order = 49
|
||||
keywords = ()
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if not filename.lower().endswith((".html", ".htm")):
|
||||
return False
|
||||
text = decode_text(data)[:4000].lower()
|
||||
return "netscape-bookmark-file" in text or "<dt><a href" in text
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
rows = parse_netscape(decode_text(data))
|
||||
result.pages.append(make_collection("Bookmarks", _SCHEMA, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,138 @@
|
||||
"""FlowDeck — iCalendar (.ics) importer (v5.6.0, Phase 4).
|
||||
|
||||
Parses VEVENT blocks (RFC 5545, best-effort) into a FlowDeck calendar
|
||||
collection (start/end, all-day, location, description).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
make_collection,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_SCHEMA = [
|
||||
{"name": "Title", "type": "title"},
|
||||
{"name": "Start", "type": "date"},
|
||||
{"name": "End", "type": "date"},
|
||||
{"name": "All day", "type": "checkbox"},
|
||||
{"name": "Location", "type": "text"},
|
||||
{"name": "Description", "type": "text"},
|
||||
{"name": "Calendar", "type": "text"},
|
||||
]
|
||||
_UNESCAPE = [("\\n", "\n"), ("\\N", "\n"), ("\\,", ","), ("\\;", ";"), ("\\\\", "\\")]
|
||||
|
||||
|
||||
def _unfold(text: str) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for raw in text.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
|
||||
if raw[:1] in (" ", "\t") and lines:
|
||||
lines[-1] += raw[1:]
|
||||
else:
|
||||
lines.append(raw)
|
||||
return lines
|
||||
|
||||
|
||||
def _unescape(value: str) -> str:
|
||||
for src, dst in _UNESCAPE:
|
||||
value = value.replace(src, dst)
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _parse_prop(line: str) -> tuple[str, dict[str, str], str]:
|
||||
if ":" not in line:
|
||||
return "", {}, ""
|
||||
head, _, value = line.partition(":")
|
||||
parts = head.split(";")
|
||||
name = parts[0].upper()
|
||||
params: dict[str, str] = {}
|
||||
for p in parts[1:]:
|
||||
if "=" in p:
|
||||
k, _, v = p.partition("=")
|
||||
params[k.upper()] = v
|
||||
return name, params, value
|
||||
|
||||
|
||||
def _iso_datetime(value: str, params: dict[str, str]) -> tuple[str, bool]:
|
||||
"""Return (iso, is_all_day)."""
|
||||
v = value.strip()
|
||||
if params.get("VALUE") == "DATE" or re.fullmatch(r"\d{8}", v):
|
||||
m = re.fullmatch(r"(\d{4})(\d{2})(\d{2})", v)
|
||||
return (f"{m.group(1)}-{m.group(2)}-{m.group(3)}", True) if m else ("", True)
|
||||
m = re.fullmatch(r"(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(Z?)", v)
|
||||
if not m:
|
||||
return v, False
|
||||
date = f"{m.group(1)}-{m.group(2)}-{m.group(3)}T{m.group(4)}:{m.group(5)}:{m.group(6)}"
|
||||
return (date + "+00:00" if m.group(7) else date), False
|
||||
|
||||
|
||||
@register_importer
|
||||
class IcsImporter(Importer):
|
||||
source_id = "ics"
|
||||
label = "Calendrier (.ics)"
|
||||
description = "Export iCalendar (Google/Outlook/Apple) → collection d'événements."
|
||||
extensions = (".ics", ".ical")
|
||||
order = 33
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if filename.lower().endswith((".ics", ".ical")):
|
||||
return True
|
||||
return "BEGIN:VCALENDAR" in decode_text(data)[:2000]
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
lines = _unfold(decode_text(data))
|
||||
calendar = ""
|
||||
rows: list[dict] = []
|
||||
event: dict[str, Any] | None = None
|
||||
for line in lines:
|
||||
upper = line.strip().upper()
|
||||
if upper == "BEGIN:VEVENT":
|
||||
event = {}
|
||||
continue
|
||||
if upper == "END:VEVENT":
|
||||
if event is not None:
|
||||
rows.append(_event_row(event, calendar))
|
||||
event = None
|
||||
continue
|
||||
name, params, value = _parse_prop(line.strip())
|
||||
if name == "X-WR-CALNAME" and not event:
|
||||
calendar = _unescape(value)
|
||||
if event is None:
|
||||
continue
|
||||
if name == "SUMMARY":
|
||||
event["title"] = _unescape(value)
|
||||
elif name == "DTSTART":
|
||||
event["start"], event["all_day"] = _iso_datetime(value, params)
|
||||
elif name == "DTEND":
|
||||
event["end"], _ = _iso_datetime(value, params)
|
||||
elif name == "LOCATION":
|
||||
event["location"] = _unescape(value)
|
||||
elif name == "DESCRIPTION":
|
||||
event["description"] = _unescape(value)
|
||||
elif name == "UID":
|
||||
event["uid"] = value
|
||||
result.pages.append(make_collection("Calendar", _SCHEMA, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
def _event_row(event: dict[str, Any], calendar: str) -> dict:
|
||||
props: dict[str, Any] = {}
|
||||
if event.get("start"):
|
||||
props["Start"] = event["start"]
|
||||
if event.get("end"):
|
||||
props["End"] = event["end"]
|
||||
props["All day"] = bool(event.get("all_day"))
|
||||
if event.get("location"):
|
||||
props["Location"] = event["location"]
|
||||
if event.get("description"):
|
||||
props["Description"] = event["description"]
|
||||
if calendar:
|
||||
props["Calendar"] = calendar
|
||||
return {"title": (event.get("title") or "Event").strip()[:200], "properties": props}
|
||||
@@ -0,0 +1,122 @@
|
||||
"""FlowDeck — Word (.docx) importer (v5.6.0, Phase 3).
|
||||
|
||||
Converts a Word document (including Google Docs Takeout ``.docx`` exports) into
|
||||
a FlowDeck page: headings, lists, tables and inline images.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.importers.base import (
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportResult,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_HEADING_STYLES = {
|
||||
"title": 1, "heading 1": 1, "heading 2": 2, "heading 3": 3,
|
||||
"heading 4": 4, "heading 5": 4, "heading 6": 4,
|
||||
}
|
||||
_MIME = {
|
||||
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
|
||||
".emf": "image/emf", ".wmf": "image/wmf", ".tiff": "image/tiff",
|
||||
}
|
||||
|
||||
|
||||
def _escape_cell(text: str) -> str:
|
||||
return text.strip().replace("|", "\\|").replace("\n", " ")
|
||||
|
||||
|
||||
def _table_markdown(table) -> str:
|
||||
rows: list[list[str]] = []
|
||||
for row in table.rows:
|
||||
rows.append([_escape_cell(cell.text) for cell in row.cells])
|
||||
if not rows:
|
||||
return ""
|
||||
width = max(len(r) for r in rows)
|
||||
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||
header = "| " + " | ".join(rows[0]) + " |"
|
||||
sep = "| " + " | ".join(["---"] * width) + " |"
|
||||
body = "\n".join("| " + " | ".join(r) + " |" for r in rows[1:])
|
||||
return "\n".join(x for x in (header, sep, body) if x)
|
||||
|
||||
|
||||
@register_importer
|
||||
class DocxImporter(Importer):
|
||||
source_id = "docx"
|
||||
label = "Word / Google Docs (.docx)"
|
||||
description = "Document Word : titres, listes, tableaux et images."
|
||||
extensions = (".docx", ".docm")
|
||||
order = 36
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
return filename.lower().endswith((".docx", ".docm"))
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
try:
|
||||
from docx import Document
|
||||
from docx.oxml.ns import qn
|
||||
except ImportError:
|
||||
result.warn("python-docx n'est pas installé : import Word indisponible")
|
||||
return result.finalize()
|
||||
try:
|
||||
doc = Document(io.BytesIO(data))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Document illisible : {exc}")
|
||||
return result.finalize()
|
||||
|
||||
lines: list[str] = []
|
||||
for para in doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
style = (para.style.name or "").lower() if para.style else ""
|
||||
images = self._paragraph_images(doc, para, qn, result)
|
||||
if text:
|
||||
level = _HEADING_STYLES.get(style)
|
||||
if level:
|
||||
lines.append("#" * level + " " + text)
|
||||
elif "list bullet" in style or "list paragraph" in style:
|
||||
lines.append("- " + text)
|
||||
elif "list number" in style:
|
||||
lines.append("1. " + text)
|
||||
elif style == "quote":
|
||||
lines.append("> " + text)
|
||||
else:
|
||||
lines.append(text)
|
||||
lines.extend(images)
|
||||
for table in doc.tables:
|
||||
md = _table_markdown(table)
|
||||
if md:
|
||||
lines.append(md)
|
||||
|
||||
markdown = re.sub(r"\n{3,}", "\n\n", "\n\n".join(lines)).strip()
|
||||
title = Path(filename).stem or "Document"
|
||||
result.pages.append(_page(title, markdown, filename))
|
||||
return result.finalize()
|
||||
|
||||
def _paragraph_images(self, doc, para, qn, result: ImportResult) -> list[str]:
|
||||
images: list[str] = []
|
||||
for blip in para._p.iter(qn("a:blip")):
|
||||
rid = blip.get(qn("r:embed")) or blip.get(qn("r:link"))
|
||||
if not rid:
|
||||
continue
|
||||
part = doc.part.related_parts.get(rid)
|
||||
if part is None or not hasattr(part, "blob"):
|
||||
continue
|
||||
name = Path(str(part.partname)).name or f"image_{len(result.attachments)}.png"
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=name, filename=name, data=part.blob,
|
||||
mime=_MIME.get(Path(name).suffix.lower(), "application/octet-stream"),
|
||||
))
|
||||
images.append(f"")
|
||||
return images
|
||||
|
||||
|
||||
def _page(title: str, markdown: str, filename: str):
|
||||
from app.services.importers.base import ImportPage
|
||||
|
||||
return ImportPage(title=title, markdown=markdown, source_path=filename, external_id=filename)
|
||||
@@ -0,0 +1,232 @@
|
||||
"""FlowDeck — forge (Gitea/GitHub) issues importer (v5.6.0, Phase 4).
|
||||
|
||||
Pulls a repository's issues, labels and milestones through a forge adapter and
|
||||
normalizes them into FlowDeck collections.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.services.importers.base import ImportResult, make_collection
|
||||
|
||||
|
||||
def _label_names(issue: dict) -> list[str]:
|
||||
labels = issue.get("labels") or []
|
||||
names: list[str] = []
|
||||
for label in labels:
|
||||
if isinstance(label, dict):
|
||||
name = label.get("name") or label.get("title")
|
||||
else:
|
||||
name = str(label)
|
||||
if name and name not in names:
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def _milestone_name(issue: dict) -> str:
|
||||
milestone = issue.get("milestone")
|
||||
if isinstance(milestone, dict):
|
||||
return str(milestone.get("title") or milestone.get("name") or "")
|
||||
return str(milestone or "")
|
||||
|
||||
|
||||
def _assignees(issue: dict) -> list[str]:
|
||||
out: list[str] = []
|
||||
for key in ("assignees", "assignee"):
|
||||
value = issue.get(key)
|
||||
if isinstance(value, list):
|
||||
for a in value:
|
||||
login = a.get("login") if isinstance(a, dict) else str(a)
|
||||
if login and login not in out:
|
||||
out.append(login)
|
||||
elif isinstance(value, dict):
|
||||
login = value.get("login")
|
||||
if login and login not in out:
|
||||
out.append(login)
|
||||
elif isinstance(value, str) and value and value not in out:
|
||||
out.append(value)
|
||||
return out
|
||||
|
||||
|
||||
_ISSUE_SCHEMA = [
|
||||
{"name": "Title", "type": "title"},
|
||||
{"name": "Number", "type": "number"},
|
||||
{"name": "State", "type": "select", "options": [
|
||||
{"name": "open", "color": "green"}, {"name": "closed", "color": "red"},
|
||||
]},
|
||||
{"name": "Labels", "type": "multi_select"},
|
||||
{"name": "Milestone", "type": "text"},
|
||||
{"name": "Assignee", "type": "text"},
|
||||
{"name": "Created", "type": "date"},
|
||||
{"name": "Updated", "type": "date"},
|
||||
{"name": "URL", "type": "url"},
|
||||
{"name": "Body", "type": "text"},
|
||||
]
|
||||
|
||||
|
||||
def build_issues_result(
|
||||
issues: list[dict],
|
||||
*,
|
||||
labels: list[dict] | None = None,
|
||||
milestones: list[dict] | None = None,
|
||||
owner: str = "",
|
||||
repo: str = "",
|
||||
provider: str = "",
|
||||
) -> ImportResult:
|
||||
"""Normalize forge issues/labels/milestones into an ImportResult."""
|
||||
result = ImportResult(source=f"forge:{provider}" if provider else "forge")
|
||||
name = f"{owner}/{repo} issues".strip("/ ") or "Issues"
|
||||
rows: list[dict] = []
|
||||
for issue in issues:
|
||||
if issue.get("pull_request"):
|
||||
continue
|
||||
props: dict[str, Any] = {}
|
||||
if issue.get("number") is not None:
|
||||
props["Number"] = issue["number"]
|
||||
if issue.get("state"):
|
||||
props["State"] = issue["state"]
|
||||
label_names = _label_names(issue)
|
||||
if label_names:
|
||||
props["Labels"] = label_names
|
||||
milestone = _milestone_name(issue)
|
||||
if milestone:
|
||||
props["Milestone"] = milestone
|
||||
assignees = _assignees(issue)
|
||||
if assignees:
|
||||
props["Assignee"] = ", ".join(assignees)
|
||||
if issue.get("created_at"):
|
||||
props["Created"] = issue["created_at"]
|
||||
if issue.get("updated_at"):
|
||||
props["Updated"] = issue["updated_at"]
|
||||
if issue.get("html_url"):
|
||||
props["URL"] = issue["html_url"]
|
||||
if issue.get("body"):
|
||||
props["Body"] = issue["body"]
|
||||
title = issue.get("title") or f"#{issue.get('number', '')}".strip()
|
||||
rows.append({"title": title[:200], "properties": props})
|
||||
|
||||
result.pages.append(make_collection(
|
||||
name, _ISSUE_SCHEMA, rows,
|
||||
source_path=f"{provider}:{owner}/{repo}:issues",
|
||||
external_id=f"{provider}:{owner}/{repo}:issues",
|
||||
))
|
||||
result.stats["rows"] = len(rows)
|
||||
|
||||
if labels:
|
||||
label_schema = [
|
||||
{"name": "Name", "type": "title"},
|
||||
{"name": "Color", "type": "text"},
|
||||
{"name": "Description", "type": "text"},
|
||||
]
|
||||
label_rows = [{
|
||||
"title": (lbl.get("name") or lbl.get("title") or "Label")[:200],
|
||||
"properties": {
|
||||
k: v for k, v in (
|
||||
("Color", lbl.get("color")),
|
||||
("Description", lbl.get("description")),
|
||||
) if v
|
||||
},
|
||||
} for lbl in labels]
|
||||
result.pages.append(make_collection(
|
||||
f"{owner}/{repo} labels".strip("/ "), label_schema, label_rows,
|
||||
source_path=f"{provider}:{owner}/{repo}:labels",
|
||||
external_id=f"{provider}:{owner}/{repo}:labels",
|
||||
))
|
||||
|
||||
if milestones:
|
||||
ms_schema = [
|
||||
{"name": "Title", "type": "title"},
|
||||
{"name": "State", "type": "select", "options": [
|
||||
{"name": "open", "color": "green"}, {"name": "closed", "color": "red"},
|
||||
]},
|
||||
{"name": "Due date", "type": "date"},
|
||||
{"name": "Description", "type": "text"},
|
||||
]
|
||||
ms_rows = []
|
||||
for ms in milestones:
|
||||
props: dict[str, Any] = {}
|
||||
if ms.get("state"):
|
||||
props["State"] = ms["state"]
|
||||
if ms.get("due_on"):
|
||||
props["Due date"] = ms["due_on"]
|
||||
if ms.get("description"):
|
||||
props["Description"] = ms["description"]
|
||||
ms_rows.append({"title": (ms.get("title") or "Milestone")[:200], "properties": props})
|
||||
result.pages.append(make_collection(
|
||||
f"{owner}/{repo} milestones".strip("/ "), ms_schema, ms_rows,
|
||||
source_path=f"{provider}:{owner}/{repo}:milestones",
|
||||
external_id=f"{provider}:{owner}/{repo}:milestones",
|
||||
))
|
||||
|
||||
return result.finalize()
|
||||
|
||||
|
||||
class GiteaForgeAdapter:
|
||||
"""Adapts a :class:`GiteaClient` to the ``list_*`` interface used here."""
|
||||
|
||||
def __init__(self, client) -> None:
|
||||
self._client = client
|
||||
|
||||
async def list_issues(self, owner: str, repo: str, state: str = "all") -> list[dict]:
|
||||
issues: list[dict] = []
|
||||
for page in range(1, 6):
|
||||
batch = await self._client.get_issues(owner, repo, state=state, page=page, limit=50)
|
||||
if not batch:
|
||||
break
|
||||
issues.extend(batch)
|
||||
if len(batch) < 50:
|
||||
break
|
||||
return issues
|
||||
|
||||
async def list_labels(self, owner: str, repo: str) -> list[dict]:
|
||||
return await self._client.get_labels(owner, repo)
|
||||
|
||||
async def list_milestones(self, owner: str, repo: str, state: str = "all") -> list[dict]:
|
||||
return await self._client.get_milestones(owner, repo, state=state)
|
||||
|
||||
async def list_repo_files(self, owner: str, repo: str, path: str = "") -> list[dict]:
|
||||
"""Recursively flatten Gitea repo contents into file entries."""
|
||||
files: list[dict] = []
|
||||
pending = [path.strip("/")]
|
||||
while pending and len(files) < 5000:
|
||||
current = pending.pop()
|
||||
items = await self._client.get_repo_contents(owner, repo, current)
|
||||
for item in items:
|
||||
if item.get("type") == "dir":
|
||||
pending.append(item.get("path") or item.get("name"))
|
||||
elif item.get("type") == "file":
|
||||
files.append({"path": item.get("path") or item.get("name"), "size": item.get("size", 0)})
|
||||
return files
|
||||
|
||||
async def get_file_content(self, owner: str, repo: str, path: str) -> str:
|
||||
return await self._client.get_file_content(owner, repo, path)
|
||||
|
||||
|
||||
async def fetch_forge_issues(
|
||||
adapter,
|
||||
owner: str,
|
||||
repo: str,
|
||||
*,
|
||||
provider: str = "",
|
||||
state: str = "all",
|
||||
include_labels: bool = True,
|
||||
include_milestones: bool = True,
|
||||
) -> ImportResult:
|
||||
"""Fetch issues (and optionally labels/milestones) then normalize them."""
|
||||
issues = await adapter.list_issues(owner, repo, state)
|
||||
labels = None
|
||||
milestones = None
|
||||
if include_labels:
|
||||
try:
|
||||
labels = await adapter.list_labels(owner, repo)
|
||||
except Exception: # noqa: BLE001 - labels are optional
|
||||
labels = None
|
||||
if include_milestones:
|
||||
try:
|
||||
milestones = await adapter.list_milestones(owner, repo, state)
|
||||
except Exception: # noqa: BLE001
|
||||
milestones = None
|
||||
return build_issues_result(
|
||||
issues, labels=labels, milestones=milestones,
|
||||
owner=owner, repo=repo, provider=provider,
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""FlowDeck — forge repository file importer (v5.6.0, Phase 5).
|
||||
|
||||
Imports a Gitea/GitHub repository's text files as pages, preserving the folder
|
||||
hierarchy. Markdown files become pages; other text files become code blocks.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.export import _CODE_LANG, _TEXTUAL_EXTS
|
||||
from app.services.importers.base import ImportPage, ImportResult
|
||||
|
||||
_MD_EXTS = {"md", "markdown"}
|
||||
|
||||
|
||||
def _ext(path: str) -> str:
|
||||
return Path(path).suffix.lower().lstrip(".")
|
||||
|
||||
|
||||
def build_repo_result(
|
||||
files: list[tuple[str, str]],
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
provider: str = "",
|
||||
) -> ImportResult:
|
||||
"""Turn ``[(path, content)]`` into pages with folder hierarchy."""
|
||||
result = ImportResult(source=f"forge-repo:{provider}" if provider else "forge-repo")
|
||||
for path, content in files:
|
||||
clean = path.replace("\\", "/").strip("/")
|
||||
if not clean:
|
||||
continue
|
||||
ext = _ext(clean)
|
||||
if ext in _MD_EXTS:
|
||||
markdown = content
|
||||
else:
|
||||
lang = _CODE_LANG.get(ext, "")
|
||||
markdown = f"```{lang}\n{content.rstrip()}\n```"
|
||||
parts = clean.split("/")
|
||||
result.pages.append(ImportPage(
|
||||
title=parts[-1] or clean,
|
||||
markdown=markdown,
|
||||
source_path=clean,
|
||||
parent_path="/".join(parts[:-1]),
|
||||
external_id=f"{provider}:{owner}/{repo}:{clean}",
|
||||
))
|
||||
result.stats["rows"] = len(result.pages)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
async def fetch_forge_repo(
|
||||
adapter,
|
||||
owner: str,
|
||||
repo: str,
|
||||
*,
|
||||
provider: str = "",
|
||||
path: str = "",
|
||||
max_files: int = 200,
|
||||
max_file_bytes: int = 512_000,
|
||||
) -> ImportResult:
|
||||
"""List a repo's files and fetch the textual ones."""
|
||||
try:
|
||||
metas = await adapter.list_repo_files(owner, repo, path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result = ImportResult(source=f"forge-repo:{provider}" if provider else "forge-repo")
|
||||
result.warn(f"Arborescence illisible : {exc}")
|
||||
return result.finalize()
|
||||
|
||||
files: list[tuple[str, str]] = []
|
||||
for meta in metas:
|
||||
file_path = meta.get("path") or ""
|
||||
if _ext(file_path) not in _TEXTUAL_EXTS:
|
||||
continue
|
||||
if int(meta.get("size") or 0) > max_file_bytes:
|
||||
continue
|
||||
if len(files) >= max_files:
|
||||
break
|
||||
try:
|
||||
content = await adapter.get_file_content(owner, repo, file_path)
|
||||
except Exception: # noqa: BLE001 - skip unreadable files
|
||||
continue
|
||||
if not content or content == "[binary file]":
|
||||
continue
|
||||
files.append((file_path, content))
|
||||
return build_repo_result(files, owner=owner, repo=repo, provider=provider)
|
||||
@@ -0,0 +1,300 @@
|
||||
"""FlowDeck — HTML notes & Google Keep importer (v5.6.0, Phase 1).
|
||||
|
||||
Covers HTML exports from Apple Notes, Bear, Ulysses and OneNote, plus the
|
||||
Google Takeout ``Keep`` JSON/HTML format. HTML is converted to Markdown and then
|
||||
to FlowDeck blocks by the pipeline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
|
||||
from app.services.importers._common import coerce_tags, normalize_title
|
||||
from app.services.importers.base import (
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_HTML_EXTS = (".html", ".htm")
|
||||
|
||||
|
||||
def _inline(node: Tag) -> str:
|
||||
out: list[str] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, NavigableString):
|
||||
out.append(str(child))
|
||||
elif isinstance(child, Tag):
|
||||
name = child.name.lower()
|
||||
if name in ("strong", "b"):
|
||||
out.append(f"**{_inline(child).strip()}**")
|
||||
elif name in ("em", "i"):
|
||||
out.append(f"*{_inline(child).strip()}*")
|
||||
elif name == "code":
|
||||
out.append(f"`{child.get_text()}`")
|
||||
elif name == "br":
|
||||
out.append("\n")
|
||||
elif name == "a":
|
||||
href = child.get("href", "")
|
||||
label = _inline(child).strip() or href
|
||||
out.append(f"[{label}]({href})" if href else label)
|
||||
elif name == "img":
|
||||
src = child.get("src", "")
|
||||
alt = child.get("alt", "")
|
||||
out.append(f"" if src else "")
|
||||
elif name in ("del", "s", "strike"):
|
||||
out.append(f"~~{_inline(child).strip()}~~")
|
||||
else:
|
||||
out.append(_inline(child))
|
||||
return re.sub(r"[ \t]+", " ", "".join(out))
|
||||
|
||||
|
||||
def _table(node: Tag) -> str:
|
||||
rows: list[list[str]] = []
|
||||
for tr in node.find_all("tr"):
|
||||
cells = tr.find_all(["th", "td"])
|
||||
rows.append([_inline(c).strip().replace("|", "\\|") for c in cells])
|
||||
if not rows:
|
||||
return ""
|
||||
width = max(len(r) for r in rows)
|
||||
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||
header = "| " + " | ".join(rows[0]) + " |"
|
||||
sep = "| " + " | ".join(["---"] * width) + " |"
|
||||
body = "\n".join("| " + " | ".join(r) + " |" for r in rows[1:])
|
||||
return "\n".join(x for x in (header, sep, body) if x)
|
||||
|
||||
|
||||
def _block(node: Tag, depth: int = 0) -> str:
|
||||
name = node.name.lower()
|
||||
if name in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
return "#" * int(name[1]) + " " + _inline(node).strip()
|
||||
if name == "p":
|
||||
return _inline(node).strip()
|
||||
if name in ("ul", "ol"):
|
||||
lines = []
|
||||
for i, li in enumerate(node.find_all("li", recursive=False)):
|
||||
marker = f"{i + 1}." if name == "ol" else "-"
|
||||
text = _inline(li).strip()
|
||||
lines.append(f"{' ' * depth}{marker} {text}")
|
||||
return "\n".join(lines)
|
||||
if name == "blockquote":
|
||||
return "\n".join(f"> {ln}" for ln in _inline(node).strip().splitlines())
|
||||
if name == "pre":
|
||||
code = node.get_text()
|
||||
lang = ""
|
||||
cls = " ".join(node.get("class", [])) if node.get("class") else ""
|
||||
m = re.search(r"(?:language|lang)-([\w+-]+)", cls)
|
||||
if m:
|
||||
lang = m.group(1)
|
||||
return f"```{lang}\n{code.rstrip()}\n```"
|
||||
if name == "hr":
|
||||
return "---"
|
||||
if name == "table":
|
||||
return _table(node)
|
||||
if name == "img":
|
||||
src = node.get("src", "")
|
||||
return f"" if src else ""
|
||||
if name in ("div", "section", "article", "body", "main", "html", "span", "font", "center"):
|
||||
inner = "\n\n".join(
|
||||
_block(c, depth) for c in node.children if isinstance(c, Tag)
|
||||
).strip()
|
||||
if inner:
|
||||
return inner
|
||||
text = _inline(node).strip()
|
||||
return text
|
||||
return _inline(node).strip()
|
||||
|
||||
|
||||
def _html_to_markdown(html: str) -> str:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "head", "nav", "footer"]):
|
||||
tag.decompose()
|
||||
root = soup.body or soup
|
||||
blocks = [_block(c) for c in root.children if isinstance(c, Tag)]
|
||||
md = "\n\n".join(b for b in blocks if b and b.strip())
|
||||
return re.sub(r"\n{3,}", "\n\n", md).strip()
|
||||
|
||||
|
||||
def _title_from_html(html: str, fallback: str) -> str:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
if soup.title and soup.title.string:
|
||||
return soup.title.string.strip()
|
||||
h1 = soup.find(["h1", "h2"])
|
||||
if h1:
|
||||
return h1.get_text().strip()
|
||||
return fallback
|
||||
|
||||
|
||||
@register_importer
|
||||
class HtmlNotesImporter(Importer):
|
||||
source_id = "html_notes"
|
||||
label = "HTML (Apple Notes, Bear, Ulysses, OneNote)"
|
||||
description = "Fichiers HTML ou archive .zip (notes exportées en HTML)."
|
||||
extensions = (".html", ".htm", ".zip")
|
||||
order = 50
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
low = filename.lower()
|
||||
if low.endswith(_HTML_EXTS):
|
||||
return True
|
||||
if low.endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return False
|
||||
names = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
return any(n.lower().endswith(_HTML_EXTS) for n in names)
|
||||
return False
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
entries: list[tuple[str, bytes]] = []
|
||||
if filename.lower().endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
result.warn(f"Archive invalide : {exc}")
|
||||
return result.finalize()
|
||||
for name in zf.namelist():
|
||||
if name.endswith("/"):
|
||||
continue
|
||||
clean = name.replace("\\", "/")
|
||||
if clean.lower().endswith(_HTML_EXTS):
|
||||
entries.append((clean, zf.read(name)))
|
||||
else:
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=clean,
|
||||
filename=clean.rsplit("/", 1)[-1],
|
||||
data=zf.read(name),
|
||||
))
|
||||
else:
|
||||
entries.append((filename, data))
|
||||
|
||||
for name, payload in entries:
|
||||
html = decode_text(payload)
|
||||
fallback = name.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
parts = name.replace("\\", "/").split("/")
|
||||
result.pages.append(ImportPage(
|
||||
title=_title_from_html(html, fallback) or "Untitled",
|
||||
markdown=_html_to_markdown(html),
|
||||
source_path=name,
|
||||
parent_path="/".join(parts[:-1]),
|
||||
external_id=name,
|
||||
))
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class GoogleKeepImporter(Importer):
|
||||
source_id = "google_keep"
|
||||
label = "Google Keep (Takeout)"
|
||||
description = "Export Google Takeout : Keep/*.json (notes, listes, labels, pièces jointes)."
|
||||
extensions = (".json", ".zip")
|
||||
order = 40
|
||||
|
||||
def _is_keep_json(self, data: bytes) -> bool:
|
||||
try:
|
||||
obj = json.loads(decode_text(data))
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
return isinstance(obj, dict) and any(
|
||||
k in obj for k in ("textContent", "listContent", "isTrashed", "color")
|
||||
)
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
low = filename.lower()
|
||||
if low.endswith(".json"):
|
||||
return self._is_keep_json(data)
|
||||
if low.endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return False
|
||||
for n in zf.namelist():
|
||||
if n.lower().endswith(".json") and "keep" in n.lower():
|
||||
try:
|
||||
if self._is_keep_json(zf.read(n)):
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return False
|
||||
|
||||
def _page_from_keep(self, obj: dict, name: str) -> ImportPage | None:
|
||||
if obj.get("isTrashed"):
|
||||
return None
|
||||
title = normalize_title(obj.get("title"))
|
||||
lines: list[str] = []
|
||||
for item in obj.get("listContent") or []:
|
||||
mark = "x" if item.get("isChecked") else " "
|
||||
lines.append(f"- [{mark}] {item.get('text', '')}")
|
||||
if obj.get("textContent"):
|
||||
lines.insert(0, obj["textContent"])
|
||||
body = "\n\n".join(lines)
|
||||
if not title:
|
||||
first = next((ln for ln in body.splitlines() if ln.strip()), "")
|
||||
first = re.sub(r"^[-*+]\s*(\[[ xX]\]\s*)?", "", first).strip()
|
||||
title = first[:60] or "Note"
|
||||
labels = coerce_tags(obj.get("labels"))
|
||||
props = {"tags": labels} if labels else {}
|
||||
return ImportPage(
|
||||
title=title,
|
||||
markdown=body,
|
||||
source_path=name,
|
||||
parent_path="",
|
||||
properties=props,
|
||||
external_id=name,
|
||||
)
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
if filename.lower().endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
result.warn(f"Archive invalide : {exc}")
|
||||
return result.finalize()
|
||||
for name in zf.namelist():
|
||||
if name.endswith("/"):
|
||||
continue
|
||||
clean = name.replace("\\", "/")
|
||||
if clean.lower().endswith(".json") and "keep" in clean.lower():
|
||||
try:
|
||||
obj = json.loads(decode_text(zf.read(name)))
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
page = self._page_from_keep(obj, clean)
|
||||
if page:
|
||||
result.pages.append(page)
|
||||
elif "/keep/" in clean.lower() and not clean.lower().endswith(".json"):
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=clean,
|
||||
filename=clean.rsplit("/", 1)[-1],
|
||||
data=zf.read(name),
|
||||
))
|
||||
return result.finalize()
|
||||
|
||||
try:
|
||||
obj = json.loads(decode_text(data))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"JSON invalide : {exc}")
|
||||
return result.finalize()
|
||||
if isinstance(obj, list):
|
||||
for i, item in enumerate(obj):
|
||||
if isinstance(item, dict):
|
||||
page = self._page_from_keep(item, f"{filename}#{i}")
|
||||
if page:
|
||||
result.pages.append(page)
|
||||
else:
|
||||
page = self._page_from_keep(obj, filename)
|
||||
if page:
|
||||
result.pages.append(page)
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,150 @@
|
||||
"""FlowDeck — background import jobs (v5.6.0, Phase 0).
|
||||
|
||||
Small in-process job manager used for large uploads (vaults, zips): the upload
|
||||
is parsed and persisted in a worker thread while the UI polls job status.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from app.db import get_conn
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportResult,
|
||||
detect_importer,
|
||||
get_importer,
|
||||
)
|
||||
from app.services.importers.pipeline import run_import
|
||||
|
||||
_JOBS: dict[str, dict[str, Any]] = {}
|
||||
_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def parse_upload(filename: str, data: bytes, source_id: str | None = None) -> tuple[Importer | None, ImportResult]:
|
||||
"""Detect (or use) an importer and parse the upload synchronously."""
|
||||
imp = get_importer(source_id) if source_id else None
|
||||
if imp is None:
|
||||
imp = detect_importer(filename, data)
|
||||
if imp is None:
|
||||
return None, ImportResult(source=source_id or "unknown", warnings=["Format non reconnu"])
|
||||
return imp, imp.parse(filename, data)
|
||||
|
||||
|
||||
def _record(job: dict, *, status: str | None = None, error: str = "",
|
||||
report: dict | None = None, progress: int | None = None) -> None:
|
||||
with _LOCK:
|
||||
if status:
|
||||
job["status"] = status
|
||||
if error:
|
||||
job["error"] = error
|
||||
if report is not None:
|
||||
job["report"] = report
|
||||
if progress is not None:
|
||||
job["progress"] = progress
|
||||
job["updated_at"] = time.time()
|
||||
_persist(job)
|
||||
|
||||
|
||||
def _persist(job: dict) -> None:
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO import_jobs (id, source, filename, status, error, report_json, created_at, updated_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?) "
|
||||
"ON CONFLICT(id) DO UPDATE SET status=excluded.status, error=excluded.error, "
|
||||
"report_json=excluded.report_json, updated_at=excluded.updated_at",
|
||||
(job["id"], job["source"], job["filename"], job["status"], job.get("error", ""),
|
||||
_json(job.get("report")), job["created_at"], job["updated_at"]),
|
||||
)
|
||||
conn.commit()
|
||||
except Exception: # noqa: BLE001 - persistence is best-effort
|
||||
pass
|
||||
|
||||
|
||||
def _json(value: Any) -> str:
|
||||
import json
|
||||
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
except (TypeError, ValueError):
|
||||
return "{}"
|
||||
|
||||
|
||||
def create_job(source: str, filename: str) -> dict:
|
||||
job = {
|
||||
"id": uuid.uuid4().hex[:16],
|
||||
"source": source,
|
||||
"filename": filename,
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"error": "",
|
||||
"report": None,
|
||||
"created_at": time.time(),
|
||||
"updated_at": time.time(),
|
||||
}
|
||||
with _LOCK:
|
||||
_JOBS[job["id"]] = job
|
||||
_persist(job)
|
||||
return job
|
||||
|
||||
|
||||
def get_job(job_id: str) -> dict | None:
|
||||
with _LOCK:
|
||||
job = _JOBS.get(job_id)
|
||||
return dict(job) if job else None
|
||||
|
||||
|
||||
def list_jobs(limit: int = 50) -> list[dict]:
|
||||
with _LOCK:
|
||||
jobs = sorted(_JOBS.values(), key=lambda j: j["created_at"], reverse=True)
|
||||
return [dict(j) for j in jobs[:limit]]
|
||||
|
||||
|
||||
def start_import_job(
|
||||
*,
|
||||
filename: str,
|
||||
data: bytes,
|
||||
source_id: str | None,
|
||||
workspace_id: int | None,
|
||||
workspace_name: str | None,
|
||||
user_login: str,
|
||||
parent_page_id: int | None,
|
||||
target_collection_id: int | None,
|
||||
dedup: bool = True,
|
||||
mapping: dict[str, str] | None = None,
|
||||
mode: str | None = None,
|
||||
) -> dict:
|
||||
"""Create a job and run parse + persist in a background thread."""
|
||||
job = create_job(source_id or "auto", filename)
|
||||
_record(job, status="running", progress=5)
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
imp, result = parse_upload(filename, data, source_id)
|
||||
if imp is None:
|
||||
_record(job, status="error", error="Format non reconnu")
|
||||
return
|
||||
_record(job, progress=40)
|
||||
report = run_import(
|
||||
result,
|
||||
workspace_id=workspace_id,
|
||||
workspace_name=workspace_name,
|
||||
user_login=user_login,
|
||||
parent_page_id=parent_page_id,
|
||||
target_collection_id=target_collection_id,
|
||||
dedup=dedup,
|
||||
mapping=mapping,
|
||||
mode=mode,
|
||||
)
|
||||
_record(job, status="done", progress=100, report=report)
|
||||
except Exception as exc: # noqa: BLE001 - surface the error to the UI
|
||||
_record(job, status="error", error=f"{exc}", report={
|
||||
"traceback": traceback.format_exc()[-2000:],
|
||||
})
|
||||
|
||||
threading.Thread(target=worker, name=f"import-{job['id']}", daemon=True).start()
|
||||
return job
|
||||
@@ -0,0 +1,87 @@
|
||||
"""FlowDeck — generic Markdown / text importer (v5.6.0, Phase 1)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_MD_EXTS = (".md", ".markdown", ".txt", ".mdx")
|
||||
|
||||
|
||||
def _title_from_name(name: str) -> str:
|
||||
base = name.replace("\\", "/").rsplit("/", 1)[-1]
|
||||
for ext in (".markdown", ".markdown", ".mdx", ".md", ".txt"):
|
||||
if base.lower().endswith(ext):
|
||||
base = base[: -len(ext)]
|
||||
break
|
||||
return base.strip() or "Untitled"
|
||||
|
||||
|
||||
@register_importer
|
||||
class MarkdownImporter(Importer):
|
||||
source_id = "markdown"
|
||||
label = "Markdown / texte"
|
||||
description = "Fichiers .md/.markdown/.txt ou archive .zip de fichiers Markdown."
|
||||
extensions = (".md", ".markdown", ".txt", ".zip")
|
||||
order = 90
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
low = filename.lower()
|
||||
if low.endswith(_MD_EXTS):
|
||||
return True
|
||||
if low.endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return False
|
||||
names = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
return bool(names) and all(n.lower().endswith(_MD_EXTS) for n in names)
|
||||
return False
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
if filename.lower().endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
result.warn(f"Archive invalide : {exc}")
|
||||
return result.finalize()
|
||||
entries = sorted(
|
||||
(n for n in zf.namelist()
|
||||
if not n.endswith("/") and n.lower().endswith(_MD_EXTS)),
|
||||
key=lambda n: (n.count("/"), n.lower()),
|
||||
)
|
||||
if not entries:
|
||||
result.warn("Aucun fichier Markdown trouvé dans l'archive")
|
||||
return result.finalize()
|
||||
for name in entries:
|
||||
try:
|
||||
text = decode_text(zf.read(name))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Lecture impossible : {name} ({exc})")
|
||||
continue
|
||||
parts = name.replace("\\", "/").split("/")
|
||||
result.pages.append(ImportPage(
|
||||
title=_title_from_name(name),
|
||||
markdown=text,
|
||||
source_path=name,
|
||||
parent_path="/".join(parts[:-1]),
|
||||
external_id=name,
|
||||
))
|
||||
return result.finalize()
|
||||
|
||||
text = decode_text(data)
|
||||
result.pages.append(ImportPage(
|
||||
title=_title_from_name(filename),
|
||||
markdown=text,
|
||||
source_path=filename,
|
||||
external_id=filename,
|
||||
))
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""FlowDeck — Notion export importer (v5.6.0, Phase 1, amélioration v5.4.0).
|
||||
|
||||
Imports a Notion "Export as Markdown & CSV" ``.zip``: complete page hierarchy,
|
||||
databases (``.csv``) turned into FlowDeck collections, and image attachments.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from urllib.parse import unquote
|
||||
|
||||
from app.services.importers._common import split_frontmatter
|
||||
from app.services.importers.base import (
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
from app.services.importers.tabular import rows_to_collection
|
||||
|
||||
_HASH_RE = re.compile(r"\s+[0-9a-f]{32}$")
|
||||
_MD_LINK_RE = re.compile(r"\]\(([^)]+)\.md\)")
|
||||
|
||||
|
||||
def _clean_name(name: str) -> str:
|
||||
base = unquote(name.replace("\\", "/").rsplit("/", 1)[-1])
|
||||
base = re.sub(r"\.(md|csv|markdown)$", "", base, flags=re.IGNORECASE)
|
||||
return _HASH_RE.sub("", base).strip() or "Untitled"
|
||||
|
||||
|
||||
def _strip_hash_link(match: re.Match) -> str:
|
||||
target = unquote(match.group(1)).strip()
|
||||
return f"]({_HASH_RE.sub('', target).strip() or target})"
|
||||
|
||||
|
||||
@register_importer
|
||||
class NotionImporter(Importer):
|
||||
source_id = "notion"
|
||||
label = "Notion (export .zip)"
|
||||
description = "Export Notion Markdown & CSV : hiérarchie, databases → collections, images."
|
||||
extensions = (".zip",)
|
||||
order = 20
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if not filename.lower().endswith(".zip"):
|
||||
return False
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return False
|
||||
names = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
md = [n for n in names if n.lower().endswith(".md")]
|
||||
csvs = [n for n in names if n.lower().endswith(".csv")]
|
||||
if not md:
|
||||
return False
|
||||
if csvs:
|
||||
return True
|
||||
return any(_HASH_RE.search(unquote(n.rsplit("/", 1)[-1])) for n in md)
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
result.warn(f"Archive invalide : {exc}")
|
||||
return result.finalize()
|
||||
|
||||
names = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
md_names = [n for n in names if n.lower().endswith(".md")]
|
||||
csv_names = [n for n in names if n.lower().endswith(".csv")]
|
||||
|
||||
pages_by_title: dict[str, ImportPage] = {}
|
||||
for name in sorted(md_names, key=lambda n: (n.count("/"), n.lower())):
|
||||
try:
|
||||
text = decode_text(zf.read(name))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Lecture impossible : {name} ({exc})")
|
||||
continue
|
||||
meta, body = split_frontmatter(text)
|
||||
title = _clean_name(name)
|
||||
body = _MD_LINK_RE.sub(_strip_hash_link, body)
|
||||
first_line = body.lstrip().splitlines()[0] if body.strip() else ""
|
||||
if first_line.strip().startswith("# ") and first_line.strip()[2:].strip() == title:
|
||||
body = "\n".join(body.lstrip().splitlines()[1:]).lstrip("\n")
|
||||
parts = unquote(name).replace("\\", "/").split("/")
|
||||
page = ImportPage(
|
||||
title=title,
|
||||
markdown=body,
|
||||
source_path=name,
|
||||
parent_path="/".join(parts[:-1]),
|
||||
properties={k: v for k, v in meta.items() if k != "title"},
|
||||
external_id=name,
|
||||
)
|
||||
pages_by_title.setdefault(title, page)
|
||||
result.pages.append(page)
|
||||
|
||||
for name in sorted(csv_names, key=lambda n: (n.count("/"), n.lower())):
|
||||
try:
|
||||
text = decode_text(zf.read(name))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Lecture impossible : {name} ({exc})")
|
||||
continue
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
headers = [h for h in (reader.fieldnames or []) if h is not None]
|
||||
rows = [dict(r) for r in reader]
|
||||
title = _clean_name(name)
|
||||
spec_page = rows_to_collection(title, headers, rows)
|
||||
parts = unquote(name).replace("\\", "/").split("/")
|
||||
existing = pages_by_title.get(title)
|
||||
if existing is not None:
|
||||
existing.collection = spec_page.collection
|
||||
existing.source_path = existing.source_path or name
|
||||
else:
|
||||
spec_page.parent_path = "/".join(parts[:-1])
|
||||
spec_page.source_path = name
|
||||
spec_page.external_id = name
|
||||
result.pages.append(spec_page)
|
||||
|
||||
for name in names:
|
||||
low = name.lower()
|
||||
if low.endswith((".md", ".csv")):
|
||||
continue
|
||||
try:
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=name,
|
||||
filename=unquote(name).replace("\\", "/").rsplit("/", 1)[-1],
|
||||
data=zf.read(name),
|
||||
))
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""FlowDeck — Obsidian vault importer (v5.6.0, Phase 1).
|
||||
|
||||
Imports a vault exported as a ``.zip``: Markdown notes (with YAML frontmatter),
|
||||
the folder hierarchy, ``[[wikilinks]]``/``![[embeds]]`` and binary attachments.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
from app.services.importers._common import (
|
||||
coerce_tags,
|
||||
convert_wikilinks,
|
||||
normalize_title,
|
||||
split_frontmatter,
|
||||
)
|
||||
from app.services.importers.base import (
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_SKIP_DIRS = (".obsidian/", ".trash/", ".git/", ".DS_Store")
|
||||
|
||||
|
||||
def _mime_for(name: str) -> str:
|
||||
import mimetypes
|
||||
|
||||
return mimetypes.guess_type(name)[0] or "application/octet-stream"
|
||||
|
||||
|
||||
@register_importer
|
||||
class ObsidianImporter(Importer):
|
||||
source_id = "obsidian"
|
||||
label = "Obsidian (vault .zip)"
|
||||
description = "Vault Obsidian : notes Markdown, frontmatter YAML, wikilinks, pièces jointes."
|
||||
extensions = (".zip",)
|
||||
order = 10
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if not filename.lower().endswith(".zip"):
|
||||
return False
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return False
|
||||
names = zf.namelist()
|
||||
if any("/.obsidian/" in n or n.startswith(".obsidian/") for n in names):
|
||||
return True
|
||||
# Heuristic: mostly-markdown archive containing wikilinks.
|
||||
md = [n for n in names if n.lower().endswith(".md")]
|
||||
if not md:
|
||||
return False
|
||||
for n in md[:20]:
|
||||
try:
|
||||
if "[[" in decode_text(zf.read(n)):
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return False
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
result.warn(f"Archive invalide : {exc}")
|
||||
return result.finalize()
|
||||
|
||||
names = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
notes = [n for n in names if n.lower().endswith(".md")]
|
||||
assets = [n for n in names if not n.lower().endswith(".md")]
|
||||
|
||||
for name in assets:
|
||||
clean = name.replace("\\", "/")
|
||||
if any(part in clean for part in _SKIP_DIRS) or clean.split("/")[-1].startswith("."):
|
||||
continue
|
||||
try:
|
||||
payload = zf.read(name)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=name,
|
||||
filename=clean.rsplit("/", 1)[-1],
|
||||
data=payload,
|
||||
mime=_mime_for(name),
|
||||
))
|
||||
|
||||
for name in sorted(notes, key=lambda n: (n.count("/"), n.lower())):
|
||||
clean = name.replace("\\", "/")
|
||||
if any(part in clean for part in _SKIP_DIRS):
|
||||
continue
|
||||
try:
|
||||
text = decode_text(zf.read(name))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Lecture impossible : {name} ({exc})")
|
||||
continue
|
||||
meta, body = split_frontmatter(text)
|
||||
body = convert_wikilinks(body)
|
||||
parts = clean.split("/")
|
||||
title = normalize_title(meta.get("title")) or parts[-1][:-3]
|
||||
props = dict(meta)
|
||||
props.pop("title", None)
|
||||
tags = coerce_tags(meta.get("tags"))
|
||||
if tags:
|
||||
props["tags"] = tags
|
||||
result.pages.append(ImportPage(
|
||||
title=title or "Untitled",
|
||||
markdown=body,
|
||||
source_path=clean,
|
||||
parent_path="/".join(parts[:-1]),
|
||||
properties=props,
|
||||
external_id=clean,
|
||||
))
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,86 @@
|
||||
"""FlowDeck — OPML importer (v5.6.0, Phase 4).
|
||||
|
||||
Imports an OPML outline (RSS readers, feed lists) as a collection of feeds.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
make_collection,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_SCHEMA = [
|
||||
{"name": "Title", "type": "title"},
|
||||
{"name": "Feed URL", "type": "url"},
|
||||
{"name": "Site URL", "type": "url"},
|
||||
{"name": "Type", "type": "select", "options": [
|
||||
{"name": "rss", "color": "orange"},
|
||||
{"name": "folder", "color": "gray"},
|
||||
]},
|
||||
{"name": "Folder", "type": "text"},
|
||||
]
|
||||
|
||||
|
||||
@register_importer
|
||||
class OpmlImporter(Importer):
|
||||
source_id = "opml"
|
||||
label = "OPML (flux RSS)"
|
||||
description = "Outline OPML → collection de flux (titre, URL, dossier)."
|
||||
extensions = (".opml", ".xml")
|
||||
order = 34
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if filename.lower().endswith(".opml"):
|
||||
return True
|
||||
head = decode_text(data)[:1000].lower()
|
||||
return "<opml" in head and "<outline" in head
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
rows: list[dict] = []
|
||||
try:
|
||||
root = ET.fromstring(decode_text(data))
|
||||
except ET.ParseError as exc:
|
||||
result.warn(f"OPML invalide : {exc}")
|
||||
return result.finalize()
|
||||
for outline in root.iter("outline"):
|
||||
attrs = {k.lower(): v for k, v in outline.attrib.items()}
|
||||
feed = attrs.get("xmlurl")
|
||||
title = attrs.get("title") or attrs.get("text") or feed or ""
|
||||
if not feed and not title:
|
||||
continue
|
||||
props: dict[str, Any] = {}
|
||||
if feed:
|
||||
props["Feed URL"] = feed
|
||||
props["Type"] = "rss"
|
||||
else:
|
||||
props["Type"] = "folder"
|
||||
if attrs.get("htmlurl"):
|
||||
props["Site URL"] = attrs["htmlurl"]
|
||||
folder = _folder_of(outline, root)
|
||||
if folder:
|
||||
props["Folder"] = folder
|
||||
rows.append({"title": title[:200] or "Feed", "properties": props})
|
||||
result.pages.append(make_collection("OPML feeds", _SCHEMA, rows, source_path=filename))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
def _folder_of(node: ET.Element, root: ET.Element) -> str:
|
||||
parents = {child: parent for parent in root.iter() for child in parent}
|
||||
parts: list[str] = []
|
||||
current = parents.get(node)
|
||||
while current is not None:
|
||||
attrs = {k.lower(): v for k, v in current.attrib.items()}
|
||||
if not attrs.get("xmlurl"):
|
||||
label = attrs.get("title") or attrs.get("text")
|
||||
if label:
|
||||
parts.append(label)
|
||||
current = parents.get(current)
|
||||
return " / ".join(reversed(parts))
|
||||
@@ -0,0 +1,164 @@
|
||||
"""FlowDeck — Logseq / Roam Research outliner importer (v5.6.0, Phase 1)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
from app.services.importers._common import (
|
||||
coerce_tags,
|
||||
convert_wikilinks,
|
||||
normalize_title,
|
||||
split_frontmatter,
|
||||
)
|
||||
from app.services.importers.base import (
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_LOGSEQ_MARKERS = ("property::", "logseq/", "journals/")
|
||||
_ROAM_MARKERS = ("{{[[TODO]]}}", "{{[[DONE]]}}", "{{[[query]]}}")
|
||||
_PROP_RE = re.compile(r"^\s*([a-zA-Z][\w-]*)::\s*(.*)$")
|
||||
|
||||
|
||||
def _journal_title(name: str) -> str:
|
||||
m = re.match(r"^(\d{4})[_-](\d{2})[_-](\d{2})", name)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
||||
return name
|
||||
|
||||
|
||||
def _clean_outline(text: str) -> tuple[dict, str]:
|
||||
meta, body = split_frontmatter(text)
|
||||
lines_out: list[str] = []
|
||||
for line in body.splitlines():
|
||||
m = _PROP_RE.match(line)
|
||||
if m and line.lstrip().startswith("-"):
|
||||
continue
|
||||
# Logseq properties appear as bare ``key:: value`` lines too.
|
||||
m2 = _PROP_RE.match(line)
|
||||
if m2 and not line.lstrip().startswith(("-", "*", "#", "|")):
|
||||
key = m2.group(1)
|
||||
if key not in meta:
|
||||
meta[key] = m2.group(2).strip()
|
||||
continue
|
||||
lines_out.append(line)
|
||||
body = "\n".join(lines_out)
|
||||
# Roam task markers → GFM checkboxes.
|
||||
body = body.replace("{{[[TODO]]}}", "[ ] ").replace("{{[[DONE]]}}", "[x] ")
|
||||
# Block references ((uuid)) → plain anchors.
|
||||
body = re.sub(r"\(\(([0-9a-fA-F-]{6,})\)\)", r"[[\1]]", body)
|
||||
body = convert_wikilinks(body)
|
||||
return meta, body
|
||||
|
||||
|
||||
class _OutlineBase(Importer):
|
||||
markers: tuple[str, ...] = ()
|
||||
property_syntax = False
|
||||
source_id = "outline"
|
||||
label = "Outliner"
|
||||
description = ""
|
||||
order = 30
|
||||
|
||||
def _text_matches(self, text: str) -> bool:
|
||||
if any(m in text for m in self.markers if not m.endswith("/")):
|
||||
return True
|
||||
return bool(self.property_syntax and re.search(r"^\s*[a-zA-Z][\w-]*::", text, re.M))
|
||||
|
||||
def _looks_like(self, filename: str, data: bytes) -> bool:
|
||||
low = filename.lower()
|
||||
if low.endswith((".md", ".markdown", ".txt")):
|
||||
return self._text_matches(decode_text(data))
|
||||
if low.endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return False
|
||||
names = zf.namelist()
|
||||
if any(m in name for m in self.markers if m.endswith("/") for name in names):
|
||||
return True
|
||||
for n in [x for x in names if x.lower().endswith(".md")][:10]:
|
||||
try:
|
||||
if self._text_matches(decode_text(zf.read(n))):
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return False
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
return self._looks_like(filename, data)
|
||||
|
||||
def _emit(self, result: ImportResult, name: str, text: str, is_journal: bool) -> None:
|
||||
meta, body = _clean_outline(text)
|
||||
parts = name.replace("\\", "/").split("/")
|
||||
raw_title = parts[-1].rsplit(".", 1)[0]
|
||||
title = normalize_title(meta.get("title")) or (
|
||||
_journal_title(raw_title) if is_journal else raw_title
|
||||
)
|
||||
props = {k: v for k, v in meta.items() if k != "title"}
|
||||
tags = coerce_tags(meta.get("tags"))
|
||||
if tags:
|
||||
props["tags"] = tags
|
||||
result.pages.append(ImportPage(
|
||||
title=title or "Untitled",
|
||||
markdown=body,
|
||||
source_path=name,
|
||||
parent_path="/".join(parts[:-1]),
|
||||
properties=props,
|
||||
external_id=name,
|
||||
))
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
if filename.lower().endswith(".zip"):
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(data))
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
result.warn(f"Archive invalide : {exc}")
|
||||
return result.finalize()
|
||||
names = [n for n in zf.namelist() if not n.endswith("/")]
|
||||
for name in [n for n in names if not n.lower().endswith(".md")]:
|
||||
try:
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=name,
|
||||
filename=name.replace("\\", "/").rsplit("/", 1)[-1],
|
||||
data=zf.read(name),
|
||||
))
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
for name in sorted(
|
||||
(n for n in names if n.lower().endswith(".md")),
|
||||
key=lambda n: (n.count("/"), n.lower()),
|
||||
):
|
||||
try:
|
||||
text = decode_text(zf.read(name))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Lecture impossible : {name} ({exc})")
|
||||
continue
|
||||
self._emit(result, name, text, is_journal="journal" in name.lower())
|
||||
return result.finalize()
|
||||
|
||||
text = decode_text(data)
|
||||
self._emit(result, filename, text, is_journal=False)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class LogseqImporter(_OutlineBase):
|
||||
source_id = "logseq"
|
||||
label = "Logseq"
|
||||
description = "Outliner Logseq : pages/journal, propriétés `key:: value`, block refs."
|
||||
markers = ("property::", "logseq/", "journals/")
|
||||
property_syntax = True
|
||||
|
||||
|
||||
@register_importer
|
||||
class RoamImporter(_OutlineBase):
|
||||
source_id = "roam"
|
||||
label = "Roam Research"
|
||||
description = "Outliner Roam : `{{[[TODO]]}}`, block refs, wikilinks."
|
||||
markers = _ROAM_MARKERS
|
||||
@@ -0,0 +1,94 @@
|
||||
"""FlowDeck — PDF importer (v5.6.0, Phase 3).
|
||||
|
||||
Best-effort text + image extraction from a PDF into a FlowDeck page (fidelity
|
||||
depends on the source PDF; scanned documents have no text layer).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.importers.base import (
|
||||
ImportAttachment,
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
|
||||
".tiff": "image/tiff", ".tif": "image/tiff"}
|
||||
|
||||
|
||||
@register_importer
|
||||
class PdfImporter(Importer):
|
||||
source_id = "pdf"
|
||||
label = "PDF"
|
||||
description = "Extraction texte + images d'un PDF (fidélité limitée)."
|
||||
extensions = (".pdf",)
|
||||
order = 37
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
return filename.lower().endswith(".pdf")
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError:
|
||||
result.warn("pypdf n'est pas installé : import PDF indisponible")
|
||||
return result.finalize()
|
||||
try:
|
||||
reader = PdfReader(io.BytesIO(data))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"PDF illisible : {exc}")
|
||||
return result.finalize()
|
||||
|
||||
chunks: list[str] = []
|
||||
empty_pages = 0
|
||||
for index, page in enumerate(reader.pages, start=1):
|
||||
try:
|
||||
text = (page.extract_text() or "").strip()
|
||||
except Exception: # noqa: BLE001
|
||||
text = ""
|
||||
if text:
|
||||
if len(reader.pages) > 1:
|
||||
chunks.append(f"## Page {index}\n\n{text}")
|
||||
else:
|
||||
chunks.append(text)
|
||||
else:
|
||||
empty_pages += 1
|
||||
chunks.extend(self._page_images(page, index, result))
|
||||
|
||||
if empty_pages:
|
||||
result.warn(f"{empty_pages} page(s) sans couche texte (document scanné ?)")
|
||||
markdown = re.sub(r"\n{3,}", "\n\n", "\n\n".join(chunks)).strip()
|
||||
title = Path(filename).stem or "Document"
|
||||
result.pages.append(ImportPage(
|
||||
title=title, markdown=markdown, source_path=filename, external_id=filename,
|
||||
))
|
||||
return result.finalize()
|
||||
|
||||
def _page_images(self, page, index: int, result: ImportResult) -> list[str]:
|
||||
images: list[str] = []
|
||||
try:
|
||||
page_images = list(page.images)
|
||||
except Exception: # noqa: BLE001
|
||||
return images
|
||||
for i, image in enumerate(page_images, start=1):
|
||||
name = getattr(image, "name", "") or f"page{index}_img{i}.png"
|
||||
name = Path(name).name
|
||||
try:
|
||||
payload = image.data
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if not payload:
|
||||
continue
|
||||
result.attachments.append(ImportAttachment(
|
||||
source_path=f"page{index}/{name}", filename=name, data=payload,
|
||||
mime=_MIME.get(Path(name).suffix.lower(), "application/octet-stream"),
|
||||
))
|
||||
images.append(f"")
|
||||
return images
|
||||
@@ -0,0 +1,566 @@
|
||||
"""FlowDeck — common import pipeline (v5.6.0, Phase 0).
|
||||
|
||||
Persists an :class:`~app.services.importers.base.ImportResult` into FlowDeck:
|
||||
resolves the workspace, rebuilds the folder hierarchy (``parent_id``), stores
|
||||
attachments, rewrites links, creates collections + rows, and records imported
|
||||
items for idempotent re-imports.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.db import get_conn
|
||||
from app.services.db_templates import materialize_properties
|
||||
from app.services.export import markdown_to_blocks
|
||||
from app.services.importers.base import ImportPage, ImportResult
|
||||
from app.services.importers.tabular import apply_type_mapping
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
|
||||
|
||||
|
||||
def _data_dir() -> Path:
|
||||
return Path(os.environ.get("FLOWDECK_DATA_DIR", "/data"))
|
||||
|
||||
|
||||
def _safe_filename(name: str) -> str:
|
||||
base = Path(name.replace("\\", "/")).name
|
||||
base = re.sub(r"[^\w.\- ()]+", "_", base).strip() or "file"
|
||||
return base[:150]
|
||||
|
||||
|
||||
def _sha1(*parts: str) -> str:
|
||||
return hashlib.sha1("||".join(parts).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _resolve_workspace(conn, workspace_id: int | None, workspace_name: str | None,
|
||||
user_login: str) -> tuple[int | None, str]:
|
||||
if workspace_id:
|
||||
row = conn.execute("SELECT id, name FROM workspaces WHERE id=?", (workspace_id,)).fetchone()
|
||||
if row:
|
||||
return row["id"], row["name"]
|
||||
name = workspace_name or user_login
|
||||
if name:
|
||||
row = conn.execute("SELECT id, name FROM workspaces WHERE name=?", (name,)).fetchone()
|
||||
if row:
|
||||
return row["id"], row["name"]
|
||||
return workspace_id, name or ""
|
||||
|
||||
|
||||
def _rewrite_links(text: str, attachment_map: dict[str, str]) -> str:
|
||||
"""Point Markdown links/images at uploaded attachment URLs."""
|
||||
def repl(m: re.Match) -> str:
|
||||
alt, target = m.group(1), m.group(2)
|
||||
url = attachment_map.get(target) or attachment_map.get(Path(target).name.lower())
|
||||
return f"" if url else m.group(0)
|
||||
|
||||
text = re.sub(r"!\[([^\]]*)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)", repl, text)
|
||||
return text
|
||||
|
||||
|
||||
def _extract_media(blocks: list[dict]) -> list[dict]:
|
||||
"""Split inline image markdown out of paragraphs into real image blocks."""
|
||||
out: list[dict] = []
|
||||
for block in blocks:
|
||||
if block.get("type") != "paragraph":
|
||||
out.append(block)
|
||||
continue
|
||||
content = str(block.get("content", ""))
|
||||
pos = 0
|
||||
found = False
|
||||
for match in _IMG_RE.finditer(content):
|
||||
found = True
|
||||
before = content[pos:match.start()].strip()
|
||||
if before:
|
||||
out.append({"type": "paragraph", "content": before})
|
||||
out.append({"type": "image", "src": match.group(2), "alt": match.group(1)})
|
||||
pos = match.end()
|
||||
if not found:
|
||||
out.append(block)
|
||||
continue
|
||||
tail = content[pos:].strip()
|
||||
if tail:
|
||||
out.append({"type": "paragraph", "content": tail})
|
||||
return out
|
||||
|
||||
|
||||
def _properties_callout(props: dict) -> dict | None:
|
||||
if not props:
|
||||
return None
|
||||
lines = [f"**{k}** : {', '.join(map(str, v)) if isinstance(v, list) else v}" for k, v in props.items()]
|
||||
return {"type": "callout", "icon": "ℹ️", "content": "\n".join(lines)}
|
||||
|
||||
|
||||
def _blocks_for(page: ImportPage, attachment_map: dict[str, str], *, include_properties: bool) -> list[dict]:
|
||||
if page.blocks:
|
||||
return _extract_media(page.blocks)
|
||||
md = _rewrite_links(page.markdown or "", attachment_map)
|
||||
blocks = _extract_media(markdown_to_blocks(md))
|
||||
if include_properties and page.properties:
|
||||
callout = _properties_callout(page.properties)
|
||||
if callout:
|
||||
blocks.insert(0, callout)
|
||||
return blocks
|
||||
|
||||
|
||||
def preview_result(result: ImportResult) -> dict[str, Any]:
|
||||
"""Dry-run preview: what would be created, without touching the database."""
|
||||
pages = []
|
||||
for page in result.pages:
|
||||
if page.collection:
|
||||
kind = "collection"
|
||||
rows = len(page.collection.get("rows", []))
|
||||
blocks = len(_blocks_for(page, {}, include_properties=False))
|
||||
schema = page.collection.get("schema", [])
|
||||
else:
|
||||
kind = "page"
|
||||
rows = 0
|
||||
blocks = len(_blocks_for(page, {}, include_properties=False))
|
||||
schema = []
|
||||
pages.append({
|
||||
"title": page.title,
|
||||
"type": kind,
|
||||
"source_path": page.source_path,
|
||||
"parent_path": page.parent_path,
|
||||
"properties": list(page.properties.keys()),
|
||||
"rows": rows,
|
||||
"blocks": blocks,
|
||||
"schema": schema,
|
||||
})
|
||||
return {
|
||||
"source": result.source,
|
||||
"dry_run": True,
|
||||
"pages": pages,
|
||||
"stats": {
|
||||
**result.stats,
|
||||
"pages": len(result.pages),
|
||||
"collections": sum(1 for p in result.pages if p.collection),
|
||||
"attachments": len(result.attachments),
|
||||
"warnings": len(result.warnings),
|
||||
},
|
||||
"warnings": result.warnings,
|
||||
}
|
||||
|
||||
|
||||
def _insert_page(conn, *, workspace: str, workspace_id: int | None, title: str,
|
||||
blocks: list[dict], parent_id: int | None, sort_order: int,
|
||||
content_format: str = "blocks") -> int:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO pages (workspace, title, content, content_format, parent_section, "
|
||||
"sort_order, workspace_id, parent_id) VALUES (?,?,?,?,'Private',?,?,?)",
|
||||
(workspace, title or "Untitled", json.dumps(blocks), content_format,
|
||||
sort_order, workspace_id, parent_id),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def _prop_id_map(conn, collection_id: int) -> dict[str, int]:
|
||||
return {
|
||||
r["name"]: r["id"]
|
||||
for r in conn.execute(
|
||||
"SELECT id, name FROM collection_properties WHERE collection_id=?",
|
||||
(collection_id,),
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
|
||||
def _rows_to_values(rows: list[dict], id_map: dict[str, int]) -> list[tuple[str, dict]]:
|
||||
"""Key row properties by property id (the shape the editor reads)."""
|
||||
out: list[tuple[str, dict]] = []
|
||||
for row in rows:
|
||||
values: dict[str, Any] = {}
|
||||
for name, value in (row.get("properties") or {}).items():
|
||||
prop_id = id_map.get(name)
|
||||
if prop_id is not None:
|
||||
values[str(prop_id)] = value
|
||||
out.append((row.get("title") or "Untitled", values))
|
||||
return out
|
||||
|
||||
|
||||
def _ensure_default_view(conn, collection_id: int) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO collection_views (collection_id, name, view_type, config_json) "
|
||||
"VALUES (?,?,?,?)",
|
||||
(collection_id, "Default View", "table",
|
||||
json.dumps({"visible_properties": ["Title"], "sorts": [], "filters": []})),
|
||||
)
|
||||
|
||||
|
||||
def _insert_collection(conn, page: ImportPage, *, workspace_id: int | None,
|
||||
parent_page_id: int | None) -> tuple[int, int]:
|
||||
spec = page.collection or {}
|
||||
schema = spec.get("schema", [])
|
||||
cur = conn.execute(
|
||||
"INSERT INTO collections (name, description, icon, schema_json, is_inline, "
|
||||
"parent_page_id, workspace_id) VALUES (?,?,?,?,1,?,?)",
|
||||
(page.title or spec.get("name") or "Imported database", "", "📥",
|
||||
json.dumps(schema), parent_page_id, workspace_id),
|
||||
)
|
||||
collection_id = cur.lastrowid
|
||||
materialize_properties(conn, collection_id, schema)
|
||||
_ensure_default_view(conn, collection_id)
|
||||
id_map = _prop_id_map(conn, collection_id)
|
||||
position = 0
|
||||
for title, values in _rows_to_values(spec.get("rows", []), id_map):
|
||||
conn.execute(
|
||||
"INSERT INTO collection_pages (collection_id, title, position, property_values_json) "
|
||||
"VALUES (?,?,?,?)",
|
||||
(collection_id, title, position, json.dumps(values)),
|
||||
)
|
||||
position += 1
|
||||
return collection_id, position
|
||||
|
||||
|
||||
def _upsert_collection_rows(conn, collection_id: int, rows: list[dict]) -> tuple[int, int]:
|
||||
"""Update existing rows by title, insert the new ones. Returns (created, updated)."""
|
||||
id_map = _prop_id_map(conn, collection_id)
|
||||
existing = {
|
||||
(r["title"] or "").strip(): r["id"]
|
||||
for r in conn.execute(
|
||||
"SELECT id, title FROM collection_pages WHERE collection_id=?", (collection_id,)
|
||||
).fetchall()
|
||||
}
|
||||
max_pos = conn.execute(
|
||||
"SELECT COALESCE(MAX(position), -1) FROM collection_pages WHERE collection_id=?",
|
||||
(collection_id,),
|
||||
).fetchone()[0]
|
||||
created = updated = 0
|
||||
for title, values in _rows_to_values(rows, id_map):
|
||||
pid = existing.get((title or "").strip())
|
||||
if pid:
|
||||
conn.execute(
|
||||
"UPDATE collection_pages SET property_values_json=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(json.dumps(values), pid),
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
max_pos += 1
|
||||
conn.execute(
|
||||
"INSERT INTO collection_pages (collection_id, title, position, property_values_json) "
|
||||
"VALUES (?,?,?,?)",
|
||||
(collection_id, title, max_pos, json.dumps(values)),
|
||||
)
|
||||
created += 1
|
||||
return created, updated
|
||||
|
||||
|
||||
def _update_page(conn, page_id: int, title: str, blocks: list[dict]) -> None:
|
||||
conn.execute(
|
||||
"UPDATE pages SET title=?, content=?, content_format='blocks', "
|
||||
"updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(title or "Untitled", json.dumps(blocks), page_id),
|
||||
)
|
||||
|
||||
|
||||
def run_import(
|
||||
result: ImportResult,
|
||||
*,
|
||||
workspace_id: int | None = None,
|
||||
workspace_name: str | None = None,
|
||||
user_login: str = "",
|
||||
parent_page_id: int | None = None,
|
||||
target_collection_id: int | None = None,
|
||||
dry_run: bool = False,
|
||||
dedup: bool = True,
|
||||
include_properties: bool = True,
|
||||
mapping: dict[str, str] | None = None,
|
||||
mode: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist an import result. Returns a report dict.
|
||||
|
||||
``mode`` controls re-import behaviour for items already imported (matched by
|
||||
``import_items``): ``skip`` (default), ``update`` (re-sync in place) or
|
||||
``duplicate`` (always create a new page).
|
||||
"""
|
||||
if dry_run:
|
||||
return preview_result(result)
|
||||
|
||||
if mapping:
|
||||
for page in result.pages:
|
||||
if page.collection:
|
||||
apply_type_mapping(page.collection, mapping)
|
||||
|
||||
if not mode:
|
||||
mode = "skip" if dedup else "duplicate"
|
||||
|
||||
report: dict[str, Any] = {
|
||||
"source": result.source,
|
||||
"status": "ok",
|
||||
"mode": mode,
|
||||
"pages_created": 0,
|
||||
"pages_updated": 0,
|
||||
"collections_created": 0,
|
||||
"rows_created": 0,
|
||||
"rows_updated": 0,
|
||||
"attachments": 0,
|
||||
"skipped": 0,
|
||||
"page_ids": [],
|
||||
"errors": [],
|
||||
"warnings": list(result.warnings),
|
||||
}
|
||||
|
||||
with get_conn() as conn:
|
||||
ws_id, ws_name = _resolve_workspace(conn, workspace_id, workspace_name, user_login)
|
||||
if not ws_name:
|
||||
report["status"] = "error"
|
||||
report["warnings"].append("Workspace introuvable")
|
||||
return report
|
||||
|
||||
attachment_map: dict[str, str] = {}
|
||||
if result.attachments and ws_id:
|
||||
dest_dir = _data_dir() / "uploads" / f"workspace_{ws_id}" / "import"
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
for att in result.attachments:
|
||||
safe = _safe_filename(att.filename)
|
||||
target = dest_dir / safe
|
||||
if target.exists():
|
||||
target = dest_dir / f"{_sha1(att.source_path)[:8]}_{safe}"
|
||||
try:
|
||||
target.write_bytes(att.data)
|
||||
except OSError:
|
||||
continue
|
||||
url = f"/api/files/{ws_id}/import/{target.name}"
|
||||
attachment_map[att.source_path] = url
|
||||
attachment_map[att.source_path.lower()] = url
|
||||
attachment_map[Path(att.source_path).name.lower()] = url
|
||||
report["attachments"] += 1
|
||||
|
||||
if target_collection_id:
|
||||
return _import_into_collection(
|
||||
conn, result, target_collection_id, report, attachment_map,
|
||||
dedup=dedup, workspace_id=ws_id, include_properties=include_properties,
|
||||
)
|
||||
|
||||
next_order = conn.execute(
|
||||
"SELECT COALESCE(MAX(sort_order), -1) + 1 FROM pages WHERE workspace=? AND parent_id IS NULL",
|
||||
(ws_name,),
|
||||
).fetchone()[0]
|
||||
order_counter = [next_order]
|
||||
path_to_page: dict[str, int] = {}
|
||||
folder_cache: dict[str, int | None] = {}
|
||||
|
||||
def ensure_folder(path: str, _depth: int = 0) -> int | None:
|
||||
path = (path or "").strip("/")
|
||||
if not path:
|
||||
return parent_page_id
|
||||
if path in folder_cache:
|
||||
return folder_cache[path]
|
||||
parent_path = "/".join(path.split("/")[:-1])
|
||||
parent_id = ensure_folder(parent_path, _depth + 1)
|
||||
title = path.split("/")[-1] or "Folder"
|
||||
pid = _insert_page(
|
||||
conn, workspace=ws_name, workspace_id=ws_id, title=title,
|
||||
blocks=[], parent_id=parent_id, sort_order=order_counter[0],
|
||||
)
|
||||
order_counter[0] += 1
|
||||
folder_cache[path] = pid
|
||||
path_to_page[path] = pid
|
||||
report["pages_created"] += 1
|
||||
report["page_ids"].append(pid)
|
||||
return pid
|
||||
|
||||
ordered = sorted(
|
||||
result.pages,
|
||||
key=lambda p: (p.parent_path.count("/") if p.parent_path else -1, p.source_path.lower()),
|
||||
)
|
||||
for page in ordered:
|
||||
external = page.external_id or page.source_path or page.title
|
||||
existing_pid = None
|
||||
if external:
|
||||
row = conn.execute(
|
||||
"SELECT page_id FROM import_items WHERE workspace_id IS ? AND source=? AND external_id=?",
|
||||
(ws_id, result.source, external),
|
||||
).fetchone()
|
||||
if row:
|
||||
existing_pid = row["page_id"]
|
||||
if existing_pid and mode == "skip":
|
||||
report["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
page_parent = ensure_folder(page.parent_path) if page.parent_path else parent_page_id
|
||||
blocks = _blocks_for(page, attachment_map, include_properties=include_properties)
|
||||
|
||||
if existing_pid and mode == "update":
|
||||
if page.collection:
|
||||
cid = _collection_for_page(conn, existing_pid)
|
||||
if cid:
|
||||
materialize_properties(conn, cid, (page.collection or {}).get("schema", []))
|
||||
created, updated = _upsert_collection_rows(
|
||||
conn, cid, (page.collection or {}).get("rows", []))
|
||||
report["rows_created"] += created
|
||||
report["rows_updated"] += updated
|
||||
blocks = blocks + [{"type": "embed", "embed_type": "collection",
|
||||
"collection_id": cid, "content": ""}]
|
||||
_update_page(conn, existing_pid, page.title, blocks)
|
||||
report["pages_updated"] += 1
|
||||
report["page_ids"].append(existing_pid)
|
||||
path_to_page[page.source_path] = existing_pid
|
||||
continue
|
||||
|
||||
if page.collection:
|
||||
pid = _insert_page(
|
||||
conn, workspace=ws_name, workspace_id=ws_id, title=page.title,
|
||||
blocks=blocks, parent_id=page_parent, sort_order=order_counter[0],
|
||||
)
|
||||
order_counter[0] += 1
|
||||
cid, rows = _insert_collection(conn, page, workspace_id=ws_id, parent_page_id=pid)
|
||||
conn.execute(
|
||||
"UPDATE pages SET content=? WHERE id=?",
|
||||
(json.dumps(blocks + [{"type": "embed", "embed_type": "collection",
|
||||
"collection_id": cid, "content": ""}]), pid),
|
||||
)
|
||||
report["collections_created"] += 1
|
||||
report["rows_created"] += rows
|
||||
report["pages_created"] += 1
|
||||
report["page_ids"].append(pid)
|
||||
else:
|
||||
pid = _insert_page(
|
||||
conn, workspace=ws_name, workspace_id=ws_id, title=page.title,
|
||||
blocks=blocks, parent_id=page_parent, sort_order=order_counter[0],
|
||||
)
|
||||
order_counter[0] += 1
|
||||
report["pages_created"] += 1
|
||||
report["page_ids"].append(pid)
|
||||
|
||||
path_to_page[page.source_path] = pid
|
||||
if external:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO import_items (workspace_id, source, external_id, page_id) VALUES (?,?,?,?)",
|
||||
(ws_id, result.source, external, pid),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - partial import must keep going
|
||||
logger.warning("import failed for %r: %s", page.title, exc)
|
||||
report["errors"].append({"title": page.title, "error": str(exc)})
|
||||
conn.commit()
|
||||
if report["errors"]:
|
||||
report["status"] = "partial"
|
||||
return report
|
||||
|
||||
|
||||
def _collection_for_page(conn, page_id: int) -> int | None:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM collections WHERE parent_page_id=? ORDER BY id LIMIT 1", (page_id,)
|
||||
).fetchone()
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
def _import_into_collection(conn, result: ImportResult, collection_id: int, report: dict,
|
||||
attachment_map: dict[str, str], *, dedup: bool,
|
||||
workspace_id: int | None, include_properties: bool) -> dict:
|
||||
exists = conn.execute("SELECT id FROM collections WHERE id=?", (collection_id,)).fetchone()
|
||||
if not exists:
|
||||
report["status"] = "error"
|
||||
report["warnings"].append("Collection cible introuvable")
|
||||
return report
|
||||
max_pos = conn.execute(
|
||||
"SELECT COALESCE(MAX(position), -1) + 1 FROM collection_pages WHERE collection_id=?",
|
||||
(collection_id,),
|
||||
).fetchone()[0]
|
||||
id_map = _prop_id_map(conn, collection_id)
|
||||
for page in result.pages:
|
||||
external = page.external_id or page.source_path or page.title
|
||||
if dedup and external:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM collection_pages WHERE collection_id=? AND title=?",
|
||||
(collection_id, page.title),
|
||||
).fetchone()
|
||||
if row:
|
||||
report["skipped"] += 1
|
||||
continue
|
||||
props = {
|
||||
str(id_map[name]): value
|
||||
for name, value in page.properties.items()
|
||||
if name in id_map
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO collection_pages (collection_id, title, position, property_values_json) "
|
||||
"VALUES (?,?,?,?)",
|
||||
(collection_id, page.title, max_pos, json.dumps(props)),
|
||||
)
|
||||
max_pos += 1
|
||||
report["rows_created"] += 1
|
||||
conn.commit()
|
||||
return report
|
||||
|
||||
|
||||
def resolve_relations(conn, workspace_id: int | None) -> dict[str, Any]:
|
||||
"""Convert text columns that reference another imported collection's titles
|
||||
into real ``relation`` properties (array of ``collection_pages`` ids)."""
|
||||
collections = conn.execute(
|
||||
"SELECT id, name FROM collections WHERE workspace_id IS ?", (workspace_id,)
|
||||
).fetchall()
|
||||
if not collections:
|
||||
return {"relations_resolved": 0, "details": []}
|
||||
|
||||
titles: dict[int, dict[str, int]] = {}
|
||||
for coll in collections:
|
||||
rows = conn.execute(
|
||||
"SELECT id, title FROM collection_pages WHERE collection_id=?", (coll["id"],)
|
||||
).fetchall()
|
||||
titles[coll["id"]] = {
|
||||
(r["title"] or "").strip(): r["id"]
|
||||
for r in rows if (r["title"] or "").strip()
|
||||
}
|
||||
|
||||
resolved = 0
|
||||
details: list[dict] = []
|
||||
for coll in collections:
|
||||
props = conn.execute(
|
||||
"SELECT id, name, prop_type FROM collection_properties WHERE collection_id=?",
|
||||
(coll["id"],),
|
||||
).fetchall()
|
||||
pages = conn.execute(
|
||||
"SELECT id, property_values_json FROM collection_pages WHERE collection_id=?",
|
||||
(coll["id"],),
|
||||
).fetchall()
|
||||
for prop in props:
|
||||
if prop["prop_type"] not in ("text", "select", "multi_select"):
|
||||
continue
|
||||
key = str(prop["id"])
|
||||
values: list[str] = []
|
||||
for page in pages:
|
||||
pv = json.loads(page["property_values_json"] or "{}")
|
||||
value = pv.get(key)
|
||||
if value in (None, "", []):
|
||||
continue
|
||||
items = value if isinstance(value, list) else [value]
|
||||
values.extend(str(v) for v in items)
|
||||
if not values:
|
||||
continue
|
||||
for target in collections:
|
||||
if target["id"] == coll["id"]:
|
||||
continue
|
||||
target_titles = titles.get(target["id"]) or {}
|
||||
if target_titles and all(v in target_titles for v in values):
|
||||
conn.execute(
|
||||
"UPDATE collection_properties SET prop_type='relation', "
|
||||
"related_collection_id=? WHERE id=?",
|
||||
(target["id"], prop["id"]),
|
||||
)
|
||||
for page in pages:
|
||||
pv = json.loads(page["property_values_json"] or "{}")
|
||||
value = pv.get(key)
|
||||
if value in (None, "", []):
|
||||
continue
|
||||
items = value if isinstance(value, list) else [value]
|
||||
pv[key] = [target_titles[str(v)] for v in items if str(v) in target_titles]
|
||||
conn.execute(
|
||||
"UPDATE collection_pages SET property_values_json=? WHERE id=?",
|
||||
(json.dumps(pv), page["id"]),
|
||||
)
|
||||
resolved += 1
|
||||
details.append({
|
||||
"collection": coll["name"], "property": prop["name"],
|
||||
"related": target["name"],
|
||||
})
|
||||
break
|
||||
conn.commit()
|
||||
return {"relations_resolved": resolved, "details": details}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""FlowDeck — Standard Notes importer (v5.6.0, Phase 4).
|
||||
|
||||
Imports a Standard Notes backup (``.json``): each non-encrypted note becomes a
|
||||
FlowDeck page. Encrypted notes are reported as warnings.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_NOTE_TYPES = ("note", "org.standardnotes.sn", "org.standardnotes.plain-text")
|
||||
|
||||
|
||||
def _note_body(content: Any) -> tuple[str, str, bool]:
|
||||
"""Return (title, markdown, encrypted)."""
|
||||
if isinstance(content, dict):
|
||||
if content.get("encrypted"):
|
||||
return "", "", True
|
||||
text = content.get("text") or content.get("preview_plain") or ""
|
||||
title = content.get("title") or ""
|
||||
return title, text, False
|
||||
if isinstance(content, str):
|
||||
stripped = content.strip()
|
||||
if stripped.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(stripped)
|
||||
if isinstance(parsed, dict) and ("encrypted" in parsed or "000" in parsed):
|
||||
return "", "", True
|
||||
if isinstance(parsed, dict):
|
||||
return parsed.get("title", ""), parsed.get("text", "") or parsed.get("preview_plain", ""), False
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return "", content, False
|
||||
return "", "", False
|
||||
|
||||
|
||||
@register_importer
|
||||
class StandardNotesImporter(Importer):
|
||||
source_id = "standard_notes"
|
||||
label = "Standard Notes"
|
||||
description = "Sauvegarde JSON Standard Notes → pages (notes chiffrées ignorées)."
|
||||
extensions = (".json",)
|
||||
order = 39
|
||||
|
||||
def _items(self, data: bytes) -> list[dict] | None:
|
||||
try:
|
||||
obj = json.loads(decode_text(data))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if isinstance(obj, dict) and isinstance(obj.get("items"), list):
|
||||
return [x for x in obj["items"] if isinstance(x, dict)]
|
||||
return None
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if not filename.lower().endswith(".json"):
|
||||
return False
|
||||
items = self._items(data)
|
||||
if not items:
|
||||
return False
|
||||
return any("content_type" in it for it in items)
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
items = self._items(data) or []
|
||||
count = 0
|
||||
for item in items:
|
||||
if item.get("deleted"):
|
||||
continue
|
||||
ctype = str(item.get("content_type", "")).lower()
|
||||
if ctype and not any(t in ctype for t in _NOTE_TYPES):
|
||||
continue
|
||||
title, body, encrypted = _note_body(item.get("content"))
|
||||
if encrypted:
|
||||
result.warn("Note chiffrée ignorée (déchiffrement non pris en charge)")
|
||||
continue
|
||||
if not body.strip():
|
||||
continue
|
||||
if not title:
|
||||
title = next((ln.strip(" #") for ln in body.splitlines() if ln.strip()), "Note")
|
||||
result.pages.append(ImportPage(
|
||||
title=title[:200] or "Note",
|
||||
markdown=body,
|
||||
source_path=item.get("uuid") or filename,
|
||||
external_id=item.get("uuid") or f"{filename}#{count}",
|
||||
))
|
||||
count += 1
|
||||
result.stats["rows"] = count
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,305 @@
|
||||
"""FlowDeck — tabular importers: typed CSV/TSV, Excel, generic JSON (v5.6.0, Phase 2).
|
||||
|
||||
Each source becomes a FlowDeck collection (database): columns are inferred from
|
||||
the data (text/number/date/checkbox/email/url/select/multi_select) and rows are
|
||||
inserted as ``collection_pages``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from app.services.importers.base import (
|
||||
Importer,
|
||||
ImportPage,
|
||||
ImportResult,
|
||||
decode_text,
|
||||
register_importer,
|
||||
)
|
||||
|
||||
_TITLE_HEADERS = ("title", "name", "task", "nom", "titre", "subject", "label")
|
||||
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2})?)?")
|
||||
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
_URL_RE = re.compile(r"^https?://\S+$", re.IGNORECASE)
|
||||
_BOOL_TRUE = {"true", "yes", "oui", "1", "x", "vrai"}
|
||||
_BOOL_FALSE = {"false", "no", "non", "0", "faux", ""}
|
||||
|
||||
|
||||
def _is_number(value: str) -> bool:
|
||||
try:
|
||||
float(str(value).replace(",", ".").replace(" ", ""))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def _is_bool(value: str) -> bool:
|
||||
return str(value).strip().lower() in _BOOL_TRUE | _BOOL_FALSE
|
||||
|
||||
|
||||
def infer_column_type(values: list[str]) -> str:
|
||||
"""Infer the FlowDeck property type from a list of raw string cells."""
|
||||
sample = [str(v).strip() for v in values if str(v).strip()]
|
||||
if not sample:
|
||||
return "text"
|
||||
if all(_is_bool(v) for v in sample):
|
||||
return "checkbox"
|
||||
if all(_is_number(v) for v in sample):
|
||||
return "number"
|
||||
if all(_DATE_RE.match(v) for v in sample):
|
||||
return "date"
|
||||
if all(_EMAIL_RE.match(v) for v in sample):
|
||||
return "email"
|
||||
if all(_URL_RE.match(v) for v in sample):
|
||||
return "url"
|
||||
unique = {v for v in sample}
|
||||
if len(unique) <= 20 and len(unique) <= max(2, len(sample) // 2):
|
||||
if any(("," in v or ";" in v) for v in sample):
|
||||
return "multi_select"
|
||||
return "select"
|
||||
return "text"
|
||||
|
||||
|
||||
def _split_multi(value: str) -> list[str]:
|
||||
return [p.strip() for p in re.split(r"[;,]", value) if p.strip()]
|
||||
|
||||
|
||||
def coerce_value(prop_type: str, value: Any) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
raw = str(value).strip()
|
||||
if raw == "":
|
||||
return None
|
||||
if prop_type == "number":
|
||||
try:
|
||||
num = float(raw.replace(",", ".").replace(" ", ""))
|
||||
return int(num) if num.is_integer() else num
|
||||
except (ValueError, TypeError):
|
||||
return raw
|
||||
if prop_type == "checkbox":
|
||||
return raw.lower() in _BOOL_TRUE
|
||||
if prop_type == "multi_select":
|
||||
return _split_multi(raw)
|
||||
return raw
|
||||
|
||||
|
||||
def build_schema(headers: list[str], rows: list[dict[str, Any]]) -> tuple[list[dict], str]:
|
||||
"""Return ``(schema, title_header)`` from headers + row dicts."""
|
||||
title_header = ""
|
||||
for h in headers:
|
||||
if h and h.strip().lower() in _TITLE_HEADERS:
|
||||
title_header = h
|
||||
break
|
||||
schema: list[dict] = []
|
||||
for h in headers:
|
||||
if not h or h == title_header:
|
||||
continue
|
||||
ptype = infer_column_type([r.get(h, "") for r in rows])
|
||||
entry: dict[str, Any] = {"name": h, "type": ptype}
|
||||
if ptype in ("select", "status", "multi_select"):
|
||||
seen: list[str] = []
|
||||
for r in rows:
|
||||
vals = _split_multi(str(r.get(h, ""))) if ptype == "multi_select" else [str(r.get(h, "")).strip()]
|
||||
for v in vals:
|
||||
if v and v not in seen:
|
||||
seen.append(v)
|
||||
entry["options"] = [{"name": v, "color": "gray"} for v in seen[:100]]
|
||||
schema.append(entry)
|
||||
if title_header:
|
||||
schema.insert(0, {"name": title_header, "type": "title"})
|
||||
return schema, title_header
|
||||
|
||||
|
||||
def rows_to_collection(name: str, headers: list[str], raw_rows: list[dict[str, Any]]) -> ImportPage:
|
||||
"""Normalize parsed rows into an ImportPage carrying a collection spec."""
|
||||
schema, title_header = build_schema(headers, raw_rows)
|
||||
rows = _normalize_rows(headers, raw_rows, schema, title_header)
|
||||
return ImportPage(
|
||||
title=name or "Imported database",
|
||||
collection={
|
||||
"name": name or "Imported database",
|
||||
"schema": schema,
|
||||
"rows": rows,
|
||||
"headers": headers,
|
||||
"title_header": title_header,
|
||||
"raw_rows": raw_rows,
|
||||
},
|
||||
source_path=name,
|
||||
external_id=name,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_rows(headers: list[str], raw_rows: list[dict[str, Any]],
|
||||
schema: list[dict], title_header: str) -> list[dict[str, Any]]:
|
||||
types = {s["name"]: s["type"] for s in schema}
|
||||
rows: list[dict[str, Any]] = []
|
||||
for raw in raw_rows:
|
||||
title = ""
|
||||
if title_header:
|
||||
title = str(raw.get(title_header, "")).strip()
|
||||
if not title:
|
||||
for h in headers:
|
||||
if h and str(raw.get(h, "")).strip():
|
||||
title = str(raw[h]).strip()
|
||||
break
|
||||
props: dict[str, Any] = {}
|
||||
for h in headers:
|
||||
if not h or h == title_header:
|
||||
continue
|
||||
val = coerce_value(types.get(h, "text"), raw.get(h))
|
||||
if val is not None and val != "":
|
||||
props[h] = val
|
||||
rows.append({"title": title or "Untitled", "properties": props})
|
||||
return rows
|
||||
|
||||
|
||||
def apply_type_mapping(spec: dict, mapping: dict[str, str]) -> dict:
|
||||
"""Override inferred column types (UI mapping) and re-coerce the rows."""
|
||||
if not mapping:
|
||||
return spec
|
||||
for entry in spec.get("schema", []):
|
||||
if entry.get("name") in mapping:
|
||||
entry["type"] = mapping[entry["name"]]
|
||||
headers = spec.get("headers")
|
||||
raw_rows = spec.get("raw_rows")
|
||||
if headers is not None and raw_rows is not None:
|
||||
spec["rows"] = _normalize_rows(headers, raw_rows, spec.get("schema", []),
|
||||
spec.get("title_header", ""))
|
||||
return spec
|
||||
|
||||
|
||||
def _sniff_delimiter(sample: str) -> str:
|
||||
try:
|
||||
return csv.Sniffer().sniff(sample, delimiters=",;\t|").delimiter
|
||||
except csv.Error:
|
||||
return "\t" if sample.count("\t") > sample.count(",") else ","
|
||||
|
||||
|
||||
@register_importer
|
||||
class CsvImporter(Importer):
|
||||
source_id = "csv"
|
||||
label = "CSV / TSV (typé)"
|
||||
description = "Tableur CSV/TSV : types inférés automatiquement, une collection par fichier."
|
||||
extensions = (".csv", ".tsv")
|
||||
order = 60
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
return filename.lower().endswith((".csv", ".tsv"))
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
text = decode_text(data)
|
||||
if not text.strip():
|
||||
result.warn("Fichier vide")
|
||||
return result.finalize()
|
||||
delimiter = "\t" if filename.lower().endswith(".tsv") else _sniff_delimiter(text[:4096])
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter=delimiter)
|
||||
headers = [h for h in (reader.fieldnames or []) if h is not None]
|
||||
rows = [dict(r) for r in reader]
|
||||
name = filename.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
result.pages.append(rows_to_collection(name, headers, rows))
|
||||
result.stats["rows"] = len(rows)
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class ExcelImporter(Importer):
|
||||
source_id = "excel"
|
||||
label = "Excel (.xlsx)"
|
||||
description = "Classeur Excel : une collection par feuille (openpyxl)."
|
||||
extensions = (".xlsx", ".xlsm")
|
||||
order = 61
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
low = filename.lower()
|
||||
if low.endswith((".xlsx", ".xlsm")):
|
||||
return True
|
||||
return low.endswith(".xls")
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
try:
|
||||
from openpyxl import load_workbook
|
||||
except ImportError:
|
||||
result.warn("openpyxl n'est pas installé : import Excel indisponible")
|
||||
return result.finalize()
|
||||
try:
|
||||
wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.warn(f"Classeur illisible : {exc}")
|
||||
return result.finalize()
|
||||
base = filename.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
total_rows = 0
|
||||
for ws in wb.worksheets:
|
||||
values = list(ws.iter_rows(values_only=True))
|
||||
if not values:
|
||||
continue
|
||||
headers = [str(h).strip() if h is not None else f"Column {i + 1}" for i, h in enumerate(values[0])]
|
||||
rows: list[dict[str, Any]] = []
|
||||
for row in values[1:]:
|
||||
if row is None or all(c is None or str(c).strip() == "" for c in row):
|
||||
continue
|
||||
rows.append({headers[i]: row[i] for i in range(min(len(headers), len(row)))})
|
||||
if not rows:
|
||||
continue
|
||||
name = f"{base} — {ws.title}" if len(wb.worksheets) > 1 else (base or ws.title)
|
||||
page = rows_to_collection(name, headers, rows)
|
||||
page.source_path = f"{filename}#{ws.title}"
|
||||
page.external_id = page.source_path
|
||||
result.pages.append(page)
|
||||
total_rows += len(rows)
|
||||
result.stats["rows"] = total_rows
|
||||
return result.finalize()
|
||||
|
||||
|
||||
@register_importer
|
||||
class JsonImporter(Importer):
|
||||
source_id = "json"
|
||||
label = "JSON (mapping générique)"
|
||||
description = "Tableau d'objets JSON → collection (union des clés)."
|
||||
extensions = (".json",)
|
||||
order = 65
|
||||
|
||||
def _records(self, data: bytes) -> list[dict] | None:
|
||||
try:
|
||||
obj = json.loads(decode_text(data))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if isinstance(obj, list) and obj and all(isinstance(x, dict) for x in obj):
|
||||
return obj
|
||||
if isinstance(obj, dict):
|
||||
for value in obj.values():
|
||||
if isinstance(value, list) and value and all(isinstance(x, dict) for x in value):
|
||||
return value
|
||||
return None
|
||||
|
||||
def detect(self, filename: str, data: bytes) -> bool:
|
||||
if not filename.lower().endswith(".json"):
|
||||
return False
|
||||
return self._records(data) is not None
|
||||
|
||||
def parse(self, filename: str, data: bytes) -> ImportResult:
|
||||
result = ImportResult(source=self.source_id)
|
||||
records = self._records(data)
|
||||
if not records:
|
||||
result.warn("Aucun tableau d'objets JSON détecté")
|
||||
return result.finalize()
|
||||
headers: list[str] = []
|
||||
for rec in records:
|
||||
for key in rec:
|
||||
if key not in headers:
|
||||
headers.append(key)
|
||||
flat: list[dict[str, Any]] = []
|
||||
for rec in records:
|
||||
row = {}
|
||||
for h in headers:
|
||||
v = rec.get(h)
|
||||
row[h] = json.dumps(v, ensure_ascii=False) if isinstance(v, (dict, list)) else v
|
||||
flat.append(row)
|
||||
name = filename.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
result.pages.append(rows_to_collection(name, headers, flat))
|
||||
result.stats["rows"] = len(flat)
|
||||
return result.finalize()
|
||||
@@ -0,0 +1,94 @@
|
||||
"""FlowDeck — URL / web clipper importer (v5.6.0, Phase 5).
|
||||
|
||||
Fetches a web page and turns it into a page: a bookmark card (OG metadata)
|
||||
followed by the article converted to FlowDeck blocks.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.export import markdown_to_blocks
|
||||
from app.services.importers.base import ImportPage, ImportResult
|
||||
from app.services.importers.html_notes import _html_to_markdown
|
||||
|
||||
_BLOCKED_HOSTS = {"localhost", "localhost.localdomain"}
|
||||
_MAX_BYTES = 3_000_000
|
||||
|
||||
|
||||
def _is_public_host(host: str) -> bool:
|
||||
"""SSRF guard: reject loopback/private/link-local/reserved addresses."""
|
||||
if not host or host.lower() in _BLOCKED_HOSTS:
|
||||
return False
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
for info in infos:
|
||||
address = info[4][0]
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
except ValueError:
|
||||
return False
|
||||
if (ip.is_private or ip.is_loopback or ip.is_link_local
|
||||
or ip.is_reserved or ip.is_multicast or ip.is_unspecified):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_url(url: str) -> str:
|
||||
parsed = urlparse(url.strip())
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise ValueError("Seules les URLs http(s) sont autorisées")
|
||||
if not parsed.hostname or not _is_public_host(parsed.hostname):
|
||||
raise ValueError("Hôte non autorisé")
|
||||
return url.strip()
|
||||
|
||||
|
||||
def _bookmark_block(url: str, meta: dict) -> dict:
|
||||
block = {"type": "bookmark", "url": url}
|
||||
for key in ("title", "description", "image", "site_name"):
|
||||
if meta.get(key):
|
||||
block[key] = meta[key]
|
||||
return block
|
||||
|
||||
|
||||
async def fetch_url_result(url: str, *, transport: httpx.BaseTransport | None = None) -> ImportResult:
|
||||
"""Fetch ``url`` and build a single-page ImportResult (raises on bad URL)."""
|
||||
safe_url = _validate_url(url)
|
||||
result = ImportResult(source="url")
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=15, follow_redirects=True, transport=transport,
|
||||
headers={"User-Agent": "FlowDeck-Importer/1.0"},
|
||||
) as client:
|
||||
response = await client.get(safe_url)
|
||||
response.raise_for_status()
|
||||
if response.url.host and not _is_public_host(response.url.host):
|
||||
raise ValueError("Redirection vers un hôte non autorisé")
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if "html" not in content_type.lower():
|
||||
raise ValueError("La ressource n'est pas une page HTML")
|
||||
body = response.text[:_MAX_BYTES]
|
||||
except httpx.HTTPError as exc:
|
||||
result.warn(f"Échec du téléchargement : {exc}")
|
||||
return result.finalize()
|
||||
|
||||
from app.services.og_fetcher import parse_og
|
||||
|
||||
meta = parse_og(body, safe_url)
|
||||
title = (meta.get("title") or urlparse(safe_url).hostname or "Page").strip()
|
||||
markdown = _html_to_markdown(body)
|
||||
blocks = [_bookmark_block(safe_url, meta)]
|
||||
if markdown:
|
||||
blocks.extend(markdown_to_blocks(markdown))
|
||||
result.pages.append(ImportPage(
|
||||
title=title[:200],
|
||||
blocks=blocks,
|
||||
source_path=safe_url,
|
||||
external_id=safe_url,
|
||||
))
|
||||
return result.finalize()
|
||||
@@ -827,6 +827,9 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div style="padding:12px 20px;border-top:1px solid var(--border);display:flex;justify-content:flex-end;">
|
||||
<a href="/import" style="font-size:13px;color:var(--accent);text-decoration:none;">Assistant d'import avancé (Obsidian, Notion, CSV, Excel…) →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="fr" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Importer — FlowDeck</title>
|
||||
<link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
|
||||
<script src="/static/js/alpine.min.js" defer data-cfasync="false"></script>
|
||||
<style>
|
||||
:root{--bg:#191919;--bg2:#1F1F1F;--bg3:#2D2D2D;--border:#333;--text:#fff;--dim:#9B9A97;--accent:#2383E2;--accent-h:#1a6bc0;--success:#0F7B6C;--danger:#E03E3E;--warn:#D9730D;}
|
||||
*{margin:0;padding:0;box-sizing:border-box;}
|
||||
body{font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:var(--bg);color:var(--text);min-height:100vh;padding:32px 20px;}
|
||||
.wrap{width:860px;max-width:96vw;margin:0 auto;}
|
||||
.brand{font-size:15px;font-weight:700;margin-bottom:22px;display:flex;align-items:center;gap:8px;color:var(--dim);}
|
||||
.brand a{color:var(--dim);text-decoration:none;}
|
||||
.brand a:hover{color:var(--text);}
|
||||
h1{font-size:24px;font-weight:700;margin-bottom:6px;}
|
||||
.sub{color:var(--dim);font-size:14px;margin-bottom:24px;line-height:1.5;}
|
||||
.panel{background:var(--bg2);border:1px solid var(--border);border-radius:14px;padding:22px;margin-bottom:18px;}
|
||||
.label{font-size:12px;font-weight:600;text-transform:uppercase;letter-spacing:.5px;color:var(--dim);margin-bottom:10px;}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(180px,1fr));gap:10px;}
|
||||
.src{background:var(--bg3);border:1px solid var(--border);border-radius:10px;padding:12px;cursor:pointer;transition:border-color .15s;}
|
||||
.src:hover{border-color:var(--accent);}
|
||||
.src.selected{border-color:var(--accent);box-shadow:0 0 0 2px rgba(35,131,226,.25);}
|
||||
.src .t{font-size:13px;font-weight:600;}
|
||||
.src .d{font-size:11px;color:var(--dim);margin-top:3px;line-height:1.35;}
|
||||
.drop{border:2px dashed var(--border);border-radius:12px;padding:26px;text-align:center;color:var(--dim);cursor:pointer;transition:border-color .15s,background .15s;}
|
||||
.drop:hover,.drop.over{border-color:var(--accent);background:rgba(35,131,226,.06);color:var(--text);}
|
||||
.drop strong{color:var(--text);}
|
||||
.btn{display:inline-flex;align-items:center;justify-content:center;gap:8px;padding:10px 18px;border:none;border-radius:10px;font-size:14px;font-weight:600;cursor:pointer;transition:background .15s;}
|
||||
.btn-primary{background:var(--accent);color:#fff;}
|
||||
.btn-primary:hover{background:var(--accent-h);}
|
||||
.btn-ghost{background:transparent;color:var(--dim);border:1px solid var(--border);}
|
||||
.btn-ghost:hover{color:var(--text);}
|
||||
.btn:disabled{opacity:.5;cursor:not-allowed;}
|
||||
.actions{display:flex;gap:10px;align-items:center;margin-top:16px;flex-wrap:wrap;}
|
||||
.check{display:flex;align-items:center;gap:8px;font-size:13px;color:var(--dim);margin-top:12px;cursor:pointer;}
|
||||
.bar{height:6px;background:var(--bg3);border-radius:3px;overflow:hidden;margin-top:16px;}
|
||||
.bar>i{display:block;height:100%;background:var(--accent);width:0;transition:width .3s;}
|
||||
table{width:100%;border-collapse:collapse;font-size:13px;}
|
||||
th,td{text-align:left;padding:8px 10px;border-bottom:1px solid var(--border);}
|
||||
th{color:var(--dim);font-weight:600;font-size:11px;text-transform:uppercase;letter-spacing:.4px;}
|
||||
.pill{font-size:11px;font-weight:600;padding:2px 8px;border-radius:99px;background:var(--bg3);color:var(--dim);}
|
||||
.pill.coll{background:rgba(35,131,226,.16);color:#79b8ff;}
|
||||
.pill.ok{background:rgba(15,123,108,.2);color:#4fd1c5;}
|
||||
.pill.err{background:rgba(224,62,62,.18);color:#f28b82;}
|
||||
.pill.run{background:rgba(217,115,13,.18);color:#e3a15b;}
|
||||
select,input[type=text]{background:var(--bg3);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:8px 10px;font-size:13px;}
|
||||
.warnings{font-size:12px;color:var(--warn);margin-top:10px;line-height:1.5;}
|
||||
.report{display:grid;grid-template-columns:repeat(auto-fit,minmax(110px,1fr));gap:10px;}
|
||||
.stat{background:var(--bg3);border-radius:10px;padding:12px;text-align:center;}
|
||||
.stat .n{font-size:22px;font-weight:700;}
|
||||
.stat .l{font-size:11px;color:var(--dim);margin-top:2px;}
|
||||
.error{color:var(--danger);font-size:13px;margin-top:10px;}
|
||||
.x-cloak,[x-cloak]{display:none!important;}
|
||||
</style>
|
||||
</head>
|
||||
<body x-data="importWizard()" x-init="init()">
|
||||
<div class="wrap">
|
||||
<div class="brand">
|
||||
<svg viewBox="0 0 24 24" width="18" height="18" fill="none" stroke="#2383E2" stroke-width="2"><path d="M13 2L3 14h7l-1 8 10-12h-7l1-8z"/></svg>
|
||||
<a href="/">FlowDeck</a> · Import
|
||||
</div>
|
||||
<h1>Importer des données</h1>
|
||||
<p class="sub">Notes, vaults, bases de données, documents ou signets — choisissez une source ou laissez la détection automatique.</p>
|
||||
|
||||
<div class="panel">
|
||||
<div class="label">1 · Source</div>
|
||||
<div class="grid">
|
||||
<div class="src" :class="{selected: sourceId===''}" @click="sourceId=''">
|
||||
<div class="t">✨ Détection auto</div>
|
||||
<div class="d">Reconnaît le format d'après le fichier.</div>
|
||||
</div>
|
||||
<template x-for="s in sources" :key="s.source_id">
|
||||
<div class="src" :class="{selected: sourceId===s.source_id}" @click="sourceId=s.source_id">
|
||||
<div class="t" x-text="s.label"></div>
|
||||
<div class="d" x-text="s.description"></div>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="label">2 · Fichiers <span style="text-transform:none;font-weight:400">(sélection multiple possible)</span></div>
|
||||
<div class="drop" :class="{over: dragOver}"
|
||||
@click="$refs.input.click()"
|
||||
@dragover.prevent="dragOver=true" @dragleave.prevent="dragOver=false"
|
||||
@drop.prevent="onDrop($event)">
|
||||
<span>Glissez un ou plusieurs fichiers ici ou <strong>cliquez pour parcourir</strong><br><small>Markdown, .zip, .csv, .tsv, .xlsx, .json, .html, .docx, .pdf, .ics, .opml…</small></span>
|
||||
</div>
|
||||
<input x-ref="input" type="file" multiple style="display:none"
|
||||
accept=".md,.markdown,.txt,.zip,.csv,.tsv,.xlsx,.xlsm,.json,.html,.htm,.docx,.docm,.pdf,.ics,.ical,.opml"
|
||||
@change="onFiles($event)">
|
||||
<template x-if="files.length">
|
||||
<table style="margin-top:12px;">
|
||||
<thead><tr><th>Fichier</th><th>Taille</th><th>Statut</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<template x-for="(f,i) in files" :key="i">
|
||||
<tr>
|
||||
<td x-text="f.name"></td>
|
||||
<td x-text="humanSize(f.size)"></td>
|
||||
<td><span class="pill" :class="f.status==='done'?'ok':(f.status==='error'?'err':(f.status==='running'?'run':''))" x-text="f.status"></span></td>
|
||||
<td><button class="btn btn-ghost" style="padding:2px 8px" @click="files.splice(i,1)" :disabled="busy">✕</button></td>
|
||||
</tr>
|
||||
</template>
|
||||
</tbody>
|
||||
</table>
|
||||
</template>
|
||||
<div style="display:flex;gap:16px;align-items:center;flex-wrap:wrap;margin-top:12px;">
|
||||
<label class="check" style="margin:0">Mode de ré-import :
|
||||
<select x-model="mode">
|
||||
<option value="skip">Ignorer l'existant</option>
|
||||
<option value="update">Mettre à jour (re-sync)</option>
|
||||
<option value="duplicate">Toujours dupliquer</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<button class="btn btn-ghost" @click="doPreview()" :disabled="!files.length || busy">Aperçu (dry-run)</button>
|
||||
<button class="btn btn-primary" @click="doImport()" :disabled="!files.length || busy">Importer</button>
|
||||
</div>
|
||||
<div class="bar" x-show="busy || progress>0"><i :style="'width:'+progress+'%'"></i></div>
|
||||
<div class="error" x-show="error" x-text="error"></div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="label">Ou · importer une page web (URL)</div>
|
||||
<div style="display:flex;gap:8px;flex-wrap:wrap;align-items:center;">
|
||||
<input type="text" style="flex:1;min-width:260px;" x-model="urlValue" placeholder="https://exemple.com/article" @keydown.enter="doUrl()">
|
||||
<button class="btn btn-ghost" @click="doUrl()" :disabled="!urlValue || busy">Importer l'URL</button>
|
||||
</div>
|
||||
<div class="error" x-show="urlError" x-text="urlError"></div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="label">Ou · importer depuis une forge</div>
|
||||
<div style="display:flex;gap:8px;flex-wrap:wrap;align-items:center;">
|
||||
<select x-model="forgeProvider">
|
||||
<option value="gitea">Gitea</option>
|
||||
<option value="github">GitHub</option>
|
||||
</select>
|
||||
<input type="text" style="width:150px;" x-model="forgeOwner" placeholder="owner">
|
||||
<input type="text" style="width:150px;" x-model="forgeRepo" placeholder="repo">
|
||||
<select x-model="forgeState">
|
||||
<option value="all">Toutes</option>
|
||||
<option value="open">Ouvertes</option>
|
||||
<option value="closed">Fermées</option>
|
||||
</select>
|
||||
<button class="btn btn-ghost" @click="doForge('issues')" :disabled="!forgeOwner || !forgeRepo || busy">Importer les issues</button>
|
||||
<button class="btn btn-ghost" @click="doForge('repo')" :disabled="!forgeOwner || !forgeRepo || busy">Importer les fichiers</button>
|
||||
</div>
|
||||
<div class="error" x-show="forgeError" x-text="forgeError"></div>
|
||||
</div>
|
||||
|
||||
<div class="panel" x-show="preview" x-cloak>
|
||||
<div class="label">Aperçu — <span x-text="preview && preview.source_label"></span></div>
|
||||
<table>
|
||||
<thead><tr><th>Titre</th><th>Type</th><th>Éléments</th><th>Colonnes</th></tr></thead>
|
||||
<tbody>
|
||||
<template x-for="(p,i) in (preview ? preview.pages : [])" :key="i">
|
||||
<tr>
|
||||
<td x-text="p.title"></td>
|
||||
<td><span class="pill" :class="{coll: p.type==='collection'}" x-text="p.type==='collection' ? 'database' : 'page'"></span></td>
|
||||
<td x-text="p.type==='collection' ? (p.rows+' lignes') : (p.blocks+' blocs')"></td>
|
||||
<td>
|
||||
<template x-for="col in (p.schema||[])" :key="col.name">
|
||||
<span style="display:inline-flex;align-items:center;gap:4px;margin:2px 6px 2px 0;">
|
||||
<span x-text="col.name" style="font-size:12px;"></span>
|
||||
<select @change="mapping[col.name]=$event.target.value">
|
||||
<template x-for="t in types" :key="t">
|
||||
<option :value="t" :selected="t===col.type" x-text="t"></option>
|
||||
</template>
|
||||
</select>
|
||||
</span>
|
||||
</template>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="warnings" x-show="preview && preview.warnings && preview.warnings.length">
|
||||
<template x-for="(w,i) in (preview ? preview.warnings : [])" :key="i"><div x-text="'⚠ '+w"></div></template>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel" x-show="report" x-cloak>
|
||||
<div class="label">Rapport d'import</div>
|
||||
<div class="report">
|
||||
<div class="stat"><div class="n" x-text="report ? report.pages_created : 0"></div><div class="l">Pages créées</div></div>
|
||||
<div class="stat"><div class="n" x-text="report ? report.pages_updated : 0"></div><div class="l">Mises à jour</div></div>
|
||||
<div class="stat"><div class="n" x-text="report ? report.collections_created : 0"></div><div class="l">Collections</div></div>
|
||||
<div class="stat"><div class="n" x-text="report ? report.rows_created : 0"></div><div class="l">Lignes</div></div>
|
||||
<div class="stat"><div class="n" x-text="report ? report.attachments : 0"></div><div class="l">Fichiers</div></div>
|
||||
<div class="stat"><div class="n" x-text="report ? report.skipped : 0"></div><div class="l">Ignorés</div></div>
|
||||
<div class="stat"><div class="n" x-text="report ? (report.errors ? report.errors.length : 0) : 0"></div><div class="l">Erreurs</div></div>
|
||||
</div>
|
||||
<div class="warnings" x-show="report && report.warnings && report.warnings.length">
|
||||
<template x-for="(w,i) in (report ? report.warnings : [])" :key="i"><div x-text="'⚠ '+w"></div></template>
|
||||
</div>
|
||||
<div class="warnings" x-show="report && report.errors && report.errors.length">
|
||||
<template x-for="(e,i) in (report ? report.errors : [])" :key="i"><div x-text="'✕ '+e.title+' : '+e.error"></div></template>
|
||||
</div>
|
||||
<div class="warnings" x-show="report && report.relations && report.relations.relations_resolved">
|
||||
<div x-text="'🔗 '+report.relations.relations_resolved+' relation(s) reconstruite(s)'"></div>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<button class="btn btn-ghost" @click="downloadReport()">Télécharger le rapport (JSON)</button>
|
||||
<a class="btn btn-primary" href="/workspaces">Retour au workspace</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function importWizard() {
|
||||
return {
|
||||
sources: [],
|
||||
sourceId: '',
|
||||
files: [],
|
||||
dragOver: false,
|
||||
mode: 'skip',
|
||||
busy: false,
|
||||
progress: 0,
|
||||
error: '',
|
||||
preview: null,
|
||||
report: null,
|
||||
mapping: {},
|
||||
csrf: '',
|
||||
urlValue: '',
|
||||
urlError: '',
|
||||
forgeProvider: 'gitea',
|
||||
forgeOwner: '',
|
||||
forgeRepo: '',
|
||||
forgeState: 'all',
|
||||
forgeError: '',
|
||||
types: ['text','number','date','checkbox','email','url','phone','select','multi_select','status'],
|
||||
async init() {
|
||||
try {
|
||||
const r = await fetch('/api/import/sources');
|
||||
this.sources = (await r.json()).sources || [];
|
||||
} catch(e) {}
|
||||
try {
|
||||
const c = await fetch('/api/csrf-token');
|
||||
this.csrf = (await c.json()).csrf_token;
|
||||
} catch(e) {}
|
||||
},
|
||||
humanSize(n) { return n > 1048576 ? (n/1048576).toFixed(1)+' Mo' : Math.max(1, Math.round(n/1024))+' Ko'; },
|
||||
addFiles(list) {
|
||||
for (const f of list) this.files.push({file:f, name:f.name, size:f.size, status:'queued'});
|
||||
this.preview = null; this.report = null; this.error = '';
|
||||
},
|
||||
onFiles(e) { this.addFiles(e.target.files); e.target.value=''; },
|
||||
onDrop(e) { this.dragOver=false; this.addFiles(e.dataTransfer.files); },
|
||||
buildForm(f) {
|
||||
const fd = new FormData();
|
||||
fd.append('file', f.file);
|
||||
if (this.sourceId) fd.append('source', this.sourceId);
|
||||
fd.append('mode', this.mode);
|
||||
return fd;
|
||||
},
|
||||
async doPreview() {
|
||||
this.busy = true; this.error=''; this.report=null; this.progress=10;
|
||||
try {
|
||||
let merged = null;
|
||||
for (let i=0;i<this.files.length;i++) {
|
||||
const r = await fetch('/api/import/preview', {method:'POST', headers:{'X-CSRF-Token':this.csrf}, body:this.buildForm(this.files[i])});
|
||||
const d = await r.json();
|
||||
if (!r.ok) { this.error = (this.files[i].name+': '+(d.detail||'Erreur')); continue; }
|
||||
if (!merged) merged = {source_label:d.source_label, pages:[], warnings:[], stats:{}};
|
||||
merged.pages.push(...(d.pages||[]));
|
||||
merged.warnings.push(...(d.warnings||[]));
|
||||
this.progress = Math.round(((i+1)/this.files.length)*100);
|
||||
}
|
||||
this.preview = merged;
|
||||
this.mapping = {};
|
||||
(merged ? merged.pages : []).forEach(p => (p.schema||[]).forEach(c => { this.mapping[c.name]=c.type; }));
|
||||
} catch(e) { this.error = 'Erreur réseau'; }
|
||||
finally { this.busy = false; setTimeout(()=>{this.progress=0;},800); }
|
||||
},
|
||||
async importOne(f) {
|
||||
f.status = 'running';
|
||||
const fd = this.buildForm(f);
|
||||
if (Object.keys(this.mapping).length) fd.append('mapping', JSON.stringify(this.mapping));
|
||||
const big = f.size > 5*1024*1024;
|
||||
try {
|
||||
if (big) {
|
||||
fd.append('async','true');
|
||||
const r = await fetch('/api/import/run', {method:'POST', headers:{'X-CSRF-Token':this.csrf}, body:fd});
|
||||
const d = await r.json();
|
||||
if (!r.ok) { f.status='error'; f.error=d.detail||'Erreur'; return null; }
|
||||
return await this.pollJob(d.job_id, f);
|
||||
}
|
||||
const r = await fetch('/api/import/run', {method:'POST', headers:{'X-CSRF-Token':this.csrf}, body:fd});
|
||||
const d = await r.json();
|
||||
if (!r.ok) { f.status='error'; f.error=d.detail||'Erreur'; return null; }
|
||||
f.status = (d.status==='partial' ? 'partial' : 'done');
|
||||
return d;
|
||||
} catch(e) { f.status='error'; f.error='Erreur réseau'; return null; }
|
||||
},
|
||||
async doImport() {
|
||||
this.busy = true; this.error=''; this.report=null; this.progress=0;
|
||||
const aggregate = {pages_created:0,pages_updated:0,collections_created:0,rows_created:0,
|
||||
attachments:0,skipped:0,warnings:[],errors:[],per_file:[]};
|
||||
for (let i=0;i<this.files.length;i++) {
|
||||
const d = await this.importOne(this.files[i]);
|
||||
if (d) {
|
||||
for (const k of ['pages_created','pages_updated','collections_created','rows_created','attachments','skipped'])
|
||||
aggregate[k] += (d[k]||0);
|
||||
aggregate.warnings.push(...(d.warnings||[]));
|
||||
aggregate.errors.push(...(d.errors||[]));
|
||||
aggregate.per_file.push({filename:this.files[i].name, report:d});
|
||||
} else {
|
||||
aggregate.errors.push({title:this.files[i].name, error:this.files[i].error||'Erreur'});
|
||||
aggregate.per_file.push({filename:this.files[i].name, report:{status:'error'}});
|
||||
}
|
||||
this.progress = Math.round(((i+1)/this.files.length)*100);
|
||||
}
|
||||
this.report = aggregate;
|
||||
this.busy = false;
|
||||
},
|
||||
async doUrl() {
|
||||
this.busy = true; this.urlError = ''; this.report = null; this.progress = 40;
|
||||
try {
|
||||
const r = await fetch('/api/import/url', {
|
||||
method:'POST', headers:{'Content-Type':'application/json','X-CSRF-Token':this.csrf},
|
||||
body: JSON.stringify({url:this.urlValue.trim()})
|
||||
});
|
||||
const d = await r.json();
|
||||
if (!r.ok) { this.urlError = d.detail || 'Erreur'; return; }
|
||||
this.report = d; this.progress = 100;
|
||||
} catch(e) { this.urlError = 'Erreur réseau'; }
|
||||
finally { this.busy = false; }
|
||||
},
|
||||
async doForge(kind) {
|
||||
this.busy = true; this.forgeError = ''; this.report = null; this.progress = 30;
|
||||
const endpoint = kind === 'repo' ? '/api/import/forge-repo' : '/api/import/forge';
|
||||
try {
|
||||
const r = await fetch(endpoint, {
|
||||
method:'POST', headers:{'Content-Type':'application/json','X-CSRF-Token':this.csrf},
|
||||
body: JSON.stringify({
|
||||
provider:this.forgeProvider, owner:this.forgeOwner.trim(),
|
||||
repo:this.forgeRepo.trim(), state:this.forgeState
|
||||
})
|
||||
});
|
||||
const d = await r.json();
|
||||
if (!r.ok) { this.forgeError = d.detail || 'Erreur'; return; }
|
||||
this.report = d; this.progress = 100;
|
||||
} catch(e) { this.forgeError = 'Erreur réseau'; }
|
||||
finally { this.busy = false; }
|
||||
},
|
||||
async pollJob(jobId, f) {
|
||||
for (let i=0;i<600;i++) {
|
||||
await new Promise(res => setTimeout(res, 700));
|
||||
const r = await fetch('/api/import/jobs/'+jobId);
|
||||
const j = await r.json();
|
||||
if (j.status === 'done') { if (f) f.status='done'; return j.report; }
|
||||
if (j.status === 'error') { if (f) { f.status='error'; f.error=j.error; } return null; }
|
||||
}
|
||||
if (f) { f.status='error'; f.error='Délai dépassé'; }
|
||||
return null;
|
||||
},
|
||||
downloadReport() {
|
||||
const blob = new Blob([JSON.stringify(this.report, null, 2)], {type:'application/json'});
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(blob);
|
||||
a.download = 'flowdeck-import-report.json';
|
||||
a.click();
|
||||
URL.revokeObjectURL(a.href);
|
||||
}
|
||||
};
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user