bruno
3b00cbc371
- unified importer framework (app/services/importers/): normalized model, registry, common pipeline (hierarchy, attachments, collections, dedup), async jobs, dry-run preview, column->type mapping - Phase 1: Obsidian, Notion, Logseq/Roam, HTML (Apple Notes/Bear/Ulysses/ OneNote), Google Keep, generic Markdown - Phase 2: typed CSV/TSV, Excel (openpyxl), generic JSON - Phase 3: Word .docx (python-docx), PDF (pypdf), HTML folders - Phase 4: Raindrop, Pocket, Readwise, Shaarli, Netscape bookmarks, .ics, OPML, Standard Notes, Gitea/GitHub issues (+labels/milestones) - Phase 5: incremental re-sync (skip/update/duplicate), partial-error resume, forge repo files, URL web clipper (SSRF guard), batch multi-file + UI queue, Notion relation resolution, exportable JSON reports - /import wizard, API /api/import/*, migration 9 (import_items, import_jobs) - fix: property values stored by property id (correct DB view rendering) - deps: openpyxl, beautifulsoup4, PyYAML, python-docx, pypdf - 43 import tests; full suite 491 green; ruff clean - bump version 5.11.5
306 lines
11 KiB
Python
306 lines
11 KiB
Python
"""FlowDeck — tabular importers: typed CSV/TSV, Excel, generic JSON (v5.6.0, Phase 2).
|
|
|
|
Each source becomes a FlowDeck collection (database): columns are inferred from
|
|
the data (text/number/date/checkbox/email/url/select/multi_select) and rows are
|
|
inserted as ``collection_pages``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
import json
|
|
import re
|
|
from typing import Any
|
|
|
|
from app.services.importers.base import (
|
|
Importer,
|
|
ImportPage,
|
|
ImportResult,
|
|
decode_text,
|
|
register_importer,
|
|
)
|
|
|
|
_TITLE_HEADERS = ("title", "name", "task", "nom", "titre", "subject", "label")
|
|
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2})?)?")
|
|
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
|
_URL_RE = re.compile(r"^https?://\S+$", re.IGNORECASE)
|
|
_BOOL_TRUE = {"true", "yes", "oui", "1", "x", "vrai"}
|
|
_BOOL_FALSE = {"false", "no", "non", "0", "faux", ""}
|
|
|
|
|
|
def _is_number(value: str) -> bool:
|
|
try:
|
|
float(str(value).replace(",", ".").replace(" ", ""))
|
|
return True
|
|
except (ValueError, TypeError):
|
|
return False
|
|
|
|
|
|
def _is_bool(value: str) -> bool:
|
|
return str(value).strip().lower() in _BOOL_TRUE | _BOOL_FALSE
|
|
|
|
|
|
def infer_column_type(values: list[str]) -> str:
|
|
"""Infer the FlowDeck property type from a list of raw string cells."""
|
|
sample = [str(v).strip() for v in values if str(v).strip()]
|
|
if not sample:
|
|
return "text"
|
|
if all(_is_bool(v) for v in sample):
|
|
return "checkbox"
|
|
if all(_is_number(v) for v in sample):
|
|
return "number"
|
|
if all(_DATE_RE.match(v) for v in sample):
|
|
return "date"
|
|
if all(_EMAIL_RE.match(v) for v in sample):
|
|
return "email"
|
|
if all(_URL_RE.match(v) for v in sample):
|
|
return "url"
|
|
unique = {v for v in sample}
|
|
if len(unique) <= 20 and len(unique) <= max(2, len(sample) // 2):
|
|
if any(("," in v or ";" in v) for v in sample):
|
|
return "multi_select"
|
|
return "select"
|
|
return "text"
|
|
|
|
|
|
def _split_multi(value: str) -> list[str]:
|
|
return [p.strip() for p in re.split(r"[;,]", value) if p.strip()]
|
|
|
|
|
|
def coerce_value(prop_type: str, value: Any) -> Any:
|
|
if value is None:
|
|
return None
|
|
raw = str(value).strip()
|
|
if raw == "":
|
|
return None
|
|
if prop_type == "number":
|
|
try:
|
|
num = float(raw.replace(",", ".").replace(" ", ""))
|
|
return int(num) if num.is_integer() else num
|
|
except (ValueError, TypeError):
|
|
return raw
|
|
if prop_type == "checkbox":
|
|
return raw.lower() in _BOOL_TRUE
|
|
if prop_type == "multi_select":
|
|
return _split_multi(raw)
|
|
return raw
|
|
|
|
|
|
def build_schema(headers: list[str], rows: list[dict[str, Any]]) -> tuple[list[dict], str]:
|
|
"""Return ``(schema, title_header)`` from headers + row dicts."""
|
|
title_header = ""
|
|
for h in headers:
|
|
if h and h.strip().lower() in _TITLE_HEADERS:
|
|
title_header = h
|
|
break
|
|
schema: list[dict] = []
|
|
for h in headers:
|
|
if not h or h == title_header:
|
|
continue
|
|
ptype = infer_column_type([r.get(h, "") for r in rows])
|
|
entry: dict[str, Any] = {"name": h, "type": ptype}
|
|
if ptype in ("select", "status", "multi_select"):
|
|
seen: list[str] = []
|
|
for r in rows:
|
|
vals = _split_multi(str(r.get(h, ""))) if ptype == "multi_select" else [str(r.get(h, "")).strip()]
|
|
for v in vals:
|
|
if v and v not in seen:
|
|
seen.append(v)
|
|
entry["options"] = [{"name": v, "color": "gray"} for v in seen[:100]]
|
|
schema.append(entry)
|
|
if title_header:
|
|
schema.insert(0, {"name": title_header, "type": "title"})
|
|
return schema, title_header
|
|
|
|
|
|
def rows_to_collection(name: str, headers: list[str], raw_rows: list[dict[str, Any]]) -> ImportPage:
|
|
"""Normalize parsed rows into an ImportPage carrying a collection spec."""
|
|
schema, title_header = build_schema(headers, raw_rows)
|
|
rows = _normalize_rows(headers, raw_rows, schema, title_header)
|
|
return ImportPage(
|
|
title=name or "Imported database",
|
|
collection={
|
|
"name": name or "Imported database",
|
|
"schema": schema,
|
|
"rows": rows,
|
|
"headers": headers,
|
|
"title_header": title_header,
|
|
"raw_rows": raw_rows,
|
|
},
|
|
source_path=name,
|
|
external_id=name,
|
|
)
|
|
|
|
|
|
def _normalize_rows(headers: list[str], raw_rows: list[dict[str, Any]],
|
|
schema: list[dict], title_header: str) -> list[dict[str, Any]]:
|
|
types = {s["name"]: s["type"] for s in schema}
|
|
rows: list[dict[str, Any]] = []
|
|
for raw in raw_rows:
|
|
title = ""
|
|
if title_header:
|
|
title = str(raw.get(title_header, "")).strip()
|
|
if not title:
|
|
for h in headers:
|
|
if h and str(raw.get(h, "")).strip():
|
|
title = str(raw[h]).strip()
|
|
break
|
|
props: dict[str, Any] = {}
|
|
for h in headers:
|
|
if not h or h == title_header:
|
|
continue
|
|
val = coerce_value(types.get(h, "text"), raw.get(h))
|
|
if val is not None and val != "":
|
|
props[h] = val
|
|
rows.append({"title": title or "Untitled", "properties": props})
|
|
return rows
|
|
|
|
|
|
def apply_type_mapping(spec: dict, mapping: dict[str, str]) -> dict:
|
|
"""Override inferred column types (UI mapping) and re-coerce the rows."""
|
|
if not mapping:
|
|
return spec
|
|
for entry in spec.get("schema", []):
|
|
if entry.get("name") in mapping:
|
|
entry["type"] = mapping[entry["name"]]
|
|
headers = spec.get("headers")
|
|
raw_rows = spec.get("raw_rows")
|
|
if headers is not None and raw_rows is not None:
|
|
spec["rows"] = _normalize_rows(headers, raw_rows, spec.get("schema", []),
|
|
spec.get("title_header", ""))
|
|
return spec
|
|
|
|
|
|
def _sniff_delimiter(sample: str) -> str:
|
|
try:
|
|
return csv.Sniffer().sniff(sample, delimiters=",;\t|").delimiter
|
|
except csv.Error:
|
|
return "\t" if sample.count("\t") > sample.count(",") else ","
|
|
|
|
|
|
@register_importer
|
|
class CsvImporter(Importer):
|
|
source_id = "csv"
|
|
label = "CSV / TSV (typé)"
|
|
description = "Tableur CSV/TSV : types inférés automatiquement, une collection par fichier."
|
|
extensions = (".csv", ".tsv")
|
|
order = 60
|
|
|
|
def detect(self, filename: str, data: bytes) -> bool:
|
|
return filename.lower().endswith((".csv", ".tsv"))
|
|
|
|
def parse(self, filename: str, data: bytes) -> ImportResult:
|
|
result = ImportResult(source=self.source_id)
|
|
text = decode_text(data)
|
|
if not text.strip():
|
|
result.warn("Fichier vide")
|
|
return result.finalize()
|
|
delimiter = "\t" if filename.lower().endswith(".tsv") else _sniff_delimiter(text[:4096])
|
|
reader = csv.DictReader(io.StringIO(text), delimiter=delimiter)
|
|
headers = [h for h in (reader.fieldnames or []) if h is not None]
|
|
rows = [dict(r) for r in reader]
|
|
name = filename.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
|
result.pages.append(rows_to_collection(name, headers, rows))
|
|
result.stats["rows"] = len(rows)
|
|
return result.finalize()
|
|
|
|
|
|
@register_importer
|
|
class ExcelImporter(Importer):
|
|
source_id = "excel"
|
|
label = "Excel (.xlsx)"
|
|
description = "Classeur Excel : une collection par feuille (openpyxl)."
|
|
extensions = (".xlsx", ".xlsm")
|
|
order = 61
|
|
|
|
def detect(self, filename: str, data: bytes) -> bool:
|
|
low = filename.lower()
|
|
if low.endswith((".xlsx", ".xlsm")):
|
|
return True
|
|
return low.endswith(".xls")
|
|
|
|
def parse(self, filename: str, data: bytes) -> ImportResult:
|
|
result = ImportResult(source=self.source_id)
|
|
try:
|
|
from openpyxl import load_workbook
|
|
except ImportError:
|
|
result.warn("openpyxl n'est pas installé : import Excel indisponible")
|
|
return result.finalize()
|
|
try:
|
|
wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
|
except Exception as exc: # noqa: BLE001
|
|
result.warn(f"Classeur illisible : {exc}")
|
|
return result.finalize()
|
|
base = filename.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
|
total_rows = 0
|
|
for ws in wb.worksheets:
|
|
values = list(ws.iter_rows(values_only=True))
|
|
if not values:
|
|
continue
|
|
headers = [str(h).strip() if h is not None else f"Column {i + 1}" for i, h in enumerate(values[0])]
|
|
rows: list[dict[str, Any]] = []
|
|
for row in values[1:]:
|
|
if row is None or all(c is None or str(c).strip() == "" for c in row):
|
|
continue
|
|
rows.append({headers[i]: row[i] for i in range(min(len(headers), len(row)))})
|
|
if not rows:
|
|
continue
|
|
name = f"{base} — {ws.title}" if len(wb.worksheets) > 1 else (base or ws.title)
|
|
page = rows_to_collection(name, headers, rows)
|
|
page.source_path = f"{filename}#{ws.title}"
|
|
page.external_id = page.source_path
|
|
result.pages.append(page)
|
|
total_rows += len(rows)
|
|
result.stats["rows"] = total_rows
|
|
return result.finalize()
|
|
|
|
|
|
@register_importer
|
|
class JsonImporter(Importer):
|
|
source_id = "json"
|
|
label = "JSON (mapping générique)"
|
|
description = "Tableau d'objets JSON → collection (union des clés)."
|
|
extensions = (".json",)
|
|
order = 65
|
|
|
|
def _records(self, data: bytes) -> list[dict] | None:
|
|
try:
|
|
obj = json.loads(decode_text(data))
|
|
except Exception: # noqa: BLE001
|
|
return None
|
|
if isinstance(obj, list) and obj and all(isinstance(x, dict) for x in obj):
|
|
return obj
|
|
if isinstance(obj, dict):
|
|
for value in obj.values():
|
|
if isinstance(value, list) and value and all(isinstance(x, dict) for x in value):
|
|
return value
|
|
return None
|
|
|
|
def detect(self, filename: str, data: bytes) -> bool:
|
|
if not filename.lower().endswith(".json"):
|
|
return False
|
|
return self._records(data) is not None
|
|
|
|
def parse(self, filename: str, data: bytes) -> ImportResult:
|
|
result = ImportResult(source=self.source_id)
|
|
records = self._records(data)
|
|
if not records:
|
|
result.warn("Aucun tableau d'objets JSON détecté")
|
|
return result.finalize()
|
|
headers: list[str] = []
|
|
for rec in records:
|
|
for key in rec:
|
|
if key not in headers:
|
|
headers.append(key)
|
|
flat: list[dict[str, Any]] = []
|
|
for rec in records:
|
|
row = {}
|
|
for h in headers:
|
|
v = rec.get(h)
|
|
row[h] = json.dumps(v, ensure_ascii=False) if isinstance(v, (dict, list)) else v
|
|
flat.append(row)
|
|
name = filename.replace("\\", "/").rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
|
result.pages.append(rows_to_collection(name, headers, flat))
|
|
result.stats["rows"] = len(flat)
|
|
return result.finalize()
|