feat(import): v5.6.0 unified data import (phases 0-5)

- unified importer framework (app/services/importers/): normalized model,
  registry, common pipeline (hierarchy, attachments, collections, dedup),
  async jobs, dry-run preview, column->type mapping
- Phase 1: Obsidian, Notion, Logseq/Roam, HTML (Apple Notes/Bear/Ulysses/
  OneNote), Google Keep, generic Markdown
- Phase 2: typed CSV/TSV, Excel (openpyxl), generic JSON
- Phase 3: Word .docx (python-docx), PDF (pypdf), HTML folders
- Phase 4: Raindrop, Pocket, Readwise, Shaarli, Netscape bookmarks, .ics,
  OPML, Standard Notes, Gitea/GitHub issues (+labels/milestones)
- Phase 5: incremental re-sync (skip/update/duplicate), partial-error resume,
  forge repo files, URL web clipper (SSRF guard), batch multi-file + UI queue,
  Notion relation resolution, exportable JSON reports
- /import wizard, API /api/import/*, migration 9 (import_items, import_jobs)
- fix: property values stored by property id (correct DB view rendering)
- deps: openpyxl, beautifulsoup4, PyYAML, python-docx, pypdf
- 43 import tests; full suite 491 green; ruff clean
- bump version 5.11.5
This commit is contained in:
2026-09-13 10:43:20 -04:00
parent d986959927
commit 3b00cbc371
32 changed files with 5275 additions and 58 deletions
+54
View File
@@ -209,6 +209,60 @@ class GitHubAdapter(ForgeAdapter):
resp.raise_for_status()
return resp.json()
# ── issues / labels / milestones ──
async def list_issues(self, owner: str, repo: str, state: str = "all") -> list[dict]:
"""List issues (pull requests are filtered out)."""
issues: list[dict] = []
for page in range(1, 6):
async with self._client() as client:
resp = await client.get(
f"{self._base}/repos/{owner}/{repo}/issues",
headers=self._headers,
params={"state": state, "per_page": 100, "page": page},
)
resp.raise_for_status()
batch = resp.json()
if not batch:
break
issues.extend(i for i in batch if "pull_request" not in i)
if len(batch) < 100:
break
return issues
async def list_labels(self, owner: str, repo: str) -> list[dict]:
async with self._client() as client:
resp = await client.get(
f"{self._base}/repos/{owner}/{repo}/labels",
headers=self._headers,
params={"per_page": 100},
)
resp.raise_for_status()
return resp.json()
async def list_milestones(self, owner: str, repo: str, state: str = "all") -> list[dict]:
async with self._client() as client:
resp = await client.get(
f"{self._base}/repos/{owner}/{repo}/milestones",
headers=self._headers,
params={"state": state, "per_page": 100},
)
resp.raise_for_status()
return resp.json()
async def list_repo_files(self, owner: str, repo: str, path: str = "") -> list[dict]:
"""Flatten the repo tree into file entries (``path``, ``size``)."""
tree = await self.get_repo_tree(owner, repo)
prefix = path.strip("/")
files = [
{"path": item["path"], "size": item.get("size", 0)}
for item in tree
if item.get("type") == "blob" and item.get("path")
]
if prefix:
files = [f for f in files if f["path"].startswith(prefix + "/") or f["path"] == prefix]
return files
# ── token validation ──
async def validate_token(self) -> bool: