"""FlowDeck — Bookmark cards (v5.5.0): Open Graph metadata via httpx. Fetches a URL server-side, extracts OG/Twitter meta tags (title, description, image, site name, favicon) and returns a safe, compact payload used to render Notion-style bookmark cards. Robust to missing tags, non-HTML bodies and slow/unreachable hosts. """ from __future__ import annotations import html as htmlmod import logging import re from urllib.parse import urljoin, urlparse from app.services.http_client import shared_client logger = logging.getLogger(__name__) _META_TAG_RE = re.compile(r"]*?>", re.I) _ATTR_RE = re.compile(r"([A-Za-z_:][-A-Za-z0-9_:.]*)\s*=\s*[\"']([^\"']*)[\"']") _TITLE_RE = re.compile(r"]*>(.*?)", re.I | re.S) _FAVICON_RE = re.compile(r"]*?>", re.I) _ICON_REL = re.compile(r"\b(?:shortcut\s+)?icon\b", re.I) # Property/name keys we look for, in priority order, mapped to our payload keys. _OG_TITLE = ("og:title", "twitter:title", "title", "og:site_name") _OG_DESC = ("og:description", "twitter:description", "description") _OG_IMG = ("og:image", "twitter:image", "twitter:image:src", "image") _OG_SITE = ("og:site_name", "twitter:site", "application-name") def _attrs(tag: str) -> dict: return {k.lower(): v for k, v in _ATTR_RE.findall(tag)} def _extract_og(body: str) -> dict: """Parse all ```` tags into a ``{key: content}`` dict. Attributes may appear in any order (``content`` before or after ``property``/``name``), which the previous implementation mishandled. First value wins so the most specific tag (top of document) is kept. """ props: dict[str, str] = {} for tag in _META_TAG_RE.finditer(body[:400_000]): attrs = _attrs(tag.group(0)) key = (attrs.get("property") or attrs.get("name") or attrs.get("itemprop") or "").lower() content = attrs.get("content") if key and content is not None and key not in props: props[key] = content return props def _pick(props: dict, keys: tuple) -> str: for k in keys: v = props.get(k) if v: return v return "" def _title_of(props: dict, body: str) -> str: t = _pick(props, _OG_TITLE) if t: return t m = _TITLE_RE.search(body[:200_000]) return m.group(1).strip() if m else "" def _site_name(url: str) -> str: host = urlparse(url).netloc.replace("www.", "") return host.split(".")[0].capitalize() if host else "" def _favicon(body: str, base_url: str) -> str: for tag in _FAVICON_RE.finditer(body): attrs = _attrs(tag.group(0)) rel = attrs.get("rel", "") href = attrs.get("href", "") if href and _ICON_REL.search(rel): return urljoin(base_url, htmlmod.unescape(href)) return "" def parse_og(body: str, url: str) -> dict: """Pure HTML → bookmark payload (no network). ``url`` is the base URL.""" src = url.strip() if not src.startswith(("http://", "https://")): src = "https://" + src props = _extract_og(body) title = htmlmod.unescape(_title_of(props, body)) desc = htmlmod.unescape(_pick(props, _OG_DESC)) img = _pick(props, _OG_IMG) site = htmlmod.unescape(_pick(props, _OG_SITE)) or _site_name(src) def abs_url(u: str) -> str: return urljoin(src, htmlmod.unescape(u)) if u else "" return { "url": src, "title": title.strip()[:200] or urlparse(src).netloc or src, "description": desc.strip()[:400], "image": abs_url(img), "site_name": site.strip()[:100], "favicon": _favicon(body, src), } _MAX_REDIRECTS = 5 async def _get_checked(client, url: str, headers: dict): """GET avec re-vérification de l'hôte à CHAQUE saut de redirection (A12 SSRF). `follow_redirects=True` laisserait une URL publique rediriger vers 169.254.169.254 / localhost — la garde doit donc tourner à chaque hop. """ from app.services.importers.url_fetch import _is_public_host current = url for _ in range(_MAX_REDIRECTS + 1): parsed = urlparse(current) if parsed.scheme not in ("http", "https") or not parsed.hostname or not _is_public_host(parsed.hostname): raise ValueError(f"hôte non autorisé: {parsed.hostname!r}") r = await client.get(current, headers=headers, follow_redirects=False) if r.status_code in (301, 302, 303, 307, 308): loc = r.headers.get("location") if not loc: return r current = urljoin(current, loc) continue r.raise_for_status() return r raise ValueError("trop de redirections") async def fetch_og_metadata(url: str, timeout: float = 6.0, transport=None) -> dict: """Fetch ``url`` and return {url, title, description, image, site_name, favicon}. Empty strings are omitted. Never raises for network errors. ``transport`` is an optional ``httpx`` transport (used by tests to mock HTTP without hitting the network). """ src = url.strip() if not src.startswith(("http://", "https://")): src = "https://" + src base = {"url": src, "title": "", "description": "", "image": "", "site_name": "", "favicon": ""} try: headers = { "User-Agent": "FlowDeck/5.5 bookmark-fetcher (+https://flowdeck.dracodev.net)", "Accept": "text/html,application/xhtml+xml", } kwargs = {"timeout": timeout} if transport is not None: kwargs["transport"] = transport async with shared_client(**kwargs) as client: resp = await _get_checked(client, src, headers) except ValueError: # A12 : hôte privé/loopback ou trop de redirections → refus explicite. raise except Exception as exc: # noqa: BLE001 - network/parse failures are non-fatal logger.debug("og fetch failed for %s: %s", src, exc) base["title"] = urlparse(src).netloc or src base["site_name"] = _site_name(src) return base ctype = (resp.headers.get("content-type") or "").lower() if "text/html" not in ctype and "xhtml" not in ctype: base["title"] = urlparse(src).netloc or src base["site_name"] = _site_name(src) return base return parse_og(resp.text, src)