Initial commit: 25 scripts Python sauvegardés
Date: 2026-08-28 11:43:34 EDT
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
airtable:30f47a4b29827da14e655356c0edd8a7
|
||||
apple-notes:5e448abf984561fb33b197045ce41388
|
||||
apple-reminders:cda2963c73800643faf4a34ef813879a
|
||||
architecture-diagram:ca5e216b2014eef4f38f0a488eaf3545
|
||||
arxiv:06b6666b948852e77545c99ef72139db
|
||||
ascii-art:3aea656d9b8fb9d054ce37565e704a04
|
||||
ascii-video:2c8277458b2ef50421ce44debb9d81ad
|
||||
audiocraft-audio-generation:c207bdbf300ea5c42decc9cb6a596d1c
|
||||
baoyu-comic:8ff68387b01dea27ce049837ae9ecc47
|
||||
baoyu-infographic:53edf7d1b9398d62f4ccb0755e27913e
|
||||
blogwatcher:3f30bdd408c771501b94fab9289579c6
|
||||
claude-code:231f7e3cb0b2b91f64ce4b23fc2cef4d
|
||||
claude-design:c1b63b7651b66fd15d096e57728ec686
|
||||
codebase-inspection:29f67c87df868dd08e76c57b86c7a5c6
|
||||
codex:a116c5f880869f663b2974c2c9a79dbd
|
||||
comfyui:7efa41ae19823ed75a05ce96d9c44e07
|
||||
debugging-hermes-tui-commands:b093ceac961144fb3e72e57b12a5d354
|
||||
design-md:b40264457352831ab1d06f3ec671b532
|
||||
dogfood:ae6e92c2cd27c3da8a0587f089d19fe3
|
||||
dspy:64659f5eba8f127700fa931f58175f01
|
||||
evaluating-llms-harness:ac24cf5202db5b024b3079023797a0f6
|
||||
excalidraw:149a572d2069ee3de2951352725a8b19
|
||||
findmy:1d7dd3ae39cf25357a374c6bfb956442
|
||||
gif-search:d02d7fb065dfc143c079f15b8d8af1a7
|
||||
github-auth:c58654476268579b4cfa5953bb4160d0
|
||||
github-code-review:b771855b5c56b5e3a2546167a51667ba
|
||||
github-issues:429ec06cde7578e4ebba451d8106f008
|
||||
github-pr-workflow:48b6aceaae5333f6d3ed2d72f595331b
|
||||
github-repo-management:c5c05bc85dfcda2b7ac2f2006be9efe0
|
||||
godmode:27f853c53d2a2ef38118a583a9dfb6c4
|
||||
google-workspace:db46b2778030ed92d03a8aab727c036c
|
||||
heartmula:96a5927a5f221065260ddb2e0f1d77ec
|
||||
hermes-agent:166e68c17dec7adf1f91b20f904f251d
|
||||
hermes-agent-skill-authoring:4fefd83beeea0a8a32f5ae006f10f00d
|
||||
himalaya:3f05533dcd15be9d89cc86ab13c94a83
|
||||
huggingface-hub:da338c5152d72db030bb81d923d1c64d
|
||||
humanizer:6645b341862575f452e86139c5c71ce9
|
||||
ideation:8cfdaafa9abda023643427110c8c3073
|
||||
imessage:f545da0f5cc64dd9ee1ffd2b7733a11b
|
||||
jupyter-live-kernel:352c43dc28428592abbc8c91cb5ce295
|
||||
kanban-orchestrator:335a645935c88361ef2a103dba1fba01
|
||||
kanban-worker:381c14fb24d058e1750d1f97aa8d8011
|
||||
linear:58a67824f7df6e4753cfe0575be9e921
|
||||
llama-cpp:0991055ce47146735f0ed02d7658a254
|
||||
llm-wiki:ca957fbfecd9e573a1f99db3937a1969
|
||||
macos-computer-use:fb1787b32be343b87078d2ae2618dbcf
|
||||
manim-video:2ad3d68c3eb5d2675c05138100d3e48b
|
||||
maps:6d94a271a5341dc94789a9c99eb2b1a8
|
||||
minecraft-modpack-server:71e51465a1df193d7505ee47a91b6dbd
|
||||
nano-pdf:6c643bd0cfb0548ff0ddaf367d4da6d1
|
||||
native-mcp:3e758c66efed8f27c2153729d2050c87
|
||||
node-inspect-debugger:55501511963a3a6410fc767b5ed3e21c
|
||||
notion:930f6c7d327dd86553f45e2da9d9901e
|
||||
obliteratus:d3333944ef536dbe9238766f2f626eae
|
||||
obsidian:be02894afa45a908623ad6f9b7594199
|
||||
ocr-and-documents:af5fba9fa8ef003951ff5fe5a0a04adf
|
||||
opencode:d2a166c7f2c74f6e47d548ed1290c458
|
||||
openhue:ce1dd061d7f49d4752a4c0711ad2666c
|
||||
p5js:5f09fa1cb8494c93bc2f5bcbd34a2ead
|
||||
pixel-art:39562dfa3598566fc09280c054ef0ede
|
||||
plan:50c4df52eb6b0932649a444bc9984e0a
|
||||
pokemon-player:db79617ea4a61e0d6727d38402fb1f94
|
||||
polymarket:7644c886e028c229bc8c1f54114c3170
|
||||
popular-web-designs:b3fd685e8fbcf981755609ce98a4eea9
|
||||
powerpoint:00a6eb2ad4b7be22c1eabf6c19158836
|
||||
pretext:f17f2b6211eb81b96e1fc5d48ecc96a4
|
||||
python-debugpy:b87e0abf179c14ea51c7559dc99eb22c
|
||||
requesting-code-review:9944f54e8ba5c294b47ee93f3e8f239d
|
||||
research-paper-writing:a4a5a13107ff94894a4abdb39a5c7a8d
|
||||
segment-anything-model:7f1317da421fb8eada27aeacdbb21d30
|
||||
serving-llms-vllm:92d66ae1f1112924634fcdcff2f86bc7
|
||||
sketch:f8833126112824f6a916c69630cfd042
|
||||
songsee:644dc0f267b6661a3df6c76ce80d7f1f
|
||||
songwriting-and-ai-music:52be403c894c7bd7d6fe70f7eeaf9460
|
||||
spike:8b01bbb339d72d13a8a3214a018969c4
|
||||
spotify:a0ffe21977c14dc8da5dca039de83b0f
|
||||
subagent-driven-development:f434c120ce9cc40fadb9ff970b45c1e7
|
||||
systematic-debugging:55a23d83e936d3c7c04d4e5fa803558e
|
||||
teams-meeting-pipeline:810d4862830d1eb3daccf898f7c5ee85
|
||||
test-driven-development:31fb75eff421db47e527bce28c493df3
|
||||
touchdesigner-mcp:0664ded9138795d5518def4d16037650
|
||||
webhook-subscriptions:b1ffc6d6a3cf3f59ea8eb7dcd178c555
|
||||
weights-and-biases:8f0e1ee92fdf7b42a1dad448176d7c64
|
||||
writing-plans:ebe31eb58f7a09a7cd8fad4d422b9d3b
|
||||
xurl:e44ca3f6818dd7391a9b12ae79d76c16
|
||||
youtube-content:e43a762cd557db987d9ab466a7a60f76
|
||||
yuanbao:7844c287c57b42dccf51127f15e0751b
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Tests for extract_schema.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from extract_schema import (
|
||||
extract_schema,
|
||||
find_negative_prompt_node,
|
||||
find_positive_prompt_node,
|
||||
trace_to_node,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Connection tracing
|
||||
# =============================================================================
|
||||
|
||||
class TestConnectionTracing:
|
||||
def test_direct_link(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
|
||||
"2": {"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["1", 0]}},
|
||||
}
|
||||
assert trace_to_node(wf, ["1", 0]) == "1"
|
||||
|
||||
def test_through_reroute(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
|
||||
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
|
||||
"3": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
|
||||
}
|
||||
assert trace_to_node(wf, ["3", 0]) == "1"
|
||||
|
||||
def test_circular_safe(self):
|
||||
wf = {
|
||||
"1": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
|
||||
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
|
||||
}
|
||||
# Should hit max_hops without infinite loop
|
||||
result = trace_to_node(wf, ["1", 0], max_hops=5)
|
||||
assert result in {"1", "2"} # any node, just don't hang
|
||||
|
||||
|
||||
class TestPositiveNegativeDetection:
|
||||
def test_basic(self, sd15_workflow):
|
||||
# In sd15_workflow.json node 6 is positive, node 7 is negative
|
||||
assert find_positive_prompt_node(sd15_workflow) == "6"
|
||||
assert find_negative_prompt_node(sd15_workflow) == "7"
|
||||
|
||||
def test_swapped_order(self):
|
||||
wf = {
|
||||
"3": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["7", 0], "negative": ["6", 0],
|
||||
"model": ["4", 0], "latent_image": ["5", 0],
|
||||
"seed": 1, "steps": 20, "cfg": 7.5,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
|
||||
}},
|
||||
"4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "ugly", "clip": ["4", 1]}},
|
||||
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "beautiful", "clip": ["4", 1]}},
|
||||
}
|
||||
# Now 7 is the positive (despite higher node ID)
|
||||
assert find_positive_prompt_node(wf) == "7"
|
||||
assert find_negative_prompt_node(wf) == "6"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema extraction
|
||||
# =============================================================================
|
||||
|
||||
class TestExtractSchema:
|
||||
def test_basic_sd15(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
params = schema["parameters"]
|
||||
assert "prompt" in params
|
||||
assert "negative_prompt" in params
|
||||
assert "seed" in params
|
||||
assert "steps" in params
|
||||
assert "cfg" in params
|
||||
assert "width" in params
|
||||
assert "height" in params
|
||||
|
||||
def test_prompt_value_correct(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
# The positive prompt in the example is the landscape one
|
||||
assert "landscape" in schema["parameters"]["prompt"]["value"]
|
||||
assert "ugly" in schema["parameters"]["negative_prompt"]["value"]
|
||||
|
||||
def test_model_dependencies(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
deps = schema["model_dependencies"]
|
||||
ckpts = [d["value"] for d in deps if d["folder"] == "checkpoints"]
|
||||
assert "v1-5-pruned-emaonly.safetensors" in ckpts
|
||||
|
||||
def test_output_nodes(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
assert "9" in schema["output_nodes"]
|
||||
|
||||
def test_summary(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
s = schema["summary"]
|
||||
assert s["has_negative_prompt"] is True
|
||||
assert s["has_seed"] is True
|
||||
assert s["is_video_workflow"] is False
|
||||
assert s["parameter_count"] > 5
|
||||
|
||||
def test_flux_workflow(self, flux_workflow):
|
||||
schema = extract_schema(flux_workflow)
|
||||
# Flux uses RandomNoise for seed
|
||||
assert schema["summary"]["has_seed"] is True
|
||||
# Flux has only positive prompt (no negative encoder)
|
||||
assert schema["summary"]["has_negative_prompt"] is False
|
||||
|
||||
def test_video_detected(self, video_workflow):
|
||||
schema = extract_schema(video_workflow)
|
||||
assert schema["summary"]["is_video_workflow"] is True
|
||||
|
||||
|
||||
class TestEmbeddingDeps:
|
||||
def test_extract_from_prompt(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
|
||||
"5": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "a cat, embedding:goodvibes, embedding:art:1.2",
|
||||
"clip": ["1", 1]
|
||||
}},
|
||||
"7": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "ugly, embedding:badhands",
|
||||
"clip": ["1", 1]
|
||||
}},
|
||||
"3": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["6", 0], "negative": ["7", 0],
|
||||
"model": ["1", 0], "latent_image": ["5", 0],
|
||||
"seed": 1, "steps": 20, "cfg": 7.5,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
|
||||
}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
|
||||
}
|
||||
schema = extract_schema(wf)
|
||||
names = [d["embedding_name"] for d in schema["embedding_dependencies"]]
|
||||
assert sorted(names) == ["art", "badhands", "goodvibes"]
|
||||
|
||||
|
||||
class TestDuplicateDeduplication:
|
||||
def test_two_ksamplers_get_unique_names(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
|
||||
"5": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "a", "clip": ["1", 1]}},
|
||||
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "b", "clip": ["1", 1]}},
|
||||
"3": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["6", 0], "negative": ["7", 0],
|
||||
"model": ["1", 0], "latent_image": ["5", 0],
|
||||
"seed": 42, "steps": 20, "cfg": 7.5,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
|
||||
}},
|
||||
"4": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["6", 0], "negative": ["7", 0],
|
||||
"model": ["1", 0], "latent_image": ["5", 0],
|
||||
"seed": 99, "steps": 30, "cfg": 8.0,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 0.6,
|
||||
}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
|
||||
}
|
||||
schema = extract_schema(wf)
|
||||
params = schema["parameters"]
|
||||
# Both seeds present with disambiguated names
|
||||
seed_keys = [k for k in params if "seed" in k]
|
||||
# Symmetric: both renamed (no bare "seed")
|
||||
assert "seed" not in params
|
||||
assert "seed_3" in params and "seed_4" in params
|
||||
assert params["seed_3"]["value"] == 42
|
||||
assert params["seed_4"]["value"] == 99
|
||||
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recover a blocked / paywalled / WAF'd page from third-party copies.
|
||||
|
||||
Ladder (cheapest first):
|
||||
1. Wayback Machine "available" API -> dated snapshot (provenance: snapshot)
|
||||
2. archive.today domain rotation -> dated snapshot (provenance: snapshot)
|
||||
3. Jina Reader (JINA_API_KEY only) -> live re-render (provenance: live)
|
||||
|
||||
Every candidate body is validated before being declared a win: byte floors,
|
||||
redirect-stub detection (meta-refresh/JS pointing back at the original host),
|
||||
and interstitial-title rejection. Fake 200s are the norm in this space.
|
||||
|
||||
Stdlib only. Usage:
|
||||
python3 recover_page.py URL [--json] [--out FILE] [--timeout N]
|
||||
|
||||
Exit codes: 0 recovered, 1 nothing worked, 2 bad invocation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
||||
)
|
||||
|
||||
ARCHIVE_TODAY_HOSTS = ["archive.ph", "archive.md", "archive.li", "archive.is"]
|
||||
|
||||
# Titles that mean "this is not the page you asked for".
|
||||
INTERSTITIAL_TITLES = (
|
||||
"just a moment",
|
||||
"redirecting",
|
||||
"google search",
|
||||
"attention required",
|
||||
"access denied",
|
||||
"are you a robot",
|
||||
"one more step",
|
||||
)
|
||||
|
||||
# Below these floors a body is a stub or an error page, not content.
|
||||
MIN_BODY_BYTES = {"wayback": 3072, "archive_today": 3072, "jina": 512}
|
||||
|
||||
REDIRECT_STUB_RE = re.compile(
|
||||
r'http-equiv=["\']?refresh|window\.location|location\.replace', re.IGNORECASE
|
||||
)
|
||||
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def _fetch(
|
||||
url: str,
|
||||
timeout: int,
|
||||
headers: dict | None = None,
|
||||
retries_on_429: int = 2,
|
||||
) -> tuple[int, bytes]:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, **(headers or {})})
|
||||
for attempt in range(retries_on_429 + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.status, resp.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 429 and attempt < retries_on_429:
|
||||
time.sleep(5 * (attempt + 1))
|
||||
continue
|
||||
return exc.code, exc.read() if exc.fp else b""
|
||||
except (urllib.error.URLError, OSError, ValueError):
|
||||
return 0, b""
|
||||
return 0, b""
|
||||
|
||||
|
||||
def _fetch_follow(url: str, timeout: int) -> tuple[int, bytes, str]:
|
||||
"""Like _fetch but also returns the final URL after redirects."""
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.status, resp.read(), resp.geturl()
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read() if exc.fp else b"", exc.geturl() or url
|
||||
except (urllib.error.URLError, OSError, ValueError):
|
||||
return 0, b"", url
|
||||
|
||||
|
||||
def _page_title(body: bytes) -> str:
|
||||
m = TITLE_RE.search(body[:65536].decode("utf-8", "replace"))
|
||||
return re.sub(r"\s+", " ", m.group(1)).strip().lower() if m else ""
|
||||
|
||||
|
||||
def validate(body: bytes, route: str, target_url: str) -> str | None:
|
||||
"""Return a rejection reason, or None if the body looks like real content."""
|
||||
floor = MIN_BODY_BYTES.get(route, 3072)
|
||||
if len(body) < floor:
|
||||
return f"body_too_small:{len(body)}<{floor}"
|
||||
title = _page_title(body)
|
||||
for marker in INTERSTITIAL_TITLES:
|
||||
if marker in title:
|
||||
return f"interstitial_title:{marker!r}"
|
||||
# Redirect stub: small-ish page whose only job is bouncing back to the
|
||||
# original (blocked) host — the classic AMP-cache failure mode.
|
||||
if len(body) < 8192 and REDIRECT_STUB_RE.search(body.decode("utf-8", "replace")):
|
||||
target_host = urllib.parse.urlsplit(target_url).hostname or ""
|
||||
if target_host and target_host.encode() in body:
|
||||
return "redirect_stub_to_origin"
|
||||
return None
|
||||
|
||||
|
||||
def try_wayback(url: str, timeout: int) -> dict | None:
|
||||
snap_url = None
|
||||
snap_ts = None
|
||||
discovery = "https://archive.org/wayback/available?url=" + urllib.parse.quote(url, safe="")
|
||||
status, raw = _fetch(discovery, timeout)
|
||||
if status == 200:
|
||||
try:
|
||||
closest = json.loads(raw).get("archived_snapshots", {}).get("closest", {})
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
closest = {}
|
||||
if closest.get("available") and closest.get("url"):
|
||||
snap_url = closest["url"].replace(
|
||||
"http://web.archive.org", "https://web.archive.org"
|
||||
)
|
||||
snap_ts = closest.get("timestamp")
|
||||
if snap_url is None:
|
||||
# Discovery API is rate-limited far more aggressively than snapshot
|
||||
# serving. Fall back to the redirect form: /web/2/<url> bounces to
|
||||
# the newest snapshot if one exists (404 page otherwise).
|
||||
snap_url = "https://web.archive.org/web/2/" + url
|
||||
status, body, final_url = _fetch_follow(snap_url, timeout)
|
||||
if status != 200 or validate(body, "wayback", url):
|
||||
return None
|
||||
if snap_ts is None:
|
||||
m = re.search(r"/web/(\d{14})", final_url)
|
||||
snap_ts = m.group(1) if m else None
|
||||
return {
|
||||
"route": "wayback",
|
||||
"provenance": "snapshot",
|
||||
"snapshot_timestamp": snap_ts,
|
||||
"source_url": final_url,
|
||||
"body": body,
|
||||
}
|
||||
|
||||
|
||||
def try_archive_today(url: str, timeout: int) -> dict | None:
|
||||
for host in ARCHIVE_TODAY_HOSTS:
|
||||
fetch_url = f"https://{host}/newest/{url}"
|
||||
status, body = _fetch(fetch_url, timeout)
|
||||
if status != 200:
|
||||
continue
|
||||
if validate(body, "archive_today", url):
|
||||
continue # 429 bodies and interstitials land here
|
||||
return {
|
||||
"route": f"archive_today:{host}",
|
||||
"provenance": "snapshot",
|
||||
"snapshot_timestamp": None, # archive.today embeds the date in-page
|
||||
"source_url": fetch_url,
|
||||
"body": body,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def try_jina(url: str, timeout: int) -> dict | None:
|
||||
key = os.environ.get("JINA_API_KEY")
|
||||
if not key:
|
||||
return None
|
||||
status, body = _fetch(
|
||||
"https://r.jina.ai/" + url, timeout, headers={"Authorization": f"Bearer {key}"}
|
||||
)
|
||||
if status != 200 or validate(body, "jina", url):
|
||||
return None
|
||||
return {
|
||||
"route": "jina_reader",
|
||||
"provenance": "live",
|
||||
"snapshot_timestamp": None,
|
||||
"source_url": "https://r.jina.ai/" + url,
|
||||
"body": body,
|
||||
}
|
||||
|
||||
|
||||
ROUTES = (try_wayback, try_archive_today, try_jina)
|
||||
|
||||
|
||||
def recover(url: str, timeout: int = 25) -> dict | None:
|
||||
for route_fn in ROUTES:
|
||||
result = route_fn(url, timeout)
|
||||
if result:
|
||||
return result
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Recover a blocked / paywalled / WAF'd page from third-party copies."
|
||||
)
|
||||
ap.add_argument("url")
|
||||
ap.add_argument("--json", action="store_true", help="print metadata as JSON")
|
||||
ap.add_argument("--out", help="write recovered body to this file")
|
||||
ap.add_argument("--timeout", type=int, default=25)
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.url.startswith(("http://", "https://")):
|
||||
print("error: URL must start with http:// or https://", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
result = recover(args.url, args.timeout)
|
||||
if not result:
|
||||
msg = {"recovered": False, "url": args.url,
|
||||
"hint": "No archive copy found. Try the API-first pivot or the browser tool."}
|
||||
print(json.dumps(msg, indent=2) if args.json else msg["hint"], file=sys.stderr)
|
||||
return 1
|
||||
|
||||
body = result.pop("body")
|
||||
result.update({"recovered": True, "url": args.url, "body_bytes": len(body)})
|
||||
if args.out:
|
||||
with open(args.out, "wb") as fh:
|
||||
fh.write(body)
|
||||
result["saved_to"] = args.out
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
for k, v in result.items():
|
||||
print(f"{k}: {v}")
|
||||
if not args.out:
|
||||
print("\n--- body (first 2000 chars) ---")
|
||||
print(body[:2000].decode("utf-8", "replace"))
|
||||
if result["provenance"] == "snapshot":
|
||||
print(
|
||||
"\nNOTE: this is an ARCHIVED SNAPSHOT, not the live page. "
|
||||
"Cite it with its timestamp.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,87 @@
|
||||
airtable:30f47a4b29827da14e655356c0edd8a7
|
||||
apple-notes:5e448abf984561fb33b197045ce41388
|
||||
apple-reminders:cda2963c73800643faf4a34ef813879a
|
||||
architecture-diagram:ca5e216b2014eef4f38f0a488eaf3545
|
||||
arxiv:06b6666b948852e77545c99ef72139db
|
||||
ascii-art:3aea656d9b8fb9d054ce37565e704a04
|
||||
ascii-video:2c8277458b2ef50421ce44debb9d81ad
|
||||
audiocraft-audio-generation:c207bdbf300ea5c42decc9cb6a596d1c
|
||||
baoyu-comic:8ff68387b01dea27ce049837ae9ecc47
|
||||
baoyu-infographic:53edf7d1b9398d62f4ccb0755e27913e
|
||||
blogwatcher:3f30bdd408c771501b94fab9289579c6
|
||||
claude-code:231f7e3cb0b2b91f64ce4b23fc2cef4d
|
||||
claude-design:c1b63b7651b66fd15d096e57728ec686
|
||||
codebase-inspection:29f67c87df868dd08e76c57b86c7a5c6
|
||||
codex:a116c5f880869f663b2974c2c9a79dbd
|
||||
comfyui:7efa41ae19823ed75a05ce96d9c44e07
|
||||
debugging-hermes-tui-commands:b093ceac961144fb3e72e57b12a5d354
|
||||
design-md:b40264457352831ab1d06f3ec671b532
|
||||
dogfood:ae6e92c2cd27c3da8a0587f089d19fe3
|
||||
dspy:64659f5eba8f127700fa931f58175f01
|
||||
evaluating-llms-harness:ac24cf5202db5b024b3079023797a0f6
|
||||
excalidraw:149a572d2069ee3de2951352725a8b19
|
||||
findmy:1d7dd3ae39cf25357a374c6bfb956442
|
||||
gif-search:d02d7fb065dfc143c079f15b8d8af1a7
|
||||
github-auth:c58654476268579b4cfa5953bb4160d0
|
||||
github-code-review:b771855b5c56b5e3a2546167a51667ba
|
||||
github-issues:429ec06cde7578e4ebba451d8106f008
|
||||
github-pr-workflow:48b6aceaae5333f6d3ed2d72f595331b
|
||||
github-repo-management:c5c05bc85dfcda2b7ac2f2006be9efe0
|
||||
godmode:27f853c53d2a2ef38118a583a9dfb6c4
|
||||
google-workspace:db46b2778030ed92d03a8aab727c036c
|
||||
heartmula:96a5927a5f221065260ddb2e0f1d77ec
|
||||
hermes-agent:166e68c17dec7adf1f91b20f904f251d
|
||||
hermes-agent-skill-authoring:4fefd83beeea0a8a32f5ae006f10f00d
|
||||
himalaya:3f05533dcd15be9d89cc86ab13c94a83
|
||||
huggingface-hub:da338c5152d72db030bb81d923d1c64d
|
||||
humanizer:6645b341862575f452e86139c5c71ce9
|
||||
ideation:8cfdaafa9abda023643427110c8c3073
|
||||
imessage:f545da0f5cc64dd9ee1ffd2b7733a11b
|
||||
jupyter-live-kernel:352c43dc28428592abbc8c91cb5ce295
|
||||
kanban-orchestrator:335a645935c88361ef2a103dba1fba01
|
||||
kanban-worker:381c14fb24d058e1750d1f97aa8d8011
|
||||
linear:58a67824f7df6e4753cfe0575be9e921
|
||||
llama-cpp:0991055ce47146735f0ed02d7658a254
|
||||
llm-wiki:ca957fbfecd9e573a1f99db3937a1969
|
||||
macos-computer-use:fb1787b32be343b87078d2ae2618dbcf
|
||||
manim-video:2ad3d68c3eb5d2675c05138100d3e48b
|
||||
maps:6d94a271a5341dc94789a9c99eb2b1a8
|
||||
minecraft-modpack-server:71e51465a1df193d7505ee47a91b6dbd
|
||||
nano-pdf:6c643bd0cfb0548ff0ddaf367d4da6d1
|
||||
native-mcp:3e758c66efed8f27c2153729d2050c87
|
||||
node-inspect-debugger:55501511963a3a6410fc767b5ed3e21c
|
||||
notion:930f6c7d327dd86553f45e2da9d9901e
|
||||
obliteratus:d3333944ef536dbe9238766f2f626eae
|
||||
obsidian:be02894afa45a908623ad6f9b7594199
|
||||
ocr-and-documents:af5fba9fa8ef003951ff5fe5a0a04adf
|
||||
opencode:d2a166c7f2c74f6e47d548ed1290c458
|
||||
openhue:ce1dd061d7f49d4752a4c0711ad2666c
|
||||
p5js:5f09fa1cb8494c93bc2f5bcbd34a2ead
|
||||
pixel-art:39562dfa3598566fc09280c054ef0ede
|
||||
plan:50c4df52eb6b0932649a444bc9984e0a
|
||||
pokemon-player:db79617ea4a61e0d6727d38402fb1f94
|
||||
polymarket:7644c886e028c229bc8c1f54114c3170
|
||||
popular-web-designs:b3fd685e8fbcf981755609ce98a4eea9
|
||||
powerpoint:00a6eb2ad4b7be22c1eabf6c19158836
|
||||
pretext:f17f2b6211eb81b96e1fc5d48ecc96a4
|
||||
python-debugpy:b87e0abf179c14ea51c7559dc99eb22c
|
||||
requesting-code-review:9944f54e8ba5c294b47ee93f3e8f239d
|
||||
research-paper-writing:a4a5a13107ff94894a4abdb39a5c7a8d
|
||||
segment-anything-model:7f1317da421fb8eada27aeacdbb21d30
|
||||
serving-llms-vllm:92d66ae1f1112924634fcdcff2f86bc7
|
||||
sketch:f8833126112824f6a916c69630cfd042
|
||||
songsee:644dc0f267b6661a3df6c76ce80d7f1f
|
||||
songwriting-and-ai-music:52be403c894c7bd7d6fe70f7eeaf9460
|
||||
spike:8b01bbb339d72d13a8a3214a018969c4
|
||||
spotify:a0ffe21977c14dc8da5dca039de83b0f
|
||||
subagent-driven-development:f434c120ce9cc40fadb9ff970b45c1e7
|
||||
systematic-debugging:55a23d83e936d3c7c04d4e5fa803558e
|
||||
teams-meeting-pipeline:810d4862830d1eb3daccf898f7c5ee85
|
||||
test-driven-development:31fb75eff421db47e527bce28c493df3
|
||||
touchdesigner-mcp:0664ded9138795d5518def4d16037650
|
||||
webhook-subscriptions:b1ffc6d6a3cf3f59ea8eb7dcd178c555
|
||||
weights-and-biases:8f0e1ee92fdf7b42a1dad448176d7c64
|
||||
writing-plans:ebe31eb58f7a09a7cd8fad4d422b9d3b
|
||||
xurl:e44ca3f6818dd7391a9b12ae79d76c16
|
||||
youtube-content:e43a762cd557db987d9ab466a7a60f76
|
||||
yuanbao:7844c287c57b42dccf51127f15e0751b
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Tests for extract_schema.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from extract_schema import (
|
||||
extract_schema,
|
||||
find_negative_prompt_node,
|
||||
find_positive_prompt_node,
|
||||
trace_to_node,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Connection tracing
|
||||
# =============================================================================
|
||||
|
||||
class TestConnectionTracing:
|
||||
def test_direct_link(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
|
||||
"2": {"class_type": "KSampler",
|
||||
"inputs": {"positive": ["1", 0], "negative": ["1", 0]}},
|
||||
}
|
||||
assert trace_to_node(wf, ["1", 0]) == "1"
|
||||
|
||||
def test_through_reroute(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
|
||||
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
|
||||
"3": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
|
||||
}
|
||||
assert trace_to_node(wf, ["3", 0]) == "1"
|
||||
|
||||
def test_circular_safe(self):
|
||||
wf = {
|
||||
"1": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
|
||||
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
|
||||
}
|
||||
# Should hit max_hops without infinite loop
|
||||
result = trace_to_node(wf, ["1", 0], max_hops=5)
|
||||
assert result in {"1", "2"} # any node, just don't hang
|
||||
|
||||
|
||||
class TestPositiveNegativeDetection:
|
||||
def test_basic(self, sd15_workflow):
|
||||
# In sd15_workflow.json node 6 is positive, node 7 is negative
|
||||
assert find_positive_prompt_node(sd15_workflow) == "6"
|
||||
assert find_negative_prompt_node(sd15_workflow) == "7"
|
||||
|
||||
def test_swapped_order(self):
|
||||
wf = {
|
||||
"3": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["7", 0], "negative": ["6", 0],
|
||||
"model": ["4", 0], "latent_image": ["5", 0],
|
||||
"seed": 1, "steps": 20, "cfg": 7.5,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
|
||||
}},
|
||||
"4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
|
||||
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "ugly", "clip": ["4", 1]}},
|
||||
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "beautiful", "clip": ["4", 1]}},
|
||||
}
|
||||
# Now 7 is the positive (despite higher node ID)
|
||||
assert find_positive_prompt_node(wf) == "7"
|
||||
assert find_negative_prompt_node(wf) == "6"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema extraction
|
||||
# =============================================================================
|
||||
|
||||
class TestExtractSchema:
|
||||
def test_basic_sd15(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
params = schema["parameters"]
|
||||
assert "prompt" in params
|
||||
assert "negative_prompt" in params
|
||||
assert "seed" in params
|
||||
assert "steps" in params
|
||||
assert "cfg" in params
|
||||
assert "width" in params
|
||||
assert "height" in params
|
||||
|
||||
def test_prompt_value_correct(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
# The positive prompt in the example is the landscape one
|
||||
assert "landscape" in schema["parameters"]["prompt"]["value"]
|
||||
assert "ugly" in schema["parameters"]["negative_prompt"]["value"]
|
||||
|
||||
def test_model_dependencies(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
deps = schema["model_dependencies"]
|
||||
ckpts = [d["value"] for d in deps if d["folder"] == "checkpoints"]
|
||||
assert "v1-5-pruned-emaonly.safetensors" in ckpts
|
||||
|
||||
def test_output_nodes(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
assert "9" in schema["output_nodes"]
|
||||
|
||||
def test_summary(self, sd15_workflow):
|
||||
schema = extract_schema(sd15_workflow)
|
||||
s = schema["summary"]
|
||||
assert s["has_negative_prompt"] is True
|
||||
assert s["has_seed"] is True
|
||||
assert s["is_video_workflow"] is False
|
||||
assert s["parameter_count"] > 5
|
||||
|
||||
def test_flux_workflow(self, flux_workflow):
|
||||
schema = extract_schema(flux_workflow)
|
||||
# Flux uses RandomNoise for seed
|
||||
assert schema["summary"]["has_seed"] is True
|
||||
# Flux has only positive prompt (no negative encoder)
|
||||
assert schema["summary"]["has_negative_prompt"] is False
|
||||
|
||||
def test_video_detected(self, video_workflow):
|
||||
schema = extract_schema(video_workflow)
|
||||
assert schema["summary"]["is_video_workflow"] is True
|
||||
|
||||
|
||||
class TestEmbeddingDeps:
|
||||
def test_extract_from_prompt(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
|
||||
"5": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "a cat, embedding:goodvibes, embedding:art:1.2",
|
||||
"clip": ["1", 1]
|
||||
}},
|
||||
"7": {"class_type": "CLIPTextEncode",
|
||||
"inputs": {
|
||||
"text": "ugly, embedding:badhands",
|
||||
"clip": ["1", 1]
|
||||
}},
|
||||
"3": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["6", 0], "negative": ["7", 0],
|
||||
"model": ["1", 0], "latent_image": ["5", 0],
|
||||
"seed": 1, "steps": 20, "cfg": 7.5,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
|
||||
}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
|
||||
}
|
||||
schema = extract_schema(wf)
|
||||
names = [d["embedding_name"] for d in schema["embedding_dependencies"]]
|
||||
assert sorted(names) == ["art", "badhands", "goodvibes"]
|
||||
|
||||
|
||||
class TestDuplicateDeduplication:
|
||||
def test_two_ksamplers_get_unique_names(self):
|
||||
wf = {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
|
||||
"5": {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
|
||||
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "a", "clip": ["1", 1]}},
|
||||
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "b", "clip": ["1", 1]}},
|
||||
"3": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["6", 0], "negative": ["7", 0],
|
||||
"model": ["1", 0], "latent_image": ["5", 0],
|
||||
"seed": 42, "steps": 20, "cfg": 7.5,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
|
||||
}},
|
||||
"4": {"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"positive": ["6", 0], "negative": ["7", 0],
|
||||
"model": ["1", 0], "latent_image": ["5", 0],
|
||||
"seed": 99, "steps": 30, "cfg": 8.0,
|
||||
"sampler_name": "euler", "scheduler": "normal", "denoise": 0.6,
|
||||
}},
|
||||
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
|
||||
}
|
||||
schema = extract_schema(wf)
|
||||
params = schema["parameters"]
|
||||
# Both seeds present with disambiguated names
|
||||
seed_keys = [k for k in params if "seed" in k]
|
||||
# Symmetric: both renamed (no bare "seed")
|
||||
assert "seed" not in params
|
||||
assert "seed_3" in params and "seed_4" in params
|
||||
assert params["seed_3"]["value"] == 42
|
||||
assert params["seed_4"]["value"] == 99
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Resolve HERMES_HOME for standalone skill scripts.
|
||||
|
||||
Skill scripts may run outside the Hermes process (system Python, nix env,
|
||||
CI) where ``hermes_constants`` is not importable. This module provides the
|
||||
same ``get_hermes_home()`` contract without requiring it on ``sys.path``.
|
||||
|
||||
When ``hermes_constants`` IS available it is used directly so profile
|
||||
resolution and any future enhancements are picked up automatically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from hermes_constants import get_hermes_home as get_hermes_home
|
||||
except (ModuleNotFoundError, ImportError):
|
||||
|
||||
def get_hermes_home() -> Path:
|
||||
"""Return the Hermes home directory (default: ``~/.hermes``)."""
|
||||
val = os.environ.get("HERMES_HOME", "").strip()
|
||||
return Path(val) if val else Path.home() / ".hermes"
|
||||
@@ -0,0 +1,678 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Citation ledger for grounded answers and documents.
|
||||
|
||||
Owns the ``url -> [n]`` mapping used by the ``grounded-citations`` skill.
|
||||
Ids are assigned at retrieval time and never change, so a draft's ``[3]``
|
||||
always resolves to the same page. The model only ever emits integers the
|
||||
ledger handed it, which is what makes the citations verifiable.
|
||||
|
||||
Subcommands
|
||||
-----------
|
||||
reset start a clean ledger
|
||||
add URL [URL ...] register source(s), print their ids
|
||||
ingest FILE|- register every url found in JSON tool output
|
||||
quote ID --text T --from FILE|- attach verbatim supporting evidence to a source
|
||||
list show the ledger
|
||||
render render a Sources block
|
||||
verify DRAFT check a draft's citations against the ledger
|
||||
|
||||
Fact-checking is evidence-backed citation: ``quote`` only accepts text that
|
||||
literally appears in the fetched page text you point it at, ``verify
|
||||
--evidence`` requires every cited source to carry at least one such quote, and
|
||||
``render --style evidence`` prints the quotes under each source so the reader
|
||||
can check the chain themselves. Claims from model knowledge are declared with
|
||||
an ``[unverified]`` marker rather than silently blended in.
|
||||
|
||||
Ledger path resolution (first wins):
|
||||
--ledger PATH
|
||||
$HERMES_CITATION_LEDGER
|
||||
$HERMES_HOME/cache/citations/ledger.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from _hermes_home import get_hermes_home # noqa: E402
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
# A citation marker in prose: [12]. Markdown links ([text](url)) and
|
||||
# reference-style labels are excluded by requiring digits only and no
|
||||
# following "(" or ":".
|
||||
_CITE_RE = re.compile(r"\[(\d{1,4})\](?![(:])")
|
||||
_SOURCES_HEADER_RE = re.compile(r"^\s*(?:#{1,6}\s*)?(?:\*\*)?sources:?(?:\*\*)?\s*$", re.IGNORECASE)
|
||||
_SOURCE_LINE_RE = re.compile(r"^\s*\[(\d{1,4})\]\s*[-–:]?\s*(\S+)")
|
||||
_URL_IN_TEXT_RE = re.compile(r"https?://[^\s\"'<>)\]}]+")
|
||||
_FENCE_RE = re.compile(r"^\s*(?:```|~~~)")
|
||||
# Explicit declaration that a claim comes from model knowledge, not a source.
|
||||
_UNVERIFIED_RE = re.compile(r"\[unverified\]", re.IGNORECASE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ledger I/O
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_ledger_path(explicit: str | None = None) -> Path:
|
||||
if explicit:
|
||||
return Path(explicit).expanduser()
|
||||
env = os.environ.get("HERMES_CITATION_LEDGER", "").strip()
|
||||
if env:
|
||||
return Path(env).expanduser()
|
||||
return get_hermes_home() / "cache" / "citations" / "ledger.json"
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
"""Canonicalize a URL for ledger identity.
|
||||
|
||||
Strips the fragment and a trailing slash so ``/page``, ``/page/`` and
|
||||
``/page#section`` are one source. Query strings are significant and are
|
||||
kept — they usually select different content.
|
||||
"""
|
||||
u = (url or "").strip()
|
||||
if "#" in u:
|
||||
u = u.split("#", 1)[0]
|
||||
stripped = u.rstrip("/")
|
||||
return stripped or u
|
||||
|
||||
|
||||
def load_ledger(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"version": SCHEMA_VERSION, "sources": []}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
raise SystemExit(f"error: ledger at {path} is unreadable ({exc}); run `reset` to start over")
|
||||
if not isinstance(data, dict) or not isinstance(data.get("sources"), list):
|
||||
raise SystemExit(f"error: ledger at {path} has an unexpected shape; run `reset`")
|
||||
return data
|
||||
|
||||
|
||||
def save_ledger(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
class _LedgerLock:
|
||||
"""Best-effort cross-process lock (O_EXCL lockfile, stdlib only).
|
||||
|
||||
Parallel subagents can share one ledger via --ledger; without a lock two
|
||||
concurrent ``add`` calls can assign the same id. Falls through after the
|
||||
timeout rather than blocking a task forever — a stale lock must never
|
||||
wedge the ledger.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path, timeout: float = 5.0) -> None:
|
||||
self.lock_path = path.with_suffix(path.suffix + ".lock")
|
||||
self.timeout = timeout
|
||||
self.fd: int | None = None
|
||||
|
||||
def __enter__(self) -> "_LedgerLock":
|
||||
self.lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
deadline = time.monotonic() + self.timeout
|
||||
while True:
|
||||
try:
|
||||
self.fd = os.open(str(self.lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
return self
|
||||
except FileExistsError:
|
||||
if time.monotonic() >= deadline:
|
||||
# Assume a stale lock from a crashed run.
|
||||
try:
|
||||
self.lock_path.unlink()
|
||||
except OSError:
|
||||
return self
|
||||
continue
|
||||
time.sleep(0.05)
|
||||
|
||||
def __exit__(self, *_exc: object) -> None:
|
||||
if self.fd is not None:
|
||||
try:
|
||||
os.close(self.fd)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
self.lock_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core operations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _by_url(sources: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {s["url"]: s for s in sources}
|
||||
|
||||
|
||||
def add_sources(
|
||||
path: Path,
|
||||
urls: Iterable[str],
|
||||
title: str | None = None,
|
||||
accessed: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Register urls, returning their ledger entries (existing or new)."""
|
||||
urls = [u for u in (str(u).strip() for u in urls) if u]
|
||||
if not urls:
|
||||
return []
|
||||
with _LedgerLock(path):
|
||||
data = load_ledger(path)
|
||||
sources = data["sources"]
|
||||
index = _by_url(sources)
|
||||
out: list[dict[str, Any]] = []
|
||||
changed = False
|
||||
for raw in urls:
|
||||
key = normalize_url(raw)
|
||||
existing = index.get(key)
|
||||
if existing is not None:
|
||||
if title and not existing.get("title"):
|
||||
existing["title"] = title
|
||||
changed = True
|
||||
out.append(existing)
|
||||
continue
|
||||
entry = {
|
||||
"id": len(sources) + 1,
|
||||
"url": key,
|
||||
"title": (title or "").strip(),
|
||||
"accessed": accessed or time.strftime("%Y-%m-%d"),
|
||||
}
|
||||
sources.append(entry)
|
||||
index[key] = entry
|
||||
out.append(entry)
|
||||
changed = True
|
||||
if changed:
|
||||
save_ledger(path, data)
|
||||
return out
|
||||
|
||||
|
||||
def urls_from_json(payload: Any) -> list[tuple[str, str]]:
|
||||
"""Walk arbitrary JSON tool output collecting (url, title) pairs.
|
||||
|
||||
Handles web_search (``data.web[]``), web_extract (``results[]``) and any
|
||||
other nesting, in document order, deduped.
|
||||
"""
|
||||
found: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
url = node.get("url") or node.get("link") or node.get("source_url")
|
||||
if isinstance(url, str) and url.startswith(("http://", "https://")):
|
||||
key = normalize_url(url)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
raw_title = node.get("title") or node.get("name") or ""
|
||||
found.append((url, raw_title if isinstance(raw_title, str) else ""))
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return found
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Evidence quotes (fact-checking)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _normalize_ws(text: str) -> str:
|
||||
"""Collapse all whitespace runs to single spaces for verbatim matching."""
|
||||
return " ".join((text or "").split())
|
||||
|
||||
|
||||
# Markdown artifacts that retrieval tools inject into otherwise-identical prose.
|
||||
# ``web_extract`` returns markdown, so the most citation-worthy sentences are
|
||||
# exactly the ones carrying inline links and emphasis around terms:
|
||||
# "including _[ERAP1](https://…/erap1/)_, _[IL1A](…)_, have also been…"
|
||||
# reads identically to the page a human sees. Matching has to see through that
|
||||
# markup, or the skill pushes the agent toward weaker evidence fragments.
|
||||
_MD_LINK_RE = re.compile(r"\[([^\]]*)\]\((?:[^()\s]|\([^()]*\))*\)")
|
||||
_MD_NOISE_RE = re.compile(r"[*_`~]|\\(?=[^\w\s])")
|
||||
|
||||
|
||||
def _match_key(text: str) -> str:
|
||||
"""Canonicalize text for verbatim comparison.
|
||||
|
||||
Whitespace-, case-, and markdown-insensitive: inline links collapse to
|
||||
their label, emphasis/code markers and backslash escapes are dropped. The
|
||||
stored quote keeps whatever the caller passed, so the rendered evidence
|
||||
block shows clean prose rather than extractor artifacts.
|
||||
"""
|
||||
collapsed = _MD_LINK_RE.sub(r"\1", text or "")
|
||||
return _normalize_ws(_MD_NOISE_RE.sub("", collapsed)).casefold()
|
||||
|
||||
|
||||
def quote_in_evidence(quote: str, evidence: str) -> bool:
|
||||
"""True when ``quote`` appears verbatim in the fetched ``evidence`` text,
|
||||
ignoring whitespace, case, and markdown markup on either side."""
|
||||
q = _match_key(quote)
|
||||
return bool(q) and q in _match_key(evidence)
|
||||
|
||||
|
||||
def attach_quote(path: Path, source_id: int, quote: str, evidence: str) -> dict[str, Any]:
|
||||
"""Attach a verbatim quote to a ledger entry after checking it against
|
||||
the evidence text. Raises SystemExit on unknown id or non-verbatim text —
|
||||
a quote the page does not contain is exactly the fabrication this guards
|
||||
against."""
|
||||
quote = (quote or "").strip()
|
||||
if len(_normalize_ws(quote).split()) < 3:
|
||||
raise SystemExit("error: quote too short — use at least 3 words of verbatim text")
|
||||
if not quote_in_evidence(quote, evidence):
|
||||
raise SystemExit(
|
||||
"error: quote not found verbatim in the evidence text — "
|
||||
"copy the exact wording from the fetched page, do not paraphrase"
|
||||
)
|
||||
with _LedgerLock(path):
|
||||
data = load_ledger(path)
|
||||
entry = next((s for s in data["sources"] if s["id"] == source_id), None)
|
||||
if entry is None:
|
||||
raise SystemExit(f"error: no source [{source_id}] in the ledger")
|
||||
quotes = entry.setdefault("quotes", [])
|
||||
norm = _match_key(quote)
|
||||
if not any(_match_key(q.get("text", "")) == norm for q in quotes):
|
||||
quotes.append({"text": quote, "added": time.strftime("%Y-%m-%d")})
|
||||
save_ledger(path, data)
|
||||
return entry
|
||||
|
||||
|
||||
def render_sources(
|
||||
sources: list[dict[str, Any]],
|
||||
style: str = "markdown",
|
||||
only: set[int] | None = None,
|
||||
) -> str:
|
||||
picked = [s for s in sources if only is None or s["id"] in only]
|
||||
picked.sort(key=lambda s: s["id"])
|
||||
if not picked:
|
||||
return ""
|
||||
lines: list[str] = []
|
||||
if style == "bibtex":
|
||||
for s in picked:
|
||||
key = f"source{s['id']}"
|
||||
title = s.get("title") or s["url"]
|
||||
lines.append(
|
||||
"@misc{%s,\n title = {%s},\n howpublished = {\\url{%s}},\n note = {Accessed %s}\n}"
|
||||
% (key, title, s["url"], s.get("accessed", ""))
|
||||
)
|
||||
return "\n".join(lines)
|
||||
if style == "footnotes":
|
||||
for s in picked:
|
||||
title = s.get("title")
|
||||
suffix = f" — {title}" if title else ""
|
||||
lines.append(f"[^{s['id']}]: {s['url']}{suffix}")
|
||||
return "\n".join(lines)
|
||||
header = "Sources:" if style == "plain" else "## Sources"
|
||||
lines.append(header)
|
||||
if style != "plain":
|
||||
lines.append("")
|
||||
for s in picked:
|
||||
title = s.get("title")
|
||||
suffix = f" — {title}" if title else ""
|
||||
lines.append(f"[{s['id']}] {s['url']}{suffix}")
|
||||
if style == "evidence":
|
||||
for q in s.get("quotes", []):
|
||||
lines.append(f' > "{q.get("text", "")}"')
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _split_draft(text: str) -> tuple[str, dict[int, str]]:
|
||||
"""Split a draft into (prose, sources_block_map).
|
||||
|
||||
The sources block is everything after the last ``Sources`` header; its
|
||||
``[n] url`` lines are parsed out so they aren't counted as prose citations.
|
||||
Fenced code blocks are dropped from prose.
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
header_idx = -1
|
||||
for i, line in enumerate(lines):
|
||||
if _SOURCES_HEADER_RE.match(line):
|
||||
header_idx = i
|
||||
listed: dict[int, str] = {}
|
||||
if header_idx >= 0:
|
||||
for line in lines[header_idx + 1:]:
|
||||
m = _SOURCE_LINE_RE.match(line)
|
||||
if m:
|
||||
url_match = _URL_IN_TEXT_RE.search(line)
|
||||
listed[int(m.group(1))] = url_match.group(0) if url_match else m.group(2)
|
||||
body_lines = lines[:header_idx]
|
||||
else:
|
||||
body_lines = lines
|
||||
prose: list[str] = []
|
||||
in_fence = False
|
||||
for line in body_lines:
|
||||
if _FENCE_RE.match(line):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if not in_fence:
|
||||
prose.append(line)
|
||||
return "\n".join(prose), listed
|
||||
|
||||
|
||||
def _strip_sources_block(text: str) -> str:
|
||||
"""Return the draft with its trailing Sources block removed.
|
||||
|
||||
Everything from the last Sources header onward goes; a draft with no such
|
||||
header is returned unchanged. This is what makes ``render --replace-in``
|
||||
idempotent instead of stacking duplicate blocks.
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
header_idx = -1
|
||||
for i, line in enumerate(lines):
|
||||
if _SOURCES_HEADER_RE.match(line):
|
||||
header_idx = i
|
||||
if header_idx < 0:
|
||||
return text
|
||||
return "\n".join(lines[:header_idx])
|
||||
|
||||
|
||||
def _sentences(prose: str) -> list[str]:
|
||||
"""Rough sentence split over prose lines, skipping headings and tables."""
|
||||
out: list[str] = []
|
||||
for line in prose.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#") or stripped.startswith("|"):
|
||||
continue
|
||||
if stripped.startswith(">"):
|
||||
stripped = stripped.lstrip("> ").strip()
|
||||
for part in re.split(r"(?<=[.!?])\s+", stripped):
|
||||
part = part.strip()
|
||||
if len(part.split()) >= 4:
|
||||
out.append(part)
|
||||
return out
|
||||
|
||||
|
||||
def verify_draft(
|
||||
draft_path: Path,
|
||||
sources: list[dict[str, Any]],
|
||||
strict: bool = False,
|
||||
min_coverage: float | None = None,
|
||||
require_evidence: bool = False,
|
||||
) -> tuple[int, list[str], list[str]]:
|
||||
"""Return (exit_code, errors, warnings)."""
|
||||
text = draft_path.read_text(encoding="utf-8")
|
||||
prose, listed = _split_draft(text)
|
||||
by_id = {s["id"]: s for s in sources}
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
cited = [int(m) for m in _CITE_RE.findall(prose)]
|
||||
cited_set = set(cited)
|
||||
|
||||
unknown = sorted(i for i in cited_set if i not in by_id)
|
||||
if unknown:
|
||||
errors.append(
|
||||
"citations not in the ledger (hallucinated or renumbered): "
|
||||
+ ", ".join(f"[{i}]" for i in unknown)
|
||||
)
|
||||
|
||||
if cited_set and not listed:
|
||||
errors.append("draft cites sources but has no `Sources:` block — run `render --cited-in`")
|
||||
|
||||
missing_from_block = sorted(cited_set - set(listed)) if listed else []
|
||||
if missing_from_block:
|
||||
errors.append(
|
||||
"cited but absent from the Sources block: "
|
||||
+ ", ".join(f"[{i}]" for i in missing_from_block)
|
||||
)
|
||||
|
||||
for sid, url in sorted(listed.items()):
|
||||
entry = by_id.get(sid)
|
||||
if entry is None:
|
||||
errors.append(f"Sources block lists [{sid}], which is not in the ledger")
|
||||
continue
|
||||
if normalize_url(url) != entry["url"]:
|
||||
errors.append(
|
||||
f"Sources block URL for [{sid}] does not match the ledger "
|
||||
f"(block: {url} / ledger: {entry['url']}) — re-run `render`"
|
||||
)
|
||||
|
||||
extra_in_block = sorted(set(listed) - cited_set)
|
||||
if extra_in_block:
|
||||
warnings.append(
|
||||
"listed in Sources but never cited inline: "
|
||||
+ ", ".join(f"[{i}]" for i in extra_in_block)
|
||||
)
|
||||
|
||||
registered_uncited = sorted(set(by_id) - cited_set)
|
||||
if registered_uncited:
|
||||
warnings.append(
|
||||
"registered in the ledger but not cited in this draft: "
|
||||
+ ", ".join(f"[{i}]" for i in registered_uncited)
|
||||
)
|
||||
|
||||
sentences = _sentences(prose)
|
||||
cited_sentences = [s for s in sentences if _CITE_RE.search(s)]
|
||||
unverified_sentences = [s for s in sentences if _UNVERIFIED_RE.search(s)]
|
||||
covered = [s for s in sentences if _CITE_RE.search(s) or _UNVERIFIED_RE.search(s)]
|
||||
coverage = (len(covered) / len(sentences)) if sentences else 0.0
|
||||
if min_coverage is not None and sentences and coverage < min_coverage:
|
||||
errors.append(
|
||||
f"citation coverage {coverage:.0%} is below the required {min_coverage:.0%} "
|
||||
f"({len(covered)}/{len(sentences)} sentences cited or marked [unverified])"
|
||||
)
|
||||
|
||||
if require_evidence:
|
||||
unevidenced = sorted(
|
||||
i for i in cited_set if i in by_id and not by_id[i].get("quotes")
|
||||
)
|
||||
if unevidenced:
|
||||
errors.append(
|
||||
"cited sources carry no verbatim evidence quote (run `quote` with the "
|
||||
"fetched page text): " + ", ".join(f"[{i}]" for i in unevidenced)
|
||||
)
|
||||
|
||||
over_cited = [s for s in sentences if len(_CITE_RE.findall(s)) > 3]
|
||||
if over_cited:
|
||||
warnings.append(f"{len(over_cited)} sentence(s) carry more than 3 citations")
|
||||
|
||||
code = 1 if errors else (1 if (strict and warnings) else 0)
|
||||
quoted = sum(1 for s in sources if s.get("quotes"))
|
||||
stats = (
|
||||
f"{len(sentences)} prose sentence(s), {len(covered)} with declared provenance "
|
||||
f"({coverage:.0%}) — {len(cited_sentences)} cited, "
|
||||
f"{len(unverified_sentences)} marked [unverified] (a sentence may be both); "
|
||||
f"{len(cited_set)} distinct source(s) cited, "
|
||||
f"{len(by_id)} in ledger ({quoted} with evidence quotes)"
|
||||
)
|
||||
warnings.insert(0, f"stats: {stats}")
|
||||
return code, errors, warnings
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_only(spec: str | None) -> set[int] | None:
|
||||
if not spec:
|
||||
return None
|
||||
out: set[int] = set()
|
||||
for chunk in spec.replace(" ", "").split(","):
|
||||
if not chunk:
|
||||
continue
|
||||
if "-" in chunk:
|
||||
lo, _, hi = chunk.partition("-")
|
||||
out.update(range(int(lo), int(hi) + 1))
|
||||
else:
|
||||
out.add(int(chunk))
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="sources.py", description="Citation ledger for grounded answers and documents."
|
||||
)
|
||||
parser.add_argument("--ledger", help="ledger file path (overrides env / default)")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("reset", help="start a clean ledger")
|
||||
|
||||
p_add = sub.add_parser("add", help="register source url(s), print their ids")
|
||||
p_add.add_argument("urls", nargs="+")
|
||||
p_add.add_argument("--title", help="title for the source (single-url calls)")
|
||||
p_add.add_argument("--accessed", help="access date (default: today)")
|
||||
p_add.add_argument("--json", action="store_true", help="emit JSON instead of ids")
|
||||
|
||||
p_ing = sub.add_parser("ingest", help="register every url in JSON tool output")
|
||||
p_ing.add_argument("file", help="JSON file, or - for stdin")
|
||||
|
||||
p_q = sub.add_parser("quote", help="attach verbatim supporting evidence to a source")
|
||||
p_q.add_argument("id", type=int, help="ledger id of the source the quote supports")
|
||||
p_q.add_argument("--text", required=True, help="the exact quote, copied from the page")
|
||||
p_q.add_argument(
|
||||
"--from",
|
||||
dest="evidence",
|
||||
required=True,
|
||||
help="file with the fetched page text (or - for stdin) the quote must appear in",
|
||||
)
|
||||
|
||||
p_list = sub.add_parser("list", help="show the ledger")
|
||||
p_list.add_argument("--json", action="store_true")
|
||||
|
||||
p_render = sub.add_parser("render", help="render a Sources block")
|
||||
p_render.add_argument(
|
||||
"--style", default="markdown", choices=["markdown", "plain", "footnotes", "bibtex", "evidence"]
|
||||
)
|
||||
p_render.add_argument("--only", help="ids to include, e.g. 1,3,5-7")
|
||||
p_render.add_argument("--cited-in", help="include only ids cited in this draft file")
|
||||
p_render.add_argument(
|
||||
"--replace-in",
|
||||
help="rewrite this draft's Sources block in place (implies --cited-in on it)",
|
||||
)
|
||||
|
||||
p_ver = sub.add_parser("verify", help="check a draft's citations against the ledger")
|
||||
p_ver.add_argument("draft")
|
||||
p_ver.add_argument("--strict", action="store_true", help="treat warnings as failures")
|
||||
p_ver.add_argument("--min-coverage", type=float, help="required cited-sentence share, e.g. 0.5")
|
||||
p_ver.add_argument(
|
||||
"--evidence",
|
||||
action="store_true",
|
||||
help="require every cited source to carry at least one verbatim quote",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
path = resolve_ledger_path(args.ledger)
|
||||
|
||||
if args.cmd == "reset":
|
||||
with _LedgerLock(path):
|
||||
save_ledger(path, {"version": SCHEMA_VERSION, "sources": []})
|
||||
print(f"ledger reset: {path}")
|
||||
return 0
|
||||
|
||||
if args.cmd == "add":
|
||||
title = args.title if len(args.urls) == 1 else None
|
||||
entries = add_sources(path, args.urls, title=title, accessed=args.accessed)
|
||||
if args.json:
|
||||
print(json.dumps(entries, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
for e in entries:
|
||||
print(f"[{e['id']}] {e['url']}")
|
||||
return 0
|
||||
|
||||
if args.cmd == "ingest":
|
||||
raw = sys.stdin.read() if args.file == "-" else Path(args.file).read_text(encoding="utf-8")
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"error: input is not valid JSON ({exc})", file=sys.stderr)
|
||||
return 2
|
||||
pairs = urls_from_json(payload)
|
||||
if not pairs:
|
||||
print("no urls found in input", file=sys.stderr)
|
||||
return 1
|
||||
for url, title in pairs:
|
||||
entry = add_sources(path, [url], title=title or None)[0]
|
||||
print(f"[{entry['id']}] {entry['url']}")
|
||||
return 0
|
||||
|
||||
if args.cmd == "quote":
|
||||
raw = (
|
||||
sys.stdin.read()
|
||||
if args.evidence == "-"
|
||||
else Path(args.evidence).read_text(encoding="utf-8")
|
||||
)
|
||||
entry = attach_quote(path, args.id, args.text, raw)
|
||||
print(f"[{entry['id']}] evidence attached ({len(entry.get('quotes', []))} quote(s))")
|
||||
return 0
|
||||
|
||||
data = load_ledger(path)
|
||||
sources = sorted(data["sources"], key=lambda s: s["id"])
|
||||
|
||||
if args.cmd == "list":
|
||||
if args.json:
|
||||
print(json.dumps(sources, indent=2, ensure_ascii=False))
|
||||
elif not sources:
|
||||
print(f"ledger is empty: {path}")
|
||||
else:
|
||||
for s in sources:
|
||||
title = f" {s['title']}" if s.get("title") else ""
|
||||
nq = len(s.get("quotes", []))
|
||||
mark = f" ({nq} quote{'s' if nq != 1 else ''})" if nq else ""
|
||||
print(f"[{s['id']}] {s['url']}{title}{mark}")
|
||||
return 0
|
||||
|
||||
if args.cmd == "render":
|
||||
only = _parse_only(args.only)
|
||||
draft_for_ids = args.replace_in or args.cited_in
|
||||
if draft_for_ids:
|
||||
draft = Path(draft_for_ids).read_text(encoding="utf-8")
|
||||
prose, _ = _split_draft(draft)
|
||||
cited = {int(m) for m in _CITE_RE.findall(prose)}
|
||||
only = cited if only is None else (only & cited)
|
||||
block = render_sources(sources, style=args.style, only=only)
|
||||
if not block:
|
||||
print("no sources to render", file=sys.stderr)
|
||||
return 1
|
||||
if args.replace_in:
|
||||
target = Path(args.replace_in)
|
||||
body = _strip_sources_block(target.read_text(encoding="utf-8"))
|
||||
target.write_text(body.rstrip("\n") + "\n\n" + block + "\n", encoding="utf-8")
|
||||
print(f"Sources block rewritten in {target}")
|
||||
return 0
|
||||
print(block)
|
||||
return 0
|
||||
|
||||
if args.cmd == "verify":
|
||||
draft_path = Path(args.draft)
|
||||
if not draft_path.is_file():
|
||||
print(f"error: no such draft: {draft_path}", file=sys.stderr)
|
||||
return 2
|
||||
code, errors, warnings = verify_draft(
|
||||
draft_path,
|
||||
sources,
|
||||
strict=args.strict,
|
||||
min_coverage=args.min_coverage,
|
||||
require_evidence=args.evidence,
|
||||
)
|
||||
for w in warnings:
|
||||
prefix = "info" if w.startswith("stats: ") else "warn"
|
||||
print(f"{prefix}: {w}")
|
||||
for e in errors:
|
||||
print(f"FAIL: {e}", file=sys.stderr)
|
||||
print("citations OK" if code == 0 else "verification failed")
|
||||
return code
|
||||
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wrapper script for last30days engine — handles config and invocation."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
ENGINE_DIR = Path.home() / "workspace" / "last30days-skill"
|
||||
ENGINE_SCRIPT = ENGINE_DIR / "skills" / "last30days" / "scripts" / "last30days.py"
|
||||
|
||||
def main():
|
||||
if not ENGINE_SCRIPT.exists():
|
||||
print(f"❌ Engine not found at {ENGINE_SCRIPT}", file=sys.stderr)
|
||||
print("Clone it: git clone https://github.com/mvanhorn/last30days-skill.git ~/workspace/last30days-skill", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Forward all args to the engine
|
||||
cmd = ["python3", str(ENGINE_SCRIPT)] + sys.argv[1:]
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(ENGINE_DIR),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
# Print stderr first (diagnostics, warnings)
|
||||
if result.stderr:
|
||||
print(result.stderr, file=sys.stderr, end="")
|
||||
|
||||
# Print stdout (the actual results)
|
||||
if result.stdout:
|
||||
print(result.stdout)
|
||||
|
||||
sys.exit(result.returncode)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wrapper for Obscura headless browser — simplifies common operations."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
OBSCURA_BIN = Path.home() / ".local" / "bin" / "obscura"
|
||||
|
||||
def fetch(url: str, dump: str = "markdown", timeout: int = 15, wait_until: str = None, output: str = None) -> str:
|
||||
"""Fetch a single page and return markdown/text/html."""
|
||||
cmd = [str(OBSCURA_BIN), "fetch", url, "--dump", dump, "--timeout", str(timeout)]
|
||||
if wait_until:
|
||||
cmd.extend(["--wait-until", wait_until])
|
||||
if output:
|
||||
cmd.extend(["--output", output])
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 10)
|
||||
if result.returncode != 0:
|
||||
return f"ERROR: {result.stderr}"
|
||||
return result.stdout
|
||||
|
||||
def eval_js(url: str, expression: str, timeout: int = 15) -> str:
|
||||
"""Fetch a page and evaluate JavaScript."""
|
||||
cmd = [str(OBSCURA_BIN), "fetch", url, "--eval", expression, "--timeout", str(timeout)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 10)
|
||||
if result.returncode != 0:
|
||||
return f"ERROR: {result.stderr}"
|
||||
# Filter out the "Fetching..." and "Page loaded..." lines
|
||||
lines = result.stdout.strip().split("\n")
|
||||
return "\n".join(line for line in lines if not line.startswith(("Fetching", "Page loaded")))
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: obscura_wrapper.py fetch <url> [--dump markdown|text|html] [--timeout N] [--wait-until networkidle0]")
|
||||
print(" obscura_wrapper.py eval <url> <js_expression>")
|
||||
sys.exit(1)
|
||||
|
||||
action = sys.argv[1]
|
||||
|
||||
if action == "fetch":
|
||||
url = sys.argv[2]
|
||||
dump = "markdown"
|
||||
timeout = 15
|
||||
wait_until = None
|
||||
|
||||
# Parse optional flags
|
||||
args = sys.argv[3:]
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--dump" and i + 1 < len(args):
|
||||
dump = args[i + 1]
|
||||
i += 2
|
||||
elif args[i] == "--timeout" and i + 1 < len(args):
|
||||
timeout = int(args[i + 1])
|
||||
i += 2
|
||||
elif args[i] == "--wait-until" and i + 1 < len(args):
|
||||
wait_until = args[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
print(fetch(url, dump=dump, timeout=timeout, wait_until=wait_until))
|
||||
|
||||
elif action == "eval":
|
||||
url = sys.argv[2]
|
||||
expression = " ".join(sys.argv[3:])
|
||||
print(eval_js(url, expression))
|
||||
|
||||
else:
|
||||
print(f"Unknown action: {action}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Polymarket CLI helper — query prediction market data.
|
||||
|
||||
Usage:
|
||||
python3 polymarket.py search "bitcoin"
|
||||
python3 polymarket.py trending [--limit 10]
|
||||
python3 polymarket.py market <slug>
|
||||
python3 polymarket.py event <slug>
|
||||
python3 polymarket.py price <token_id>
|
||||
python3 polymarket.py book <token_id>
|
||||
python3 polymarket.py history <condition_id> [--interval all] [--fidelity 50]
|
||||
python3 polymarket.py trades [--limit 10] [--market CONDITION_ID]
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
|
||||
GAMMA = "https://gamma-api.polymarket.com"
|
||||
CLOB = "https://clob.polymarket.com"
|
||||
DATA = "https://data-api.polymarket.com"
|
||||
|
||||
|
||||
def _get(url: str) -> dict | list:
|
||||
"""GET request, return parsed JSON."""
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "hermes-agent/1.0"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f"HTTP {e.code}: {e.reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except urllib.error.URLError as e:
|
||||
print(f"Connection error: {e.reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _parse_json_field(val):
|
||||
"""Parse double-encoded JSON fields (outcomePrices, outcomes, clobTokenIds)."""
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
return json.loads(val)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return val
|
||||
return val
|
||||
|
||||
|
||||
def _fmt_pct(price_str: str) -> str:
|
||||
"""Format price string as percentage."""
|
||||
try:
|
||||
return f"{float(price_str) * 100:.1f}%"
|
||||
except (ValueError, TypeError):
|
||||
return price_str
|
||||
|
||||
|
||||
def _fmt_volume(vol) -> str:
|
||||
"""Format volume as human-readable."""
|
||||
try:
|
||||
v = float(vol)
|
||||
if v >= 1_000_000:
|
||||
return f"${v / 1_000_000:.1f}M"
|
||||
if v >= 1_000:
|
||||
return f"${v / 1_000:.1f}K"
|
||||
return f"${v:.0f}"
|
||||
except (ValueError, TypeError):
|
||||
return str(vol)
|
||||
|
||||
|
||||
def _print_market(m: dict, indent: str = ""):
|
||||
"""Print a market summary."""
|
||||
question = m.get("question", "?")
|
||||
prices = _parse_json_field(m.get("outcomePrices", "[]"))
|
||||
outcomes = _parse_json_field(m.get("outcomes", "[]"))
|
||||
vol = _fmt_volume(m.get("volume", 0))
|
||||
closed = m.get("closed", False)
|
||||
status = " [CLOSED]" if closed else ""
|
||||
|
||||
if isinstance(prices, list) and len(prices) >= 2:
|
||||
outcome_labels = outcomes if isinstance(outcomes, list) else ["Yes", "No"]
|
||||
price_str = " / ".join(
|
||||
f"{outcome_labels[i]}: {_fmt_pct(prices[i])}"
|
||||
for i in range(min(len(prices), len(outcome_labels)))
|
||||
)
|
||||
print(f"{indent}{question}{status}")
|
||||
print(f"{indent} {price_str} | Volume: {vol}")
|
||||
else:
|
||||
print(f"{indent}{question}{status} | Volume: {vol}")
|
||||
|
||||
slug = m.get("slug", "")
|
||||
if slug:
|
||||
print(f"{indent} slug: {slug}")
|
||||
|
||||
|
||||
def cmd_search(query: str):
|
||||
"""Search for markets."""
|
||||
q = urllib.parse.quote(query)
|
||||
data = _get(f"{GAMMA}/public-search?q={q}")
|
||||
events = data.get("events", [])
|
||||
total = data.get("pagination", {}).get("totalResults", len(events))
|
||||
print(f"Found {total} results for \"{query}\":\n")
|
||||
for evt in events[:10]:
|
||||
print(f"=== {evt['title']} ===")
|
||||
print(f" Volume: {_fmt_volume(evt.get('volume', 0))} | slug: {evt.get('slug', '')}")
|
||||
markets = evt.get("markets", [])
|
||||
for m in markets[:5]:
|
||||
_print_market(m, indent=" ")
|
||||
if len(markets) > 5:
|
||||
print(f" ... and {len(markets) - 5} more markets")
|
||||
print()
|
||||
|
||||
|
||||
def cmd_trending(limit: int = 10):
|
||||
"""Show trending events by volume."""
|
||||
events = _get(f"{GAMMA}/events?limit={limit}&active=true&closed=false&order=volume&ascending=false")
|
||||
print(f"Top {len(events)} trending events:\n")
|
||||
for i, evt in enumerate(events, 1):
|
||||
print(f"{i}. {evt['title']}")
|
||||
print(f" Volume: {_fmt_volume(evt.get('volume', 0))} | Markets: {len(evt.get('markets', []))}")
|
||||
print(f" slug: {evt.get('slug', '')}")
|
||||
markets = evt.get("markets", [])
|
||||
for m in markets[:3]:
|
||||
_print_market(m, indent=" ")
|
||||
if len(markets) > 3:
|
||||
print(f" ... and {len(markets) - 3} more markets")
|
||||
print()
|
||||
|
||||
|
||||
def cmd_market(slug: str):
|
||||
"""Get market details by slug."""
|
||||
markets = _get(f"{GAMMA}/markets?slug={urllib.parse.quote(slug)}")
|
||||
if not markets:
|
||||
print(f"No market found with slug: {slug}")
|
||||
return
|
||||
m = markets[0]
|
||||
print(f"Market: {m.get('question', '?')}")
|
||||
print(f"Status: {'CLOSED' if m.get('closed') else 'ACTIVE'}")
|
||||
_print_market(m)
|
||||
print(f"\n conditionId: {m.get('conditionId', 'N/A')}")
|
||||
tokens = _parse_json_field(m.get("clobTokenIds", "[]"))
|
||||
if isinstance(tokens, list):
|
||||
outcomes = _parse_json_field(m.get("outcomes", "[]"))
|
||||
for i, t in enumerate(tokens):
|
||||
label = outcomes[i] if isinstance(outcomes, list) and i < len(outcomes) else f"Outcome {i}"
|
||||
print(f" token ({label}): {t}")
|
||||
desc = m.get("description", "")
|
||||
if desc:
|
||||
print(f"\n Description: {desc[:500]}")
|
||||
|
||||
|
||||
def cmd_event(slug: str):
|
||||
"""Get event details by slug."""
|
||||
events = _get(f"{GAMMA}/events?slug={urllib.parse.quote(slug)}")
|
||||
if not events:
|
||||
print(f"No event found with slug: {slug}")
|
||||
return
|
||||
evt = events[0]
|
||||
print(f"Event: {evt['title']}")
|
||||
print(f"Volume: {_fmt_volume(evt.get('volume', 0))}")
|
||||
print(f"Status: {'CLOSED' if evt.get('closed') else 'ACTIVE'}")
|
||||
print(f"Markets: {len(evt.get('markets', []))}\n")
|
||||
for m in evt.get("markets", []):
|
||||
_print_market(m, indent=" ")
|
||||
print()
|
||||
|
||||
|
||||
def cmd_price(token_id: str):
|
||||
"""Get current price for a token."""
|
||||
buy = _get(f"{CLOB}/price?token_id={token_id}&side=buy")
|
||||
mid = _get(f"{CLOB}/midpoint?token_id={token_id}")
|
||||
spread = _get(f"{CLOB}/spread?token_id={token_id}")
|
||||
print(f"Token: {token_id[:30]}...")
|
||||
print(f" Buy price: {_fmt_pct(buy.get('price', '?'))}")
|
||||
print(f" Midpoint: {_fmt_pct(mid.get('mid', '?'))}")
|
||||
print(f" Spread: {spread.get('spread', '?')}")
|
||||
|
||||
|
||||
def cmd_book(token_id: str):
|
||||
"""Get orderbook for a token."""
|
||||
book = _get(f"{CLOB}/book?token_id={token_id}")
|
||||
bids = book.get("bids", [])
|
||||
asks = book.get("asks", [])
|
||||
last = book.get("last_trade_price", "?")
|
||||
print(f"Orderbook for {token_id[:30]}...")
|
||||
print(f"Last trade: {_fmt_pct(last)} | Tick size: {book.get('tick_size', '?')}")
|
||||
print(f"\n Top bids ({len(bids)} total):")
|
||||
# Show bids sorted by price descending (best bids first)
|
||||
sorted_bids = sorted(bids, key=lambda x: float(x.get("price", 0)), reverse=True)
|
||||
for b in sorted_bids[:10]:
|
||||
print(f" {_fmt_pct(b['price']):>7} | Size: {float(b['size']):>10.2f}")
|
||||
print(f"\n Top asks ({len(asks)} total):")
|
||||
sorted_asks = sorted(asks, key=lambda x: float(x.get("price", 0)))
|
||||
for a in sorted_asks[:10]:
|
||||
print(f" {_fmt_pct(a['price']):>7} | Size: {float(a['size']):>10.2f}")
|
||||
|
||||
|
||||
def cmd_history(condition_id: str, interval: str = "all", fidelity: int = 50):
|
||||
"""Get price history for a market."""
|
||||
data = _get(f"{CLOB}/prices-history?market={condition_id}&interval={interval}&fidelity={fidelity}")
|
||||
history = data.get("history", [])
|
||||
if not history:
|
||||
print("No price history available for this market.")
|
||||
return
|
||||
print(f"Price history ({len(history)} points, interval={interval}):\n")
|
||||
from datetime import datetime, timezone
|
||||
for pt in history:
|
||||
ts = datetime.fromtimestamp(pt["t"], tz=timezone.utc).strftime("%Y-%m-%d %H:%M")
|
||||
price = _fmt_pct(pt["p"])
|
||||
bar = "█" * int(float(pt["p"]) * 40)
|
||||
print(f" {ts} {price:>7} {bar}")
|
||||
|
||||
|
||||
def cmd_trades(limit: int = 10, market: str = None):
|
||||
"""Get recent trades."""
|
||||
url = f"{DATA}/trades?limit={limit}"
|
||||
if market:
|
||||
url += f"&market={market}"
|
||||
trades = _get(url)
|
||||
if not isinstance(trades, list):
|
||||
print(f"Unexpected response: {trades}")
|
||||
return
|
||||
print(f"Recent trades ({len(trades)}):\n")
|
||||
for t in trades:
|
||||
side = t.get("side", "?")
|
||||
price = _fmt_pct(t.get("price", "?"))
|
||||
size = t.get("size", "?")
|
||||
outcome = t.get("outcome", "?")
|
||||
title = t.get("title", "?")[:50]
|
||||
ts = t.get("timestamp", "")
|
||||
print(f" {side:4} {price:>7} x{float(size):>8.2f} [{outcome}] {title}")
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
if not args or args[0] in {"-h", "--help", "help"}:
|
||||
print(__doc__)
|
||||
return
|
||||
|
||||
cmd = args[0]
|
||||
|
||||
if cmd == "search" and len(args) >= 2:
|
||||
cmd_search(" ".join(args[1:]))
|
||||
elif cmd == "trending":
|
||||
limit = 10
|
||||
if "--limit" in args:
|
||||
idx = args.index("--limit")
|
||||
limit = int(args[idx + 1]) if idx + 1 < len(args) else 10
|
||||
cmd_trending(limit)
|
||||
elif cmd == "market" and len(args) >= 2:
|
||||
cmd_market(args[1])
|
||||
elif cmd == "event" and len(args) >= 2:
|
||||
cmd_event(args[1])
|
||||
elif cmd == "price" and len(args) >= 2:
|
||||
cmd_price(args[1])
|
||||
elif cmd == "book" and len(args) >= 2:
|
||||
cmd_book(args[1])
|
||||
elif cmd == "history" and len(args) >= 2:
|
||||
interval = "all"
|
||||
fidelity = 50
|
||||
if "--interval" in args:
|
||||
idx = args.index("--interval")
|
||||
interval = args[idx + 1] if idx + 1 < len(args) else "all"
|
||||
if "--fidelity" in args:
|
||||
idx = args.index("--fidelity")
|
||||
fidelity = int(args[idx + 1]) if idx + 1 < len(args) else 50
|
||||
cmd_history(args[1], interval, fidelity)
|
||||
elif cmd == "trades":
|
||||
limit = 10
|
||||
market = None
|
||||
if "--limit" in args:
|
||||
idx = args.index("--limit")
|
||||
limit = int(args[idx + 1]) if idx + 1 < len(args) else 10
|
||||
if "--market" in args:
|
||||
idx = args.index("--market")
|
||||
market = args[idx + 1] if idx + 1 < len(args) else None
|
||||
cmd_trades(limit, market)
|
||||
else:
|
||||
print(f"Unknown command: {cmd}")
|
||||
print(__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user