Initial commit: 25 scripts Python sauvegardés

Date: 2026-08-28 11:43:34 EDT
This commit is contained in:
2026-08-28 11:43:34 -04:00
parent 0ba9701e0f
commit 994ea86d47
204 changed files with 70104 additions and 2 deletions
@@ -0,0 +1,87 @@
airtable:30f47a4b29827da14e655356c0edd8a7
apple-notes:5e448abf984561fb33b197045ce41388
apple-reminders:cda2963c73800643faf4a34ef813879a
architecture-diagram:ca5e216b2014eef4f38f0a488eaf3545
arxiv:06b6666b948852e77545c99ef72139db
ascii-art:3aea656d9b8fb9d054ce37565e704a04
ascii-video:2c8277458b2ef50421ce44debb9d81ad
audiocraft-audio-generation:c207bdbf300ea5c42decc9cb6a596d1c
baoyu-comic:8ff68387b01dea27ce049837ae9ecc47
baoyu-infographic:53edf7d1b9398d62f4ccb0755e27913e
blogwatcher:3f30bdd408c771501b94fab9289579c6
claude-code:231f7e3cb0b2b91f64ce4b23fc2cef4d
claude-design:c1b63b7651b66fd15d096e57728ec686
codebase-inspection:29f67c87df868dd08e76c57b86c7a5c6
codex:a116c5f880869f663b2974c2c9a79dbd
comfyui:7efa41ae19823ed75a05ce96d9c44e07
debugging-hermes-tui-commands:b093ceac961144fb3e72e57b12a5d354
design-md:b40264457352831ab1d06f3ec671b532
dogfood:ae6e92c2cd27c3da8a0587f089d19fe3
dspy:64659f5eba8f127700fa931f58175f01
evaluating-llms-harness:ac24cf5202db5b024b3079023797a0f6
excalidraw:149a572d2069ee3de2951352725a8b19
findmy:1d7dd3ae39cf25357a374c6bfb956442
gif-search:d02d7fb065dfc143c079f15b8d8af1a7
github-auth:c58654476268579b4cfa5953bb4160d0
github-code-review:b771855b5c56b5e3a2546167a51667ba
github-issues:429ec06cde7578e4ebba451d8106f008
github-pr-workflow:48b6aceaae5333f6d3ed2d72f595331b
github-repo-management:c5c05bc85dfcda2b7ac2f2006be9efe0
godmode:27f853c53d2a2ef38118a583a9dfb6c4
google-workspace:db46b2778030ed92d03a8aab727c036c
heartmula:96a5927a5f221065260ddb2e0f1d77ec
hermes-agent:166e68c17dec7adf1f91b20f904f251d
hermes-agent-skill-authoring:4fefd83beeea0a8a32f5ae006f10f00d
himalaya:3f05533dcd15be9d89cc86ab13c94a83
huggingface-hub:da338c5152d72db030bb81d923d1c64d
humanizer:6645b341862575f452e86139c5c71ce9
ideation:8cfdaafa9abda023643427110c8c3073
imessage:f545da0f5cc64dd9ee1ffd2b7733a11b
jupyter-live-kernel:352c43dc28428592abbc8c91cb5ce295
kanban-orchestrator:335a645935c88361ef2a103dba1fba01
kanban-worker:381c14fb24d058e1750d1f97aa8d8011
linear:58a67824f7df6e4753cfe0575be9e921
llama-cpp:0991055ce47146735f0ed02d7658a254
llm-wiki:ca957fbfecd9e573a1f99db3937a1969
macos-computer-use:fb1787b32be343b87078d2ae2618dbcf
manim-video:2ad3d68c3eb5d2675c05138100d3e48b
maps:6d94a271a5341dc94789a9c99eb2b1a8
minecraft-modpack-server:71e51465a1df193d7505ee47a91b6dbd
nano-pdf:6c643bd0cfb0548ff0ddaf367d4da6d1
native-mcp:3e758c66efed8f27c2153729d2050c87
node-inspect-debugger:55501511963a3a6410fc767b5ed3e21c
notion:930f6c7d327dd86553f45e2da9d9901e
obliteratus:d3333944ef536dbe9238766f2f626eae
obsidian:be02894afa45a908623ad6f9b7594199
ocr-and-documents:af5fba9fa8ef003951ff5fe5a0a04adf
opencode:d2a166c7f2c74f6e47d548ed1290c458
openhue:ce1dd061d7f49d4752a4c0711ad2666c
p5js:5f09fa1cb8494c93bc2f5bcbd34a2ead
pixel-art:39562dfa3598566fc09280c054ef0ede
plan:50c4df52eb6b0932649a444bc9984e0a
pokemon-player:db79617ea4a61e0d6727d38402fb1f94
polymarket:7644c886e028c229bc8c1f54114c3170
popular-web-designs:b3fd685e8fbcf981755609ce98a4eea9
powerpoint:00a6eb2ad4b7be22c1eabf6c19158836
pretext:f17f2b6211eb81b96e1fc5d48ecc96a4
python-debugpy:b87e0abf179c14ea51c7559dc99eb22c
requesting-code-review:9944f54e8ba5c294b47ee93f3e8f239d
research-paper-writing:a4a5a13107ff94894a4abdb39a5c7a8d
segment-anything-model:7f1317da421fb8eada27aeacdbb21d30
serving-llms-vllm:92d66ae1f1112924634fcdcff2f86bc7
sketch:f8833126112824f6a916c69630cfd042
songsee:644dc0f267b6661a3df6c76ce80d7f1f
songwriting-and-ai-music:52be403c894c7bd7d6fe70f7eeaf9460
spike:8b01bbb339d72d13a8a3214a018969c4
spotify:a0ffe21977c14dc8da5dca039de83b0f
subagent-driven-development:f434c120ce9cc40fadb9ff970b45c1e7
systematic-debugging:55a23d83e936d3c7c04d4e5fa803558e
teams-meeting-pipeline:810d4862830d1eb3daccf898f7c5ee85
test-driven-development:31fb75eff421db47e527bce28c493df3
touchdesigner-mcp:0664ded9138795d5518def4d16037650
webhook-subscriptions:b1ffc6d6a3cf3f59ea8eb7dcd178c555
weights-and-biases:8f0e1ee92fdf7b42a1dad448176d7c64
writing-plans:ebe31eb58f7a09a7cd8fad4d422b9d3b
xurl:e44ca3f6818dd7391a9b12ae79d76c16
youtube-content:e43a762cd557db987d9ab466a7a60f76
yuanbao:7844c287c57b42dccf51127f15e0751b
+185
View File
@@ -0,0 +1,185 @@
"""Tests for extract_schema.py."""
from __future__ import annotations
import pytest
from extract_schema import (
extract_schema,
find_negative_prompt_node,
find_positive_prompt_node,
trace_to_node,
)
# =============================================================================
# Connection tracing
# =============================================================================
class TestConnectionTracing:
def test_direct_link(self):
wf = {
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
"2": {"class_type": "KSampler",
"inputs": {"positive": ["1", 0], "negative": ["1", 0]}},
}
assert trace_to_node(wf, ["1", 0]) == "1"
def test_through_reroute(self):
wf = {
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
"3": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
}
assert trace_to_node(wf, ["3", 0]) == "1"
def test_circular_safe(self):
wf = {
"1": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
}
# Should hit max_hops without infinite loop
result = trace_to_node(wf, ["1", 0], max_hops=5)
assert result in {"1", "2"} # any node, just don't hang
class TestPositiveNegativeDetection:
def test_basic(self, sd15_workflow):
# In sd15_workflow.json node 6 is positive, node 7 is negative
assert find_positive_prompt_node(sd15_workflow) == "6"
assert find_negative_prompt_node(sd15_workflow) == "7"
def test_swapped_order(self):
wf = {
"3": {"class_type": "KSampler",
"inputs": {
"positive": ["7", 0], "negative": ["6", 0],
"model": ["4", 0], "latent_image": ["5", 0],
"seed": 1, "steps": 20, "cfg": 7.5,
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
}},
"4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "ugly", "clip": ["4", 1]}},
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "beautiful", "clip": ["4", 1]}},
}
# Now 7 is the positive (despite higher node ID)
assert find_positive_prompt_node(wf) == "7"
assert find_negative_prompt_node(wf) == "6"
# =============================================================================
# Schema extraction
# =============================================================================
class TestExtractSchema:
def test_basic_sd15(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
params = schema["parameters"]
assert "prompt" in params
assert "negative_prompt" in params
assert "seed" in params
assert "steps" in params
assert "cfg" in params
assert "width" in params
assert "height" in params
def test_prompt_value_correct(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
# The positive prompt in the example is the landscape one
assert "landscape" in schema["parameters"]["prompt"]["value"]
assert "ugly" in schema["parameters"]["negative_prompt"]["value"]
def test_model_dependencies(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
deps = schema["model_dependencies"]
ckpts = [d["value"] for d in deps if d["folder"] == "checkpoints"]
assert "v1-5-pruned-emaonly.safetensors" in ckpts
def test_output_nodes(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
assert "9" in schema["output_nodes"]
def test_summary(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
s = schema["summary"]
assert s["has_negative_prompt"] is True
assert s["has_seed"] is True
assert s["is_video_workflow"] is False
assert s["parameter_count"] > 5
def test_flux_workflow(self, flux_workflow):
schema = extract_schema(flux_workflow)
# Flux uses RandomNoise for seed
assert schema["summary"]["has_seed"] is True
# Flux has only positive prompt (no negative encoder)
assert schema["summary"]["has_negative_prompt"] is False
def test_video_detected(self, video_workflow):
schema = extract_schema(video_workflow)
assert schema["summary"]["is_video_workflow"] is True
class TestEmbeddingDeps:
def test_extract_from_prompt(self):
wf = {
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
"5": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "CLIPTextEncode",
"inputs": {
"text": "a cat, embedding:goodvibes, embedding:art:1.2",
"clip": ["1", 1]
}},
"7": {"class_type": "CLIPTextEncode",
"inputs": {
"text": "ugly, embedding:badhands",
"clip": ["1", 1]
}},
"3": {"class_type": "KSampler",
"inputs": {
"positive": ["6", 0], "negative": ["7", 0],
"model": ["1", 0], "latent_image": ["5", 0],
"seed": 1, "steps": 20, "cfg": 7.5,
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
}},
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
}
schema = extract_schema(wf)
names = [d["embedding_name"] for d in schema["embedding_dependencies"]]
assert sorted(names) == ["art", "badhands", "goodvibes"]
class TestDuplicateDeduplication:
def test_two_ksamplers_get_unique_names(self):
wf = {
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
"5": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "a", "clip": ["1", 1]}},
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "b", "clip": ["1", 1]}},
"3": {"class_type": "KSampler",
"inputs": {
"positive": ["6", 0], "negative": ["7", 0],
"model": ["1", 0], "latent_image": ["5", 0],
"seed": 42, "steps": 20, "cfg": 7.5,
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
}},
"4": {"class_type": "KSampler",
"inputs": {
"positive": ["6", 0], "negative": ["7", 0],
"model": ["1", 0], "latent_image": ["5", 0],
"seed": 99, "steps": 30, "cfg": 8.0,
"sampler_name": "euler", "scheduler": "normal", "denoise": 0.6,
}},
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
}
schema = extract_schema(wf)
params = schema["parameters"]
# Both seeds present with disambiguated names
seed_keys = [k for k in params if "seed" in k]
# Symmetric: both renamed (no bare "seed")
assert "seed" not in params
assert "seed_3" in params and "seed_4" in params
assert params["seed_3"]["value"] == 42
assert params["seed_4"]["value"] == 99
@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""Recover a blocked / paywalled / WAF'd page from third-party copies.
Ladder (cheapest first):
1. Wayback Machine "available" API -> dated snapshot (provenance: snapshot)
2. archive.today domain rotation -> dated snapshot (provenance: snapshot)
3. Jina Reader (JINA_API_KEY only) -> live re-render (provenance: live)
Every candidate body is validated before being declared a win: byte floors,
redirect-stub detection (meta-refresh/JS pointing back at the original host),
and interstitial-title rejection. Fake 200s are the norm in this space.
Stdlib only. Usage:
python3 recover_page.py URL [--json] [--out FILE] [--timeout N]
Exit codes: 0 recovered, 1 nothing worked, 2 bad invocation.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
USER_AGENT = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
)
ARCHIVE_TODAY_HOSTS = ["archive.ph", "archive.md", "archive.li", "archive.is"]
# Titles that mean "this is not the page you asked for".
INTERSTITIAL_TITLES = (
"just a moment",
"redirecting",
"google search",
"attention required",
"access denied",
"are you a robot",
"one more step",
)
# Below these floors a body is a stub or an error page, not content.
MIN_BODY_BYTES = {"wayback": 3072, "archive_today": 3072, "jina": 512}
REDIRECT_STUB_RE = re.compile(
r'http-equiv=["\']?refresh|window\.location|location\.replace', re.IGNORECASE
)
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
def _fetch(
url: str,
timeout: int,
headers: dict | None = None,
retries_on_429: int = 2,
) -> tuple[int, bytes]:
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, **(headers or {})})
for attempt in range(retries_on_429 + 1):
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read()
except urllib.error.HTTPError as exc:
if exc.code == 429 and attempt < retries_on_429:
time.sleep(5 * (attempt + 1))
continue
return exc.code, exc.read() if exc.fp else b""
except (urllib.error.URLError, OSError, ValueError):
return 0, b""
return 0, b""
def _fetch_follow(url: str, timeout: int) -> tuple[int, bytes, str]:
"""Like _fetch but also returns the final URL after redirects."""
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read(), resp.geturl()
except urllib.error.HTTPError as exc:
return exc.code, exc.read() if exc.fp else b"", exc.geturl() or url
except (urllib.error.URLError, OSError, ValueError):
return 0, b"", url
def _page_title(body: bytes) -> str:
m = TITLE_RE.search(body[:65536].decode("utf-8", "replace"))
return re.sub(r"\s+", " ", m.group(1)).strip().lower() if m else ""
def validate(body: bytes, route: str, target_url: str) -> str | None:
"""Return a rejection reason, or None if the body looks like real content."""
floor = MIN_BODY_BYTES.get(route, 3072)
if len(body) < floor:
return f"body_too_small:{len(body)}<{floor}"
title = _page_title(body)
for marker in INTERSTITIAL_TITLES:
if marker in title:
return f"interstitial_title:{marker!r}"
# Redirect stub: small-ish page whose only job is bouncing back to the
# original (blocked) host — the classic AMP-cache failure mode.
if len(body) < 8192 and REDIRECT_STUB_RE.search(body.decode("utf-8", "replace")):
target_host = urllib.parse.urlsplit(target_url).hostname or ""
if target_host and target_host.encode() in body:
return "redirect_stub_to_origin"
return None
def try_wayback(url: str, timeout: int) -> dict | None:
snap_url = None
snap_ts = None
discovery = "https://archive.org/wayback/available?url=" + urllib.parse.quote(url, safe="")
status, raw = _fetch(discovery, timeout)
if status == 200:
try:
closest = json.loads(raw).get("archived_snapshots", {}).get("closest", {})
except (json.JSONDecodeError, AttributeError):
closest = {}
if closest.get("available") and closest.get("url"):
snap_url = closest["url"].replace(
"http://web.archive.org", "https://web.archive.org"
)
snap_ts = closest.get("timestamp")
if snap_url is None:
# Discovery API is rate-limited far more aggressively than snapshot
# serving. Fall back to the redirect form: /web/2/<url> bounces to
# the newest snapshot if one exists (404 page otherwise).
snap_url = "https://web.archive.org/web/2/" + url
status, body, final_url = _fetch_follow(snap_url, timeout)
if status != 200 or validate(body, "wayback", url):
return None
if snap_ts is None:
m = re.search(r"/web/(\d{14})", final_url)
snap_ts = m.group(1) if m else None
return {
"route": "wayback",
"provenance": "snapshot",
"snapshot_timestamp": snap_ts,
"source_url": final_url,
"body": body,
}
def try_archive_today(url: str, timeout: int) -> dict | None:
for host in ARCHIVE_TODAY_HOSTS:
fetch_url = f"https://{host}/newest/{url}"
status, body = _fetch(fetch_url, timeout)
if status != 200:
continue
if validate(body, "archive_today", url):
continue # 429 bodies and interstitials land here
return {
"route": f"archive_today:{host}",
"provenance": "snapshot",
"snapshot_timestamp": None, # archive.today embeds the date in-page
"source_url": fetch_url,
"body": body,
}
return None
def try_jina(url: str, timeout: int) -> dict | None:
key = os.environ.get("JINA_API_KEY")
if not key:
return None
status, body = _fetch(
"https://r.jina.ai/" + url, timeout, headers={"Authorization": f"Bearer {key}"}
)
if status != 200 or validate(body, "jina", url):
return None
return {
"route": "jina_reader",
"provenance": "live",
"snapshot_timestamp": None,
"source_url": "https://r.jina.ai/" + url,
"body": body,
}
ROUTES = (try_wayback, try_archive_today, try_jina)
def recover(url: str, timeout: int = 25) -> dict | None:
for route_fn in ROUTES:
result = route_fn(url, timeout)
if result:
return result
return None
def main() -> int:
ap = argparse.ArgumentParser(
description="Recover a blocked / paywalled / WAF'd page from third-party copies."
)
ap.add_argument("url")
ap.add_argument("--json", action="store_true", help="print metadata as JSON")
ap.add_argument("--out", help="write recovered body to this file")
ap.add_argument("--timeout", type=int, default=25)
args = ap.parse_args()
if not args.url.startswith(("http://", "https://")):
print("error: URL must start with http:// or https://", file=sys.stderr)
return 2
result = recover(args.url, args.timeout)
if not result:
msg = {"recovered": False, "url": args.url,
"hint": "No archive copy found. Try the API-first pivot or the browser tool."}
print(json.dumps(msg, indent=2) if args.json else msg["hint"], file=sys.stderr)
return 1
body = result.pop("body")
result.update({"recovered": True, "url": args.url, "body_bytes": len(body)})
if args.out:
with open(args.out, "wb") as fh:
fh.write(body)
result["saved_to"] = args.out
if args.json:
print(json.dumps(result, indent=2))
else:
for k, v in result.items():
print(f"{k}: {v}")
if not args.out:
print("\n--- body (first 2000 chars) ---")
print(body[:2000].decode("utf-8", "replace"))
if result["provenance"] == "snapshot":
print(
"\nNOTE: this is an ARCHIVED SNAPSHOT, not the live page. "
"Cite it with its timestamp.",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,87 @@
airtable:30f47a4b29827da14e655356c0edd8a7
apple-notes:5e448abf984561fb33b197045ce41388
apple-reminders:cda2963c73800643faf4a34ef813879a
architecture-diagram:ca5e216b2014eef4f38f0a488eaf3545
arxiv:06b6666b948852e77545c99ef72139db
ascii-art:3aea656d9b8fb9d054ce37565e704a04
ascii-video:2c8277458b2ef50421ce44debb9d81ad
audiocraft-audio-generation:c207bdbf300ea5c42decc9cb6a596d1c
baoyu-comic:8ff68387b01dea27ce049837ae9ecc47
baoyu-infographic:53edf7d1b9398d62f4ccb0755e27913e
blogwatcher:3f30bdd408c771501b94fab9289579c6
claude-code:231f7e3cb0b2b91f64ce4b23fc2cef4d
claude-design:c1b63b7651b66fd15d096e57728ec686
codebase-inspection:29f67c87df868dd08e76c57b86c7a5c6
codex:a116c5f880869f663b2974c2c9a79dbd
comfyui:7efa41ae19823ed75a05ce96d9c44e07
debugging-hermes-tui-commands:b093ceac961144fb3e72e57b12a5d354
design-md:b40264457352831ab1d06f3ec671b532
dogfood:ae6e92c2cd27c3da8a0587f089d19fe3
dspy:64659f5eba8f127700fa931f58175f01
evaluating-llms-harness:ac24cf5202db5b024b3079023797a0f6
excalidraw:149a572d2069ee3de2951352725a8b19
findmy:1d7dd3ae39cf25357a374c6bfb956442
gif-search:d02d7fb065dfc143c079f15b8d8af1a7
github-auth:c58654476268579b4cfa5953bb4160d0
github-code-review:b771855b5c56b5e3a2546167a51667ba
github-issues:429ec06cde7578e4ebba451d8106f008
github-pr-workflow:48b6aceaae5333f6d3ed2d72f595331b
github-repo-management:c5c05bc85dfcda2b7ac2f2006be9efe0
godmode:27f853c53d2a2ef38118a583a9dfb6c4
google-workspace:db46b2778030ed92d03a8aab727c036c
heartmula:96a5927a5f221065260ddb2e0f1d77ec
hermes-agent:166e68c17dec7adf1f91b20f904f251d
hermes-agent-skill-authoring:4fefd83beeea0a8a32f5ae006f10f00d
himalaya:3f05533dcd15be9d89cc86ab13c94a83
huggingface-hub:da338c5152d72db030bb81d923d1c64d
humanizer:6645b341862575f452e86139c5c71ce9
ideation:8cfdaafa9abda023643427110c8c3073
imessage:f545da0f5cc64dd9ee1ffd2b7733a11b
jupyter-live-kernel:352c43dc28428592abbc8c91cb5ce295
kanban-orchestrator:335a645935c88361ef2a103dba1fba01
kanban-worker:381c14fb24d058e1750d1f97aa8d8011
linear:58a67824f7df6e4753cfe0575be9e921
llama-cpp:0991055ce47146735f0ed02d7658a254
llm-wiki:ca957fbfecd9e573a1f99db3937a1969
macos-computer-use:fb1787b32be343b87078d2ae2618dbcf
manim-video:2ad3d68c3eb5d2675c05138100d3e48b
maps:6d94a271a5341dc94789a9c99eb2b1a8
minecraft-modpack-server:71e51465a1df193d7505ee47a91b6dbd
nano-pdf:6c643bd0cfb0548ff0ddaf367d4da6d1
native-mcp:3e758c66efed8f27c2153729d2050c87
node-inspect-debugger:55501511963a3a6410fc767b5ed3e21c
notion:930f6c7d327dd86553f45e2da9d9901e
obliteratus:d3333944ef536dbe9238766f2f626eae
obsidian:be02894afa45a908623ad6f9b7594199
ocr-and-documents:af5fba9fa8ef003951ff5fe5a0a04adf
opencode:d2a166c7f2c74f6e47d548ed1290c458
openhue:ce1dd061d7f49d4752a4c0711ad2666c
p5js:5f09fa1cb8494c93bc2f5bcbd34a2ead
pixel-art:39562dfa3598566fc09280c054ef0ede
plan:50c4df52eb6b0932649a444bc9984e0a
pokemon-player:db79617ea4a61e0d6727d38402fb1f94
polymarket:7644c886e028c229bc8c1f54114c3170
popular-web-designs:b3fd685e8fbcf981755609ce98a4eea9
powerpoint:00a6eb2ad4b7be22c1eabf6c19158836
pretext:f17f2b6211eb81b96e1fc5d48ecc96a4
python-debugpy:b87e0abf179c14ea51c7559dc99eb22c
requesting-code-review:9944f54e8ba5c294b47ee93f3e8f239d
research-paper-writing:a4a5a13107ff94894a4abdb39a5c7a8d
segment-anything-model:7f1317da421fb8eada27aeacdbb21d30
serving-llms-vllm:92d66ae1f1112924634fcdcff2f86bc7
sketch:f8833126112824f6a916c69630cfd042
songsee:644dc0f267b6661a3df6c76ce80d7f1f
songwriting-and-ai-music:52be403c894c7bd7d6fe70f7eeaf9460
spike:8b01bbb339d72d13a8a3214a018969c4
spotify:a0ffe21977c14dc8da5dca039de83b0f
subagent-driven-development:f434c120ce9cc40fadb9ff970b45c1e7
systematic-debugging:55a23d83e936d3c7c04d4e5fa803558e
teams-meeting-pipeline:810d4862830d1eb3daccf898f7c5ee85
test-driven-development:31fb75eff421db47e527bce28c493df3
touchdesigner-mcp:0664ded9138795d5518def4d16037650
webhook-subscriptions:b1ffc6d6a3cf3f59ea8eb7dcd178c555
weights-and-biases:8f0e1ee92fdf7b42a1dad448176d7c64
writing-plans:ebe31eb58f7a09a7cd8fad4d422b9d3b
xurl:e44ca3f6818dd7391a9b12ae79d76c16
youtube-content:e43a762cd557db987d9ab466a7a60f76
yuanbao:7844c287c57b42dccf51127f15e0751b
@@ -0,0 +1,185 @@
"""Tests for extract_schema.py."""
from __future__ import annotations
import pytest
from extract_schema import (
extract_schema,
find_negative_prompt_node,
find_positive_prompt_node,
trace_to_node,
)
# =============================================================================
# Connection tracing
# =============================================================================
class TestConnectionTracing:
def test_direct_link(self):
wf = {
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
"2": {"class_type": "KSampler",
"inputs": {"positive": ["1", 0], "negative": ["1", 0]}},
}
assert trace_to_node(wf, ["1", 0]) == "1"
def test_through_reroute(self):
wf = {
"1": {"class_type": "CLIPTextEncode", "inputs": {"text": "x"}},
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
"3": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
}
assert trace_to_node(wf, ["3", 0]) == "1"
def test_circular_safe(self):
wf = {
"1": {"class_type": "Reroute", "inputs": {"input": ["2", 0]}},
"2": {"class_type": "Reroute", "inputs": {"input": ["1", 0]}},
}
# Should hit max_hops without infinite loop
result = trace_to_node(wf, ["1", 0], max_hops=5)
assert result in {"1", "2"} # any node, just don't hang
class TestPositiveNegativeDetection:
def test_basic(self, sd15_workflow):
# In sd15_workflow.json node 6 is positive, node 7 is negative
assert find_positive_prompt_node(sd15_workflow) == "6"
assert find_negative_prompt_node(sd15_workflow) == "7"
def test_swapped_order(self):
wf = {
"3": {"class_type": "KSampler",
"inputs": {
"positive": ["7", 0], "negative": ["6", 0],
"model": ["4", 0], "latent_image": ["5", 0],
"seed": 1, "steps": 20, "cfg": 7.5,
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
}},
"4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
"5": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "ugly", "clip": ["4", 1]}},
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "beautiful", "clip": ["4", 1]}},
}
# Now 7 is the positive (despite higher node ID)
assert find_positive_prompt_node(wf) == "7"
assert find_negative_prompt_node(wf) == "6"
# =============================================================================
# Schema extraction
# =============================================================================
class TestExtractSchema:
def test_basic_sd15(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
params = schema["parameters"]
assert "prompt" in params
assert "negative_prompt" in params
assert "seed" in params
assert "steps" in params
assert "cfg" in params
assert "width" in params
assert "height" in params
def test_prompt_value_correct(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
# The positive prompt in the example is the landscape one
assert "landscape" in schema["parameters"]["prompt"]["value"]
assert "ugly" in schema["parameters"]["negative_prompt"]["value"]
def test_model_dependencies(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
deps = schema["model_dependencies"]
ckpts = [d["value"] for d in deps if d["folder"] == "checkpoints"]
assert "v1-5-pruned-emaonly.safetensors" in ckpts
def test_output_nodes(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
assert "9" in schema["output_nodes"]
def test_summary(self, sd15_workflow):
schema = extract_schema(sd15_workflow)
s = schema["summary"]
assert s["has_negative_prompt"] is True
assert s["has_seed"] is True
assert s["is_video_workflow"] is False
assert s["parameter_count"] > 5
def test_flux_workflow(self, flux_workflow):
schema = extract_schema(flux_workflow)
# Flux uses RandomNoise for seed
assert schema["summary"]["has_seed"] is True
# Flux has only positive prompt (no negative encoder)
assert schema["summary"]["has_negative_prompt"] is False
def test_video_detected(self, video_workflow):
schema = extract_schema(video_workflow)
assert schema["summary"]["is_video_workflow"] is True
class TestEmbeddingDeps:
def test_extract_from_prompt(self):
wf = {
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
"5": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "CLIPTextEncode",
"inputs": {
"text": "a cat, embedding:goodvibes, embedding:art:1.2",
"clip": ["1", 1]
}},
"7": {"class_type": "CLIPTextEncode",
"inputs": {
"text": "ugly, embedding:badhands",
"clip": ["1", 1]
}},
"3": {"class_type": "KSampler",
"inputs": {
"positive": ["6", 0], "negative": ["7", 0],
"model": ["1", 0], "latent_image": ["5", 0],
"seed": 1, "steps": 20, "cfg": 7.5,
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
}},
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
}
schema = extract_schema(wf)
names = [d["embedding_name"] for d in schema["embedding_dependencies"]]
assert sorted(names) == ["art", "badhands", "goodvibes"]
class TestDuplicateDeduplication:
def test_two_ksamplers_get_unique_names(self):
wf = {
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "x"}},
"5": {"class_type": "EmptyLatentImage",
"inputs": {"width": 512, "height": 512, "batch_size": 1}},
"6": {"class_type": "CLIPTextEncode", "inputs": {"text": "a", "clip": ["1", 1]}},
"7": {"class_type": "CLIPTextEncode", "inputs": {"text": "b", "clip": ["1", 1]}},
"3": {"class_type": "KSampler",
"inputs": {
"positive": ["6", 0], "negative": ["7", 0],
"model": ["1", 0], "latent_image": ["5", 0],
"seed": 42, "steps": 20, "cfg": 7.5,
"sampler_name": "euler", "scheduler": "normal", "denoise": 1.0,
}},
"4": {"class_type": "KSampler",
"inputs": {
"positive": ["6", 0], "negative": ["7", 0],
"model": ["1", 0], "latent_image": ["5", 0],
"seed": 99, "steps": 30, "cfg": 8.0,
"sampler_name": "euler", "scheduler": "normal", "denoise": 0.6,
}},
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": "x", "images": ["3", 0]}},
}
schema = extract_schema(wf)
params = schema["parameters"]
# Both seeds present with disambiguated names
seed_keys = [k for k in params if "seed" in k]
# Symmetric: both renamed (no bare "seed")
assert "seed" not in params
assert "seed_3" in params and "seed_4" in params
assert params["seed_3"]["value"] == 42
assert params["seed_4"]["value"] == 99
@@ -0,0 +1,23 @@
"""Resolve HERMES_HOME for standalone skill scripts.
Skill scripts may run outside the Hermes process (system Python, nix env,
CI) where ``hermes_constants`` is not importable. This module provides the
same ``get_hermes_home()`` contract without requiring it on ``sys.path``.
When ``hermes_constants`` IS available it is used directly so profile
resolution and any future enhancements are picked up automatically.
"""
from __future__ import annotations
import os
from pathlib import Path
try:
from hermes_constants import get_hermes_home as get_hermes_home
except (ModuleNotFoundError, ImportError):
def get_hermes_home() -> Path:
"""Return the Hermes home directory (default: ``~/.hermes``)."""
val = os.environ.get("HERMES_HOME", "").strip()
return Path(val) if val else Path.home() / ".hermes"
@@ -0,0 +1,678 @@
#!/usr/bin/env python3
"""Citation ledger for grounded answers and documents.
Owns the ``url -> [n]`` mapping used by the ``grounded-citations`` skill.
Ids are assigned at retrieval time and never change, so a draft's ``[3]``
always resolves to the same page. The model only ever emits integers the
ledger handed it, which is what makes the citations verifiable.
Subcommands
-----------
reset start a clean ledger
add URL [URL ...] register source(s), print their ids
ingest FILE|- register every url found in JSON tool output
quote ID --text T --from FILE|- attach verbatim supporting evidence to a source
list show the ledger
render render a Sources block
verify DRAFT check a draft's citations against the ledger
Fact-checking is evidence-backed citation: ``quote`` only accepts text that
literally appears in the fetched page text you point it at, ``verify
--evidence`` requires every cited source to carry at least one such quote, and
``render --style evidence`` prints the quotes under each source so the reader
can check the chain themselves. Claims from model knowledge are declared with
an ``[unverified]`` marker rather than silently blended in.
Ledger path resolution (first wins):
--ledger PATH
$HERMES_CITATION_LEDGER
$HERMES_HOME/cache/citations/ledger.json
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
import time
from pathlib import Path
from typing import Any, Iterable
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _hermes_home import get_hermes_home # noqa: E402
SCHEMA_VERSION = 1
# A citation marker in prose: [12]. Markdown links ([text](url)) and
# reference-style labels are excluded by requiring digits only and no
# following "(" or ":".
_CITE_RE = re.compile(r"\[(\d{1,4})\](?![(:])")
_SOURCES_HEADER_RE = re.compile(r"^\s*(?:#{1,6}\s*)?(?:\*\*)?sources:?(?:\*\*)?\s*$", re.IGNORECASE)
_SOURCE_LINE_RE = re.compile(r"^\s*\[(\d{1,4})\]\s*[-–:]?\s*(\S+)")
_URL_IN_TEXT_RE = re.compile(r"https?://[^\s\"'<>)\]}]+")
_FENCE_RE = re.compile(r"^\s*(?:```|~~~)")
# Explicit declaration that a claim comes from model knowledge, not a source.
_UNVERIFIED_RE = re.compile(r"\[unverified\]", re.IGNORECASE)
# ---------------------------------------------------------------------------
# Ledger I/O
# ---------------------------------------------------------------------------
def resolve_ledger_path(explicit: str | None = None) -> Path:
if explicit:
return Path(explicit).expanduser()
env = os.environ.get("HERMES_CITATION_LEDGER", "").strip()
if env:
return Path(env).expanduser()
return get_hermes_home() / "cache" / "citations" / "ledger.json"
def normalize_url(url: str) -> str:
"""Canonicalize a URL for ledger identity.
Strips the fragment and a trailing slash so ``/page``, ``/page/`` and
``/page#section`` are one source. Query strings are significant and are
kept — they usually select different content.
"""
u = (url or "").strip()
if "#" in u:
u = u.split("#", 1)[0]
stripped = u.rstrip("/")
return stripped or u
def load_ledger(path: Path) -> dict[str, Any]:
if not path.exists():
return {"version": SCHEMA_VERSION, "sources": []}
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
raise SystemExit(f"error: ledger at {path} is unreadable ({exc}); run `reset` to start over")
if not isinstance(data, dict) or not isinstance(data.get("sources"), list):
raise SystemExit(f"error: ledger at {path} has an unexpected shape; run `reset`")
return data
def save_ledger(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
os.replace(tmp, path)
class _LedgerLock:
"""Best-effort cross-process lock (O_EXCL lockfile, stdlib only).
Parallel subagents can share one ledger via --ledger; without a lock two
concurrent ``add`` calls can assign the same id. Falls through after the
timeout rather than blocking a task forever — a stale lock must never
wedge the ledger.
"""
def __init__(self, path: Path, timeout: float = 5.0) -> None:
self.lock_path = path.with_suffix(path.suffix + ".lock")
self.timeout = timeout
self.fd: int | None = None
def __enter__(self) -> "_LedgerLock":
self.lock_path.parent.mkdir(parents=True, exist_ok=True)
deadline = time.monotonic() + self.timeout
while True:
try:
self.fd = os.open(str(self.lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
return self
except FileExistsError:
if time.monotonic() >= deadline:
# Assume a stale lock from a crashed run.
try:
self.lock_path.unlink()
except OSError:
return self
continue
time.sleep(0.05)
def __exit__(self, *_exc: object) -> None:
if self.fd is not None:
try:
os.close(self.fd)
except OSError:
pass
try:
self.lock_path.unlink()
except OSError:
pass
# ---------------------------------------------------------------------------
# Core operations
# ---------------------------------------------------------------------------
def _by_url(sources: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {s["url"]: s for s in sources}
def add_sources(
path: Path,
urls: Iterable[str],
title: str | None = None,
accessed: str | None = None,
) -> list[dict[str, Any]]:
"""Register urls, returning their ledger entries (existing or new)."""
urls = [u for u in (str(u).strip() for u in urls) if u]
if not urls:
return []
with _LedgerLock(path):
data = load_ledger(path)
sources = data["sources"]
index = _by_url(sources)
out: list[dict[str, Any]] = []
changed = False
for raw in urls:
key = normalize_url(raw)
existing = index.get(key)
if existing is not None:
if title and not existing.get("title"):
existing["title"] = title
changed = True
out.append(existing)
continue
entry = {
"id": len(sources) + 1,
"url": key,
"title": (title or "").strip(),
"accessed": accessed or time.strftime("%Y-%m-%d"),
}
sources.append(entry)
index[key] = entry
out.append(entry)
changed = True
if changed:
save_ledger(path, data)
return out
def urls_from_json(payload: Any) -> list[tuple[str, str]]:
"""Walk arbitrary JSON tool output collecting (url, title) pairs.
Handles web_search (``data.web[]``), web_extract (``results[]``) and any
other nesting, in document order, deduped.
"""
found: list[tuple[str, str]] = []
seen: set[str] = set()
def walk(node: Any) -> None:
if isinstance(node, dict):
url = node.get("url") or node.get("link") or node.get("source_url")
if isinstance(url, str) and url.startswith(("http://", "https://")):
key = normalize_url(url)
if key not in seen:
seen.add(key)
raw_title = node.get("title") or node.get("name") or ""
found.append((url, raw_title if isinstance(raw_title, str) else ""))
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return found
# ---------------------------------------------------------------------------
# Evidence quotes (fact-checking)
# ---------------------------------------------------------------------------
def _normalize_ws(text: str) -> str:
"""Collapse all whitespace runs to single spaces for verbatim matching."""
return " ".join((text or "").split())
# Markdown artifacts that retrieval tools inject into otherwise-identical prose.
# ``web_extract`` returns markdown, so the most citation-worthy sentences are
# exactly the ones carrying inline links and emphasis around terms:
# "including _[ERAP1](https://…/erap1/)_, _[IL1A](…)_, have also been…"
# reads identically to the page a human sees. Matching has to see through that
# markup, or the skill pushes the agent toward weaker evidence fragments.
_MD_LINK_RE = re.compile(r"\[([^\]]*)\]\((?:[^()\s]|\([^()]*\))*\)")
_MD_NOISE_RE = re.compile(r"[*_`~]|\\(?=[^\w\s])")
def _match_key(text: str) -> str:
"""Canonicalize text for verbatim comparison.
Whitespace-, case-, and markdown-insensitive: inline links collapse to
their label, emphasis/code markers and backslash escapes are dropped. The
stored quote keeps whatever the caller passed, so the rendered evidence
block shows clean prose rather than extractor artifacts.
"""
collapsed = _MD_LINK_RE.sub(r"\1", text or "")
return _normalize_ws(_MD_NOISE_RE.sub("", collapsed)).casefold()
def quote_in_evidence(quote: str, evidence: str) -> bool:
"""True when ``quote`` appears verbatim in the fetched ``evidence`` text,
ignoring whitespace, case, and markdown markup on either side."""
q = _match_key(quote)
return bool(q) and q in _match_key(evidence)
def attach_quote(path: Path, source_id: int, quote: str, evidence: str) -> dict[str, Any]:
"""Attach a verbatim quote to a ledger entry after checking it against
the evidence text. Raises SystemExit on unknown id or non-verbatim text —
a quote the page does not contain is exactly the fabrication this guards
against."""
quote = (quote or "").strip()
if len(_normalize_ws(quote).split()) < 3:
raise SystemExit("error: quote too short — use at least 3 words of verbatim text")
if not quote_in_evidence(quote, evidence):
raise SystemExit(
"error: quote not found verbatim in the evidence text — "
"copy the exact wording from the fetched page, do not paraphrase"
)
with _LedgerLock(path):
data = load_ledger(path)
entry = next((s for s in data["sources"] if s["id"] == source_id), None)
if entry is None:
raise SystemExit(f"error: no source [{source_id}] in the ledger")
quotes = entry.setdefault("quotes", [])
norm = _match_key(quote)
if not any(_match_key(q.get("text", "")) == norm for q in quotes):
quotes.append({"text": quote, "added": time.strftime("%Y-%m-%d")})
save_ledger(path, data)
return entry
def render_sources(
sources: list[dict[str, Any]],
style: str = "markdown",
only: set[int] | None = None,
) -> str:
picked = [s for s in sources if only is None or s["id"] in only]
picked.sort(key=lambda s: s["id"])
if not picked:
return ""
lines: list[str] = []
if style == "bibtex":
for s in picked:
key = f"source{s['id']}"
title = s.get("title") or s["url"]
lines.append(
"@misc{%s,\n title = {%s},\n howpublished = {\\url{%s}},\n note = {Accessed %s}\n}"
% (key, title, s["url"], s.get("accessed", ""))
)
return "\n".join(lines)
if style == "footnotes":
for s in picked:
title = s.get("title")
suffix = f" — {title}" if title else ""
lines.append(f"[^{s['id']}]: {s['url']}{suffix}")
return "\n".join(lines)
header = "Sources:" if style == "plain" else "## Sources"
lines.append(header)
if style != "plain":
lines.append("")
for s in picked:
title = s.get("title")
suffix = f" — {title}" if title else ""
lines.append(f"[{s['id']}] {s['url']}{suffix}")
if style == "evidence":
for q in s.get("quotes", []):
lines.append(f' > "{q.get("text", "")}"')
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Verification
# ---------------------------------------------------------------------------
def _split_draft(text: str) -> tuple[str, dict[int, str]]:
"""Split a draft into (prose, sources_block_map).
The sources block is everything after the last ``Sources`` header; its
``[n] url`` lines are parsed out so they aren't counted as prose citations.
Fenced code blocks are dropped from prose.
"""
lines = text.splitlines()
header_idx = -1
for i, line in enumerate(lines):
if _SOURCES_HEADER_RE.match(line):
header_idx = i
listed: dict[int, str] = {}
if header_idx >= 0:
for line in lines[header_idx + 1:]:
m = _SOURCE_LINE_RE.match(line)
if m:
url_match = _URL_IN_TEXT_RE.search(line)
listed[int(m.group(1))] = url_match.group(0) if url_match else m.group(2)
body_lines = lines[:header_idx]
else:
body_lines = lines
prose: list[str] = []
in_fence = False
for line in body_lines:
if _FENCE_RE.match(line):
in_fence = not in_fence
continue
if not in_fence:
prose.append(line)
return "\n".join(prose), listed
def _strip_sources_block(text: str) -> str:
"""Return the draft with its trailing Sources block removed.
Everything from the last Sources header onward goes; a draft with no such
header is returned unchanged. This is what makes ``render --replace-in``
idempotent instead of stacking duplicate blocks.
"""
lines = text.splitlines()
header_idx = -1
for i, line in enumerate(lines):
if _SOURCES_HEADER_RE.match(line):
header_idx = i
if header_idx < 0:
return text
return "\n".join(lines[:header_idx])
def _sentences(prose: str) -> list[str]:
"""Rough sentence split over prose lines, skipping headings and tables."""
out: list[str] = []
for line in prose.splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or stripped.startswith("|"):
continue
if stripped.startswith(">"):
stripped = stripped.lstrip("> ").strip()
for part in re.split(r"(?<=[.!?])\s+", stripped):
part = part.strip()
if len(part.split()) >= 4:
out.append(part)
return out
def verify_draft(
draft_path: Path,
sources: list[dict[str, Any]],
strict: bool = False,
min_coverage: float | None = None,
require_evidence: bool = False,
) -> tuple[int, list[str], list[str]]:
"""Return (exit_code, errors, warnings)."""
text = draft_path.read_text(encoding="utf-8")
prose, listed = _split_draft(text)
by_id = {s["id"]: s for s in sources}
errors: list[str] = []
warnings: list[str] = []
cited = [int(m) for m in _CITE_RE.findall(prose)]
cited_set = set(cited)
unknown = sorted(i for i in cited_set if i not in by_id)
if unknown:
errors.append(
"citations not in the ledger (hallucinated or renumbered): "
+ ", ".join(f"[{i}]" for i in unknown)
)
if cited_set and not listed:
errors.append("draft cites sources but has no `Sources:` block — run `render --cited-in`")
missing_from_block = sorted(cited_set - set(listed)) if listed else []
if missing_from_block:
errors.append(
"cited but absent from the Sources block: "
+ ", ".join(f"[{i}]" for i in missing_from_block)
)
for sid, url in sorted(listed.items()):
entry = by_id.get(sid)
if entry is None:
errors.append(f"Sources block lists [{sid}], which is not in the ledger")
continue
if normalize_url(url) != entry["url"]:
errors.append(
f"Sources block URL for [{sid}] does not match the ledger "
f"(block: {url} / ledger: {entry['url']}) — re-run `render`"
)
extra_in_block = sorted(set(listed) - cited_set)
if extra_in_block:
warnings.append(
"listed in Sources but never cited inline: "
+ ", ".join(f"[{i}]" for i in extra_in_block)
)
registered_uncited = sorted(set(by_id) - cited_set)
if registered_uncited:
warnings.append(
"registered in the ledger but not cited in this draft: "
+ ", ".join(f"[{i}]" for i in registered_uncited)
)
sentences = _sentences(prose)
cited_sentences = [s for s in sentences if _CITE_RE.search(s)]
unverified_sentences = [s for s in sentences if _UNVERIFIED_RE.search(s)]
covered = [s for s in sentences if _CITE_RE.search(s) or _UNVERIFIED_RE.search(s)]
coverage = (len(covered) / len(sentences)) if sentences else 0.0
if min_coverage is not None and sentences and coverage < min_coverage:
errors.append(
f"citation coverage {coverage:.0%} is below the required {min_coverage:.0%} "
f"({len(covered)}/{len(sentences)} sentences cited or marked [unverified])"
)
if require_evidence:
unevidenced = sorted(
i for i in cited_set if i in by_id and not by_id[i].get("quotes")
)
if unevidenced:
errors.append(
"cited sources carry no verbatim evidence quote (run `quote` with the "
"fetched page text): " + ", ".join(f"[{i}]" for i in unevidenced)
)
over_cited = [s for s in sentences if len(_CITE_RE.findall(s)) > 3]
if over_cited:
warnings.append(f"{len(over_cited)} sentence(s) carry more than 3 citations")
code = 1 if errors else (1 if (strict and warnings) else 0)
quoted = sum(1 for s in sources if s.get("quotes"))
stats = (
f"{len(sentences)} prose sentence(s), {len(covered)} with declared provenance "
f"({coverage:.0%}) — {len(cited_sentences)} cited, "
f"{len(unverified_sentences)} marked [unverified] (a sentence may be both); "
f"{len(cited_set)} distinct source(s) cited, "
f"{len(by_id)} in ledger ({quoted} with evidence quotes)"
)
warnings.insert(0, f"stats: {stats}")
return code, errors, warnings
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def _parse_only(spec: str | None) -> set[int] | None:
if not spec:
return None
out: set[int] = set()
for chunk in spec.replace(" ", "").split(","):
if not chunk:
continue
if "-" in chunk:
lo, _, hi = chunk.partition("-")
out.update(range(int(lo), int(hi) + 1))
else:
out.add(int(chunk))
return out
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
prog="sources.py", description="Citation ledger for grounded answers and documents."
)
parser.add_argument("--ledger", help="ledger file path (overrides env / default)")
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("reset", help="start a clean ledger")
p_add = sub.add_parser("add", help="register source url(s), print their ids")
p_add.add_argument("urls", nargs="+")
p_add.add_argument("--title", help="title for the source (single-url calls)")
p_add.add_argument("--accessed", help="access date (default: today)")
p_add.add_argument("--json", action="store_true", help="emit JSON instead of ids")
p_ing = sub.add_parser("ingest", help="register every url in JSON tool output")
p_ing.add_argument("file", help="JSON file, or - for stdin")
p_q = sub.add_parser("quote", help="attach verbatim supporting evidence to a source")
p_q.add_argument("id", type=int, help="ledger id of the source the quote supports")
p_q.add_argument("--text", required=True, help="the exact quote, copied from the page")
p_q.add_argument(
"--from",
dest="evidence",
required=True,
help="file with the fetched page text (or - for stdin) the quote must appear in",
)
p_list = sub.add_parser("list", help="show the ledger")
p_list.add_argument("--json", action="store_true")
p_render = sub.add_parser("render", help="render a Sources block")
p_render.add_argument(
"--style", default="markdown", choices=["markdown", "plain", "footnotes", "bibtex", "evidence"]
)
p_render.add_argument("--only", help="ids to include, e.g. 1,3,5-7")
p_render.add_argument("--cited-in", help="include only ids cited in this draft file")
p_render.add_argument(
"--replace-in",
help="rewrite this draft's Sources block in place (implies --cited-in on it)",
)
p_ver = sub.add_parser("verify", help="check a draft's citations against the ledger")
p_ver.add_argument("draft")
p_ver.add_argument("--strict", action="store_true", help="treat warnings as failures")
p_ver.add_argument("--min-coverage", type=float, help="required cited-sentence share, e.g. 0.5")
p_ver.add_argument(
"--evidence",
action="store_true",
help="require every cited source to carry at least one verbatim quote",
)
args = parser.parse_args(argv)
path = resolve_ledger_path(args.ledger)
if args.cmd == "reset":
with _LedgerLock(path):
save_ledger(path, {"version": SCHEMA_VERSION, "sources": []})
print(f"ledger reset: {path}")
return 0
if args.cmd == "add":
title = args.title if len(args.urls) == 1 else None
entries = add_sources(path, args.urls, title=title, accessed=args.accessed)
if args.json:
print(json.dumps(entries, indent=2, ensure_ascii=False))
else:
for e in entries:
print(f"[{e['id']}] {e['url']}")
return 0
if args.cmd == "ingest":
raw = sys.stdin.read() if args.file == "-" else Path(args.file).read_text(encoding="utf-8")
try:
payload = json.loads(raw)
except json.JSONDecodeError as exc:
print(f"error: input is not valid JSON ({exc})", file=sys.stderr)
return 2
pairs = urls_from_json(payload)
if not pairs:
print("no urls found in input", file=sys.stderr)
return 1
for url, title in pairs:
entry = add_sources(path, [url], title=title or None)[0]
print(f"[{entry['id']}] {entry['url']}")
return 0
if args.cmd == "quote":
raw = (
sys.stdin.read()
if args.evidence == "-"
else Path(args.evidence).read_text(encoding="utf-8")
)
entry = attach_quote(path, args.id, args.text, raw)
print(f"[{entry['id']}] evidence attached ({len(entry.get('quotes', []))} quote(s))")
return 0
data = load_ledger(path)
sources = sorted(data["sources"], key=lambda s: s["id"])
if args.cmd == "list":
if args.json:
print(json.dumps(sources, indent=2, ensure_ascii=False))
elif not sources:
print(f"ledger is empty: {path}")
else:
for s in sources:
title = f" {s['title']}" if s.get("title") else ""
nq = len(s.get("quotes", []))
mark = f" ({nq} quote{'s' if nq != 1 else ''})" if nq else ""
print(f"[{s['id']}] {s['url']}{title}{mark}")
return 0
if args.cmd == "render":
only = _parse_only(args.only)
draft_for_ids = args.replace_in or args.cited_in
if draft_for_ids:
draft = Path(draft_for_ids).read_text(encoding="utf-8")
prose, _ = _split_draft(draft)
cited = {int(m) for m in _CITE_RE.findall(prose)}
only = cited if only is None else (only & cited)
block = render_sources(sources, style=args.style, only=only)
if not block:
print("no sources to render", file=sys.stderr)
return 1
if args.replace_in:
target = Path(args.replace_in)
body = _strip_sources_block(target.read_text(encoding="utf-8"))
target.write_text(body.rstrip("\n") + "\n\n" + block + "\n", encoding="utf-8")
print(f"Sources block rewritten in {target}")
return 0
print(block)
return 0
if args.cmd == "verify":
draft_path = Path(args.draft)
if not draft_path.is_file():
print(f"error: no such draft: {draft_path}", file=sys.stderr)
return 2
code, errors, warnings = verify_draft(
draft_path,
sources,
strict=args.strict,
min_coverage=args.min_coverage,
require_evidence=args.evidence,
)
for w in warnings:
prefix = "info" if w.startswith("stats: ") else "warn"
print(f"{prefix}: {w}")
for e in errors:
print(f"FAIL: {e}", file=sys.stderr)
print("citations OK" if code == 0 else "verification failed")
return code
return 2
if __name__ == "__main__":
sys.exit(main())
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Wrapper script for last30days engine — handles config and invocation."""
import subprocess
import sys
import os
from pathlib import Path
ENGINE_DIR = Path.home() / "workspace" / "last30days-skill"
ENGINE_SCRIPT = ENGINE_DIR / "skills" / "last30days" / "scripts" / "last30days.py"
def main():
if not ENGINE_SCRIPT.exists():
print(f"❌ Engine not found at {ENGINE_SCRIPT}", file=sys.stderr)
print("Clone it: git clone https://github.com/mvanhorn/last30days-skill.git ~/workspace/last30days-skill", file=sys.stderr)
sys.exit(1)
# Forward all args to the engine
cmd = ["python3", str(ENGINE_SCRIPT)] + sys.argv[1:]
result = subprocess.run(
cmd,
cwd=str(ENGINE_DIR),
capture_output=True,
text=True,
timeout=120
)
# Print stderr first (diagnostics, warnings)
if result.stderr:
print(result.stderr, file=sys.stderr, end="")
# Print stdout (the actual results)
if result.stdout:
print(result.stdout)
sys.exit(result.returncode)
if __name__ == "__main__":
main()
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Wrapper for Obscura headless browser — simplifies common operations."""
import subprocess
import sys
import os
from pathlib import Path
OBSCURA_BIN = Path.home() / ".local" / "bin" / "obscura"
def fetch(url: str, dump: str = "markdown", timeout: int = 15, wait_until: str = None, output: str = None) -> str:
"""Fetch a single page and return markdown/text/html."""
cmd = [str(OBSCURA_BIN), "fetch", url, "--dump", dump, "--timeout", str(timeout)]
if wait_until:
cmd.extend(["--wait-until", wait_until])
if output:
cmd.extend(["--output", output])
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 10)
if result.returncode != 0:
return f"ERROR: {result.stderr}"
return result.stdout
def eval_js(url: str, expression: str, timeout: int = 15) -> str:
"""Fetch a page and evaluate JavaScript."""
cmd = [str(OBSCURA_BIN), "fetch", url, "--eval", expression, "--timeout", str(timeout)]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 10)
if result.returncode != 0:
return f"ERROR: {result.stderr}"
# Filter out the "Fetching..." and "Page loaded..." lines
lines = result.stdout.strip().split("\n")
return "\n".join(line for line in lines if not line.startswith(("Fetching", "Page loaded")))
def main():
if len(sys.argv) < 2:
print("Usage: obscura_wrapper.py fetch <url> [--dump markdown|text|html] [--timeout N] [--wait-until networkidle0]")
print(" obscura_wrapper.py eval <url> <js_expression>")
sys.exit(1)
action = sys.argv[1]
if action == "fetch":
url = sys.argv[2]
dump = "markdown"
timeout = 15
wait_until = None
# Parse optional flags
args = sys.argv[3:]
i = 0
while i < len(args):
if args[i] == "--dump" and i + 1 < len(args):
dump = args[i + 1]
i += 2
elif args[i] == "--timeout" and i + 1 < len(args):
timeout = int(args[i + 1])
i += 2
elif args[i] == "--wait-until" and i + 1 < len(args):
wait_until = args[i + 1]
i += 2
else:
i += 1
print(fetch(url, dump=dump, timeout=timeout, wait_until=wait_until))
elif action == "eval":
url = sys.argv[2]
expression = " ".join(sys.argv[3:])
print(eval_js(url, expression))
else:
print(f"Unknown action: {action}")
sys.exit(1)
if __name__ == "__main__":
main()
@@ -0,0 +1,284 @@
#!/usr/bin/env python3
"""Polymarket CLI helper — query prediction market data.
Usage:
python3 polymarket.py search "bitcoin"
python3 polymarket.py trending [--limit 10]
python3 polymarket.py market <slug>
python3 polymarket.py event <slug>
python3 polymarket.py price <token_id>
python3 polymarket.py book <token_id>
python3 polymarket.py history <condition_id> [--interval all] [--fidelity 50]
python3 polymarket.py trades [--limit 10] [--market CONDITION_ID]
"""
import json
import sys
import urllib.request
import urllib.parse
import urllib.error
GAMMA = "https://gamma-api.polymarket.com"
CLOB = "https://clob.polymarket.com"
DATA = "https://data-api.polymarket.com"
def _get(url: str) -> dict | list:
"""GET request, return parsed JSON."""
req = urllib.request.Request(url, headers={"User-Agent": "hermes-agent/1.0"})
try:
with urllib.request.urlopen(req, timeout=15) as resp:
return json.loads(resp.read().decode())
except urllib.error.HTTPError as e:
print(f"HTTP {e.code}: {e.reason}", file=sys.stderr)
sys.exit(1)
except urllib.error.URLError as e:
print(f"Connection error: {e.reason}", file=sys.stderr)
sys.exit(1)
def _parse_json_field(val):
"""Parse double-encoded JSON fields (outcomePrices, outcomes, clobTokenIds)."""
if isinstance(val, str):
try:
return json.loads(val)
except (json.JSONDecodeError, TypeError):
return val
return val
def _fmt_pct(price_str: str) -> str:
"""Format price string as percentage."""
try:
return f"{float(price_str) * 100:.1f}%"
except (ValueError, TypeError):
return price_str
def _fmt_volume(vol) -> str:
"""Format volume as human-readable."""
try:
v = float(vol)
if v >= 1_000_000:
return f"${v / 1_000_000:.1f}M"
if v >= 1_000:
return f"${v / 1_000:.1f}K"
return f"${v:.0f}"
except (ValueError, TypeError):
return str(vol)
def _print_market(m: dict, indent: str = ""):
"""Print a market summary."""
question = m.get("question", "?")
prices = _parse_json_field(m.get("outcomePrices", "[]"))
outcomes = _parse_json_field(m.get("outcomes", "[]"))
vol = _fmt_volume(m.get("volume", 0))
closed = m.get("closed", False)
status = " [CLOSED]" if closed else ""
if isinstance(prices, list) and len(prices) >= 2:
outcome_labels = outcomes if isinstance(outcomes, list) else ["Yes", "No"]
price_str = " / ".join(
f"{outcome_labels[i]}: {_fmt_pct(prices[i])}"
for i in range(min(len(prices), len(outcome_labels)))
)
print(f"{indent}{question}{status}")
print(f"{indent} {price_str} | Volume: {vol}")
else:
print(f"{indent}{question}{status} | Volume: {vol}")
slug = m.get("slug", "")
if slug:
print(f"{indent} slug: {slug}")
def cmd_search(query: str):
"""Search for markets."""
q = urllib.parse.quote(query)
data = _get(f"{GAMMA}/public-search?q={q}")
events = data.get("events", [])
total = data.get("pagination", {}).get("totalResults", len(events))
print(f"Found {total} results for \"{query}\":\n")
for evt in events[:10]:
print(f"=== {evt['title']} ===")
print(f" Volume: {_fmt_volume(evt.get('volume', 0))} | slug: {evt.get('slug', '')}")
markets = evt.get("markets", [])
for m in markets[:5]:
_print_market(m, indent=" ")
if len(markets) > 5:
print(f" ... and {len(markets) - 5} more markets")
print()
def cmd_trending(limit: int = 10):
"""Show trending events by volume."""
events = _get(f"{GAMMA}/events?limit={limit}&active=true&closed=false&order=volume&ascending=false")
print(f"Top {len(events)} trending events:\n")
for i, evt in enumerate(events, 1):
print(f"{i}. {evt['title']}")
print(f" Volume: {_fmt_volume(evt.get('volume', 0))} | Markets: {len(evt.get('markets', []))}")
print(f" slug: {evt.get('slug', '')}")
markets = evt.get("markets", [])
for m in markets[:3]:
_print_market(m, indent=" ")
if len(markets) > 3:
print(f" ... and {len(markets) - 3} more markets")
print()
def cmd_market(slug: str):
"""Get market details by slug."""
markets = _get(f"{GAMMA}/markets?slug={urllib.parse.quote(slug)}")
if not markets:
print(f"No market found with slug: {slug}")
return
m = markets[0]
print(f"Market: {m.get('question', '?')}")
print(f"Status: {'CLOSED' if m.get('closed') else 'ACTIVE'}")
_print_market(m)
print(f"\n conditionId: {m.get('conditionId', 'N/A')}")
tokens = _parse_json_field(m.get("clobTokenIds", "[]"))
if isinstance(tokens, list):
outcomes = _parse_json_field(m.get("outcomes", "[]"))
for i, t in enumerate(tokens):
label = outcomes[i] if isinstance(outcomes, list) and i < len(outcomes) else f"Outcome {i}"
print(f" token ({label}): {t}")
desc = m.get("description", "")
if desc:
print(f"\n Description: {desc[:500]}")
def cmd_event(slug: str):
"""Get event details by slug."""
events = _get(f"{GAMMA}/events?slug={urllib.parse.quote(slug)}")
if not events:
print(f"No event found with slug: {slug}")
return
evt = events[0]
print(f"Event: {evt['title']}")
print(f"Volume: {_fmt_volume(evt.get('volume', 0))}")
print(f"Status: {'CLOSED' if evt.get('closed') else 'ACTIVE'}")
print(f"Markets: {len(evt.get('markets', []))}\n")
for m in evt.get("markets", []):
_print_market(m, indent=" ")
print()
def cmd_price(token_id: str):
"""Get current price for a token."""
buy = _get(f"{CLOB}/price?token_id={token_id}&side=buy")
mid = _get(f"{CLOB}/midpoint?token_id={token_id}")
spread = _get(f"{CLOB}/spread?token_id={token_id}")
print(f"Token: {token_id[:30]}...")
print(f" Buy price: {_fmt_pct(buy.get('price', '?'))}")
print(f" Midpoint: {_fmt_pct(mid.get('mid', '?'))}")
print(f" Spread: {spread.get('spread', '?')}")
def cmd_book(token_id: str):
"""Get orderbook for a token."""
book = _get(f"{CLOB}/book?token_id={token_id}")
bids = book.get("bids", [])
asks = book.get("asks", [])
last = book.get("last_trade_price", "?")
print(f"Orderbook for {token_id[:30]}...")
print(f"Last trade: {_fmt_pct(last)} | Tick size: {book.get('tick_size', '?')}")
print(f"\n Top bids ({len(bids)} total):")
# Show bids sorted by price descending (best bids first)
sorted_bids = sorted(bids, key=lambda x: float(x.get("price", 0)), reverse=True)
for b in sorted_bids[:10]:
print(f" {_fmt_pct(b['price']):>7} | Size: {float(b['size']):>10.2f}")
print(f"\n Top asks ({len(asks)} total):")
sorted_asks = sorted(asks, key=lambda x: float(x.get("price", 0)))
for a in sorted_asks[:10]:
print(f" {_fmt_pct(a['price']):>7} | Size: {float(a['size']):>10.2f}")
def cmd_history(condition_id: str, interval: str = "all", fidelity: int = 50):
"""Get price history for a market."""
data = _get(f"{CLOB}/prices-history?market={condition_id}&interval={interval}&fidelity={fidelity}")
history = data.get("history", [])
if not history:
print("No price history available for this market.")
return
print(f"Price history ({len(history)} points, interval={interval}):\n")
from datetime import datetime, timezone
for pt in history:
ts = datetime.fromtimestamp(pt["t"], tz=timezone.utc).strftime("%Y-%m-%d %H:%M")
price = _fmt_pct(pt["p"])
bar = "█" * int(float(pt["p"]) * 40)
print(f" {ts} {price:>7} {bar}")
def cmd_trades(limit: int = 10, market: str = None):
"""Get recent trades."""
url = f"{DATA}/trades?limit={limit}"
if market:
url += f"&market={market}"
trades = _get(url)
if not isinstance(trades, list):
print(f"Unexpected response: {trades}")
return
print(f"Recent trades ({len(trades)}):\n")
for t in trades:
side = t.get("side", "?")
price = _fmt_pct(t.get("price", "?"))
size = t.get("size", "?")
outcome = t.get("outcome", "?")
title = t.get("title", "?")[:50]
ts = t.get("timestamp", "")
print(f" {side:4} {price:>7} x{float(size):>8.2f} [{outcome}] {title}")
def main():
args = sys.argv[1:]
if not args or args[0] in {"-h", "--help", "help"}:
print(__doc__)
return
cmd = args[0]
if cmd == "search" and len(args) >= 2:
cmd_search(" ".join(args[1:]))
elif cmd == "trending":
limit = 10
if "--limit" in args:
idx = args.index("--limit")
limit = int(args[idx + 1]) if idx + 1 < len(args) else 10
cmd_trending(limit)
elif cmd == "market" and len(args) >= 2:
cmd_market(args[1])
elif cmd == "event" and len(args) >= 2:
cmd_event(args[1])
elif cmd == "price" and len(args) >= 2:
cmd_price(args[1])
elif cmd == "book" and len(args) >= 2:
cmd_book(args[1])
elif cmd == "history" and len(args) >= 2:
interval = "all"
fidelity = 50
if "--interval" in args:
idx = args.index("--interval")
interval = args[idx + 1] if idx + 1 < len(args) else "all"
if "--fidelity" in args:
idx = args.index("--fidelity")
fidelity = int(args[idx + 1]) if idx + 1 < len(args) else 50
cmd_history(args[1], interval, fidelity)
elif cmd == "trades":
limit = 10
market = None
if "--limit" in args:
idx = args.index("--limit")
limit = int(args[idx + 1]) if idx + 1 < len(args) else 10
if "--market" in args:
idx = args.index("--market")
market = args[idx + 1] if idx + 1 < len(args) else None
cmd_trades(limit, market)
else:
print(f"Unknown command: {cmd}")
print(__doc__)
if __name__ == "__main__":
main()