Initial commit: 25 scripts Python sauvegardés
Date: 2026-08-28 11:43:34 EDT
This commit is contained in:
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""List today's events from ALL Google Calendars (not just primary).
|
||||
|
||||
Usage:
|
||||
python3 agenda_all_calendars.py [YYYYMMDD] # default: today
|
||||
|
||||
Outputs JSON array of events with added 'calendar' field.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Paths
|
||||
# ----------------------------------------------------------------
|
||||
HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")))
|
||||
TOKEN_PATH = HERMES_HOME / "google_token.json"
|
||||
CLIENT_SECRET_PATH = HERMES_HOME / "google_client_secret.json"
|
||||
|
||||
SCOPES = [
|
||||
"https://www.googleapis.com/auth/calendar",
|
||||
"https://www.googleapis.com/auth/gmail.readonly",
|
||||
"https://www.googleapis.com/auth/gmail.send",
|
||||
"https://www.googleapis.com/auth/gmail.modify",
|
||||
"https://www.googleapis.com/auth/drive",
|
||||
"https://www.googleapis.com/auth/contacts.readonly",
|
||||
"https://www.googleapis.com/auth/spreadsheets",
|
||||
"https://www.googleapis.com/auth/documents",
|
||||
]
|
||||
|
||||
|
||||
def get_credentials():
|
||||
"""Load and refresh credentials."""
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
if not TOKEN_PATH.exists():
|
||||
print("[]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH), SCOPES)
|
||||
if creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
# Save refreshed token
|
||||
TOKEN_PATH.write_text(
|
||||
json.dumps(
|
||||
{**json.loads(creds.to_json()), "type": "authorized_user"},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return creds
|
||||
|
||||
|
||||
def get_all_calendar_events(date_id_str=None):
|
||||
"""
|
||||
Discover all calendars and get today's events from each.
|
||||
Returns list of events with 'calendar' field added.
|
||||
"""
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
if not date_id_str:
|
||||
date_id_str = datetime.now().strftime("%Y%m%d")
|
||||
|
||||
dt = datetime.strptime(date_id_str, "%Y%m%d")
|
||||
# Use Toronto timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
tz = ZoneInfo("America/Toronto")
|
||||
|
||||
# Start of day in Toronto time
|
||||
start_dt = datetime(dt.year, dt.month, dt.day, tzinfo=tz)
|
||||
end_dt = start_dt + timedelta(days=1)
|
||||
|
||||
time_min = start_dt.isoformat()
|
||||
time_max = end_dt.isoformat()
|
||||
|
||||
creds = get_credentials()
|
||||
service = build("calendar", "v3", credentials=creds)
|
||||
|
||||
# 1. Discover all calendars
|
||||
cal_list = service.calendarList().list().execute()
|
||||
calendars = cal_list.get("items", [])
|
||||
|
||||
all_events = []
|
||||
for cal in calendars:
|
||||
cal_id = cal["id"]
|
||||
cal_name = cal.get("summary", cal_id)
|
||||
cal_selected = cal.get("selected", True) # user has this calendar visible
|
||||
|
||||
if not cal_selected:
|
||||
continue # skip hidden calendars
|
||||
|
||||
try:
|
||||
result = service.events().list(
|
||||
calendarId=cal_id,
|
||||
timeMin=time_min,
|
||||
timeMax=time_max,
|
||||
singleEvents=True,
|
||||
orderBy="startTime",
|
||||
maxResults=50,
|
||||
).execute()
|
||||
except Exception:
|
||||
continue # skip inaccessible calendars
|
||||
|
||||
for ev in result.get("items", []):
|
||||
ev["calendar"] = cal_name # add calendar name
|
||||
all_events.append({
|
||||
"id": ev["id"],
|
||||
"summary": ev.get("summary", "(no title)"),
|
||||
"start": ev.get("start", {}).get("dateTime", ev.get("start", {}).get("date", "")),
|
||||
"end": ev.get("end", {}).get("dateTime", ev.get("end", {}).get("date", "")),
|
||||
"location": ev.get("location", ""),
|
||||
"description": ev.get("description", ""),
|
||||
"calendar": cal_name,
|
||||
})
|
||||
|
||||
return all_events
|
||||
|
||||
|
||||
def main():
|
||||
date_id_str = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
events = get_all_calendar_events(date_id_str)
|
||||
print(json.dumps(events, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,972 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a daily briefing markdown file and print the ObsiGate link.
|
||||
|
||||
Usage:
|
||||
python3 generate_briefing.py # today's briefing
|
||||
python3 generate_briefing.py 20260520 # specific date (YYYYMMDD)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen, Request
|
||||
from urllib.error import URLError
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Config
|
||||
# ----------------------------------------------------------------
|
||||
HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")))
|
||||
GAPI = HERMES_HOME / "skills/productivity/google-workspace/scripts/google_api.py"
|
||||
VAULT_BASE = Path("/NFS/OBSIDIAN_DOC/Obsidian_IT")
|
||||
BRIEFING_DIR = VAULT_BASE / "05_Journal_Quotidien/5.1_Quotidien/HermesBerriPi/briefing-matin"
|
||||
OBSIGATE_BASE = "https://og.dracodev.net/popout/IT"
|
||||
|
||||
# Beloeil, QC
|
||||
LAT, LON = "45.5667", "-73.2"
|
||||
|
||||
# Calendar emoji map (by summary keyword)
|
||||
CAL_EMOJI = {
|
||||
"famille": "👪",
|
||||
"travail": "💼",
|
||||
"desjardins": "💼",
|
||||
"finances": "💰",
|
||||
"paie": "💰",
|
||||
}
|
||||
|
||||
# Email action prefixes (from GitHub notification subjects)
|
||||
ACTION_PREFIXES = {
|
||||
"PR #": "🔀",
|
||||
"Issue #": "💬",
|
||||
"Merged #": "✅",
|
||||
"pushé": "📌",
|
||||
"commit": "📌",
|
||||
}
|
||||
|
||||
# Weather WMO codes
|
||||
WMO_EMOJI = {
|
||||
0: "☀️", 1: "🌤️", 2: "⛅", 3: "☁️",
|
||||
45: "🌫️", 48: "🌫️",
|
||||
51: "🌧️", 53: "🌧️", 55: "🌧️",
|
||||
61: "🌧️", 63: "🌧️", 65: "🌧️",
|
||||
71: "❄️", 73: "❄️", 75: "❄️",
|
||||
80: "🌧️", 81: "🌧️", 82: "🌧️",
|
||||
85: "❄️", 86: "❄️",
|
||||
95: "⛈️", 96: "⛈️", 99: "⛈️",
|
||||
}
|
||||
WMO_LABELS = {
|
||||
0: "Ciel clair", 1: "Peu nuageux", 2: "Partiellement nuageux",
|
||||
3: "Nuageux", 45: "Brouillard", 48: "Brouillard givrant",
|
||||
51: "Bruine légère", 53: "Bruine modérée", 55: "Bruine dense",
|
||||
61: "Pluie légère", 63: "Pluie modérée", 65: "Pluie forte",
|
||||
71: "Neige légère", 73: "Neige modérée", 75: "Neige forte",
|
||||
80: "Averses légères", 81: "Averses modérées", 82: "Averses violentes",
|
||||
85: "Neige légère", 86: "Neige forte",
|
||||
95: "Orage", 96: "Orage + grêle légère", 99: "Orage + grêle forte",
|
||||
}
|
||||
|
||||
|
||||
def shell(cmd, **kwargs):
|
||||
"""Run a shell command and return stdout, exit_code."""
|
||||
result = subprocess.run(
|
||||
cmd, shell=True, capture_output=True, text=True,
|
||||
timeout=kwargs.get("timeout", 30)
|
||||
)
|
||||
return result.stdout.strip(), result.returncode
|
||||
|
||||
|
||||
# ================================================================
|
||||
# Section generators
|
||||
# ================================================================
|
||||
|
||||
def get_date(date_id=None):
|
||||
"""Return (date_id_yyyymmdd, day_fr, date_fr, jour_complet)."""
|
||||
if date_id:
|
||||
dt = datetime.strptime(date_id, "%Y%m%d")
|
||||
else:
|
||||
dt = datetime.now()
|
||||
|
||||
jours_fr = ["lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi", "dimanche"]
|
||||
mois_fr = ["janvier", "février", "mars", "avril", "mai", "juin",
|
||||
"juillet", "août", "septembre", "octobre", "novembre", "décembre"]
|
||||
|
||||
date_id_str = dt.strftime("%Y%m%d")
|
||||
jour = jours_fr[dt.weekday()]
|
||||
date_str = f"{jour} {dt.day} {mois_fr[dt.month-1]} {dt.year}"
|
||||
return date_id_str, jour.capitalize(), date_str, f"{jour.capitalize()} {dt.day} {mois_fr[dt.month-1]} {dt.year}"
|
||||
|
||||
|
||||
def get_citation():
|
||||
"""Fetch random quote from zenquotes.io."""
|
||||
try:
|
||||
out, _ = shell("curl -s --max-time 5 'https://zenquotes.io/api/random'")
|
||||
data = json.loads(out)
|
||||
if data and isinstance(data, list):
|
||||
q = data[0]["q"]
|
||||
a = data[0]["a"]
|
||||
return q, a
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback
|
||||
return (
|
||||
"La meilleure façon de prédire l'avenir est de le créer.",
|
||||
"Peter Drucker"
|
||||
)
|
||||
|
||||
|
||||
def get_emails(max_count=5):
|
||||
"""Fetch unread emails and format as markdown table rows."""
|
||||
try:
|
||||
out, _ = shell(
|
||||
f"cd {GAPI.parent} && python3 {GAPI.name} gmail search 'is:unread' --max {max_count} 2>&1"
|
||||
)
|
||||
data = json.loads(out)
|
||||
if not data:
|
||||
return None, 0
|
||||
|
||||
rows = []
|
||||
for i, email in enumerate(data, 1):
|
||||
sender = email.get("from", "Inconnu")
|
||||
# Clean sender: remove email address, keep name
|
||||
if "<" in sender:
|
||||
sender = sender.split("<")[0].strip().strip('"')
|
||||
|
||||
subject = email.get("subject", "(Sans objet)")
|
||||
snippet = email.get("snippet", "")[:100]
|
||||
|
||||
# Action prefix
|
||||
prefix = ""
|
||||
subj_lower = subject.lower()
|
||||
snippet_lower = snippet.lower()
|
||||
if "pr #" in subj_lower:
|
||||
if "merged" in snippet_lower:
|
||||
prefix = "✅ "
|
||||
else:
|
||||
prefix = "🔀 "
|
||||
elif "issue #" in subj_lower:
|
||||
prefix = "💬 "
|
||||
elif "push" in snippet_lower or "commit" in snippet_lower:
|
||||
prefix = "📌 "
|
||||
|
||||
rows.append(f"| {i} | **{sender[:30]}** | `{subject[:50]}` | {prefix}{snippet[:80]} |")
|
||||
|
||||
return rows, len(data)
|
||||
except Exception as e:
|
||||
return None, 0
|
||||
|
||||
|
||||
def get_agenda(date_id_str):
|
||||
"""Fetch today's calendar events from ALL calendars."""
|
||||
try:
|
||||
# Use the dedicated script
|
||||
script = HERMES_HOME / "skills/productivity/briefing-quotidien/scripts/agenda_all_calendars.py"
|
||||
if script.exists():
|
||||
out, _ = shell(f"python3 {script} {date_id_str} 2>&1")
|
||||
else:
|
||||
# Fallback: use google_api.py with primary + common calendars
|
||||
out, _ = shell(
|
||||
f"cd {GAPI.parent} && python3 {GAPI.name} calendar list --max 20 2>&1"
|
||||
)
|
||||
|
||||
data = json.loads(out)
|
||||
if not data:
|
||||
return None, 0
|
||||
|
||||
# Filter today's events (date_id_str = YYYYMMDD)
|
||||
today_events = []
|
||||
for ev in data:
|
||||
start = ev.get("start", "")
|
||||
if date_id_str in start.replace("-", ""):
|
||||
today_events.append(ev)
|
||||
|
||||
if not today_events:
|
||||
return None, 0
|
||||
|
||||
rows = []
|
||||
for ev in today_events:
|
||||
start = ev.get("start", "")
|
||||
summary = ev.get("summary", "(Sans titre)")
|
||||
|
||||
# Format time
|
||||
if "T" in start:
|
||||
time_part = start.split("T")[1][:5]
|
||||
time_str = f"🕐 **{time_part}**"
|
||||
else:
|
||||
time_str = "📆 Journée"
|
||||
|
||||
# Determine category emoji
|
||||
summary_lower = summary.lower()
|
||||
cat_emoji = "📅" # default
|
||||
for keyword, emoji in CAL_EMOJI.items():
|
||||
if keyword in summary_lower:
|
||||
cat_emoji = emoji
|
||||
break
|
||||
|
||||
# Also check calendar name if available
|
||||
cal_name = ev.get("calendar", "").lower()
|
||||
for keyword, emoji in CAL_EMOJI.items():
|
||||
if keyword in cal_name:
|
||||
cat_emoji = emoji
|
||||
break
|
||||
|
||||
rows.append(f"| {time_str} | {summary} | {cat_emoji} |")
|
||||
|
||||
return rows, len(today_events)
|
||||
except Exception as e:
|
||||
# If agenda_all_calendars.py doesn't exist, try direct approach
|
||||
return None, 0
|
||||
|
||||
|
||||
def get_meteo():
|
||||
"""Fetch weather for Beloeil from Open-Meteo."""
|
||||
try:
|
||||
out, _ = shell(
|
||||
f"curl -s --max-time 5 "
|
||||
f"'https://api.open-meteo.com/v1/forecast?"
|
||||
f"latitude={LAT}&longitude={LON}"
|
||||
f"¤t=temperature_2m,relative_humidity_2m,apparent_temperature,wind_speed_10m,weather_code"
|
||||
f"&daily=temperature_2m_max,temperature_2m_min"
|
||||
f"&timezone=America%2FToronto&forecast_days=3'"
|
||||
)
|
||||
data = json.loads(out)
|
||||
|
||||
current = data.get("current", {})
|
||||
daily = data.get("daily", {})
|
||||
|
||||
wmo = current.get("weather_code", 0)
|
||||
emoji = WMO_EMOJI.get(wmo, "❓")
|
||||
label = WMO_LABELS.get(wmo, "Inconnu")
|
||||
temp = current.get("temperature_2m", "?")
|
||||
feels = current.get("apparent_temperature", "?")
|
||||
wind = current.get("wind_speed_10m", "?")
|
||||
hum = current.get("relative_humidity_2m", "?")
|
||||
|
||||
# Current table
|
||||
current_row = f"| **Aujourd'hui** | {emoji} {label} | **{temp}°C** | {feels}°C | {wind} km/h | {hum}% |"
|
||||
|
||||
# Forecast table
|
||||
jours = ["lun.", "mar.", "mer.", "jeu.", "ven.", "sam.", "dim."]
|
||||
forecast_rows = []
|
||||
for i in range(1, min(3, len(daily.get("time", [])))):
|
||||
date_str = daily["time"][i]
|
||||
dt = datetime.strptime(date_str, "%Y-%m-%d")
|
||||
jour = jours[dt.weekday()]
|
||||
tmax = daily["temperature_2m_max"][i]
|
||||
tmin = daily["temperature_2m_min"][i]
|
||||
label_jour = "Demain" if i == 1 else "Après-demain"
|
||||
forecast_rows.append(f"| 📅 **{label_jour}** ({jour}) | {tmin}°C | {tmax}°C |")
|
||||
|
||||
return current_row, forecast_rows
|
||||
except Exception as e:
|
||||
return None, None
|
||||
|
||||
|
||||
def get_system():
|
||||
"""Check Hermes services and system resources."""
|
||||
# Services
|
||||
services = ["hermes-gateway", "hermes-dashboard", "hermes-workspace"]
|
||||
svc_rows = []
|
||||
svc_ok = 0
|
||||
for svc in services:
|
||||
out, rc = shell(f"systemctl --user is-active {svc} 2>/dev/null")
|
||||
if out == "active":
|
||||
svc_rows.append(f"| {svc.replace('hermes-', '').capitalize()} | 🟢 |")
|
||||
svc_ok += 1
|
||||
else:
|
||||
svc_rows.append(f"| {svc.replace('hermes-', '').capitalize()} | 🔴 |")
|
||||
|
||||
svc_status = f"🟢 {svc_ok}/3 en ligne" if svc_ok == 3 else f"⚠️ {svc_ok}/3 en ligne"
|
||||
|
||||
# Resources
|
||||
try:
|
||||
# CPU
|
||||
out, _ = shell("top -bn1 | grep 'Cpu(s)' | awk '{print $2}'")
|
||||
cpu = out.replace("%", "").strip() if out else "?"
|
||||
cpu = f"{cpu}% `us`" if cpu and cpu != "?" else "?"
|
||||
|
||||
# RAM
|
||||
out, _ = shell("free -m | awk 'NR==2{printf \"%.0f\", $3*100/$2}'")
|
||||
ram = f"{out}%" if out else "?"
|
||||
|
||||
# Disk
|
||||
out, _ = shell("df -h / | awk 'NR==2{print $5}' | sed 's/%//'")
|
||||
disk = f"{out}%" if out else "?"
|
||||
|
||||
# Uptime
|
||||
out, _ = shell("uptime -p | sed 's/up //'")
|
||||
uptime = out if out else "?"
|
||||
except Exception:
|
||||
cpu = ram = disk = uptime = "?"
|
||||
|
||||
return svc_status, svc_rows, cpu, ram, disk, uptime
|
||||
|
||||
|
||||
def get_docker_services():
|
||||
"""List running Docker containers with status and uptime."""
|
||||
try:
|
||||
out, rc = shell("docker ps --format '{{.Names}}\t{{.Status}}\t{{.Image}}' 2>/dev/null")
|
||||
if rc != 0 or not out:
|
||||
return None, 0
|
||||
|
||||
containers = []
|
||||
for line in out.split("\n"):
|
||||
parts = line.split("\t")
|
||||
if len(parts) >= 2:
|
||||
name = parts[0]
|
||||
status = parts[1]
|
||||
# Parse status: "Up 2 hours" or "Up 3 days (healthy)"
|
||||
is_healthy = "(healthy)" in status or "(unhealthy)" not in status
|
||||
is_up = status.startswith("Up")
|
||||
emoji = "🟢" if (is_up and is_healthy) else ("🟡" if is_up else "🔴")
|
||||
containers.append((name, emoji, status))
|
||||
|
||||
return containers, len(containers)
|
||||
except Exception:
|
||||
return None, 0
|
||||
|
||||
|
||||
def get_docker_stats():
|
||||
"""Get resource usage stats for Docker containers."""
|
||||
try:
|
||||
out, rc = shell(
|
||||
"docker stats --no-stream --format "
|
||||
"'{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}\t{{.MemUsage}}\t{{.NetIO}}' 2>/dev/null"
|
||||
)
|
||||
if rc != 0 or not out:
|
||||
return None
|
||||
|
||||
stats = []
|
||||
for line in out.split("\n"):
|
||||
parts = line.split("\t")
|
||||
if len(parts) >= 4:
|
||||
# Clean up: remove spaces in percentages, trim
|
||||
name = parts[0]
|
||||
cpu = parts[1].strip()
|
||||
mem_pct = parts[2].strip()
|
||||
# NetIO: "1.2MB / 3.4MB" → take IN only
|
||||
net = parts[4].strip() if len(parts) >= 5 else "—"
|
||||
net_in = net.split(" / ")[0] if " / " in net else net
|
||||
stats.append((name, cpu, mem_pct, net_in))
|
||||
|
||||
return stats
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_honcho_stats():
|
||||
"""Query Honcho API for workspace statistics."""
|
||||
try:
|
||||
import urllib.request
|
||||
|
||||
BASE = "http://localhost:8000/v3"
|
||||
WS = "hermes"
|
||||
|
||||
def post_json(path, data=None):
|
||||
body = json.dumps(data or {}).encode()
|
||||
req = urllib.request.Request(f"{BASE}{path}", data=body, method="POST",
|
||||
headers={"Content-Type": "application/json"})
|
||||
resp = urllib.request.urlopen(req, timeout=3)
|
||||
return json.loads(resp.read())
|
||||
|
||||
# Health check
|
||||
try:
|
||||
req = urllib.request.urlopen("http://localhost:8000/health", timeout=3)
|
||||
health = json.loads(req.read())
|
||||
api_ok = health.get("status") == "ok"
|
||||
except Exception:
|
||||
api_ok = False
|
||||
|
||||
# Peers
|
||||
try:
|
||||
data = post_json(f"/workspaces/{WS}/peers/list")
|
||||
items = data.get("items", data)
|
||||
nb_peers = len(items) if isinstance(items, list) else "?"
|
||||
except Exception:
|
||||
nb_peers = "?"
|
||||
|
||||
# Sessions
|
||||
try:
|
||||
data = post_json(f"/workspaces/{WS}/sessions/list")
|
||||
items = data.get("items", data)
|
||||
nb_sessions = len(items) if isinstance(items, list) else "?"
|
||||
except Exception:
|
||||
nb_sessions = "?"
|
||||
|
||||
# Conclusions (observations)
|
||||
try:
|
||||
data = post_json(f"/workspaces/{WS}/conclusions/list")
|
||||
items = data.get("items", data)
|
||||
nb_conclusions = len(items) if isinstance(items, list) else "?"
|
||||
except Exception:
|
||||
nb_conclusions = "?"
|
||||
|
||||
return {
|
||||
"api": api_ok,
|
||||
"peers": nb_peers,
|
||||
"sessions": nb_sessions,
|
||||
"conclusions": nb_conclusions,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_priorities():
|
||||
"""Read priorities file if it exists."""
|
||||
priorities_file = Path.home() / ".hermes" / "priorities.txt"
|
||||
if priorities_file.exists():
|
||||
return priorities_file.read_text().strip()
|
||||
return None
|
||||
|
||||
|
||||
def get_news_freshrss():
|
||||
"""Collect news from FreshRSS via Fever API, grouped by FreshRSS categories.
|
||||
Returns dict: {category_name: [(title, url, source, cat_emoji), ...]}
|
||||
Falls back to None if FreshRSS is unavailable."""
|
||||
try:
|
||||
FRESHRSS_API = "http://localhost:8088/api/fever.php"
|
||||
API_KEY = "c2cc2566601a7b5c6f28fcc8268786d4" # md5("admin:freshrss2026")
|
||||
|
||||
# 1. Get all unread item IDs
|
||||
data = f"api&email=admin&api_key={API_KEY}&unread_item_ids".encode()
|
||||
req = Request(FRESHRSS_API, data=data)
|
||||
with urlopen(req, timeout=10) as resp:
|
||||
result = json.loads(resp.read())
|
||||
unread_ids = result.get("unread_item_ids", "")
|
||||
if not unread_ids:
|
||||
return None
|
||||
all_ids = unread_ids.split(",")
|
||||
|
||||
# Sample evenly across the full ID range to hit all feeds/categories.
|
||||
# FreshRSS returns IDs sorted by date (newest first), so taking only
|
||||
# the first N would capture only the most active feed (usually International).
|
||||
# By sampling every Nth ID we span the full time range and get all categories.
|
||||
MAX_SAMPLE = 200
|
||||
step = max(1, len(all_ids) // MAX_SAMPLE)
|
||||
sampled_ids = all_ids[::step][:MAX_SAMPLE]
|
||||
|
||||
# 2. Get feeds and groups first (to know which feeds exist)
|
||||
data = f"api&email=admin&api_key={API_KEY}&feeds&groups".encode()
|
||||
req = Request(FRESHRSS_API, data=data)
|
||||
with urlopen(req, timeout=10) as resp:
|
||||
result = json.loads(resp.read())
|
||||
|
||||
feeds = {f["id"]: f for f in result.get("feeds", [])}
|
||||
groups = {g["id"]: g["title"] for g in result.get("groups", [])}
|
||||
feeds_groups = {}
|
||||
for fg in result.get("feeds_groups", []):
|
||||
for fid in fg["feed_ids"].split(","):
|
||||
fid_int = int(fid) if fid.isdigit() else fid
|
||||
feeds_groups[fid_int] = fg["group_id"]
|
||||
|
||||
# 3. Get items for sampled IDs
|
||||
data = f"api&email=admin&api_key={API_KEY}&items&with_ids={','.join(sampled_ids)}".encode()
|
||||
req = Request(FRESHRSS_API, data=data)
|
||||
with urlopen(req, timeout=10) as resp:
|
||||
result = json.loads(resp.read())
|
||||
items = result.get("items", [])
|
||||
|
||||
# Category emoji mapping
|
||||
CAT_EMOJI = {
|
||||
"International": "🌍",
|
||||
"Regional": "🇨🇦",
|
||||
"Sciences": "🔬",
|
||||
"Technologies": "💻",
|
||||
"Développement": "⚙️",
|
||||
"Intelligence Artificielle": "🤖",
|
||||
}
|
||||
MAX_PER_CATEGORY = 5 # Keep briefing concise but diverse
|
||||
|
||||
# Group items by category
|
||||
cat_items = {}
|
||||
for item in items:
|
||||
fid = item.get("feed_id")
|
||||
if fid not in feeds:
|
||||
continue
|
||||
feed = feeds[fid]
|
||||
gid = feeds_groups.get(fid, 1)
|
||||
cat_name = groups.get(gid, "Sans catégorie")
|
||||
if cat_name == "Sans catégorie":
|
||||
continue # Skip uncategorized items
|
||||
emoji = CAT_EMOJI.get(cat_name, "📰")
|
||||
|
||||
if cat_name not in cat_items:
|
||||
cat_items[cat_name] = []
|
||||
if len(cat_items[cat_name]) >= MAX_PER_CATEGORY:
|
||||
continue # Limit per category
|
||||
|
||||
cat_items[cat_name].append((
|
||||
item.get("title", "")[:120],
|
||||
item.get("html_url", item.get("url", "")),
|
||||
feed.get("title", ""),
|
||||
emoji,
|
||||
))
|
||||
|
||||
return cat_items if cat_items else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_news_rss():
|
||||
"""Collect tech/AI news from RSS feeds (fallback when FreshRSS is unavailable).
|
||||
Returns list of (title, url, source, category)."""
|
||||
FEEDS = [
|
||||
("https://feeds.arstechnica.com/arstechnica/index", "Ars Technica", "🔬"),
|
||||
("https://www.theverge.com/rss/index.xml", "The Verge", "💻"),
|
||||
("https://news.mit.edu/topic/mitartificial-intelligence2-rss.xml", "MIT News", "🔬"),
|
||||
("https://techcrunch.com/feed/", "TechCrunch", "🏛️"),
|
||||
("https://www.wired.com/feed/category/security/latest/rss", "Wired Security", "🛡️"),
|
||||
]
|
||||
|
||||
items = []
|
||||
for url, source, default_cat in FEEDS:
|
||||
try:
|
||||
req = Request(url, headers={"User-Agent": "HermesBerryPi/1.0"})
|
||||
with urlopen(req, timeout=10) as resp:
|
||||
tree = ET.fromstring(resp.read())
|
||||
|
||||
# RSS 2.0
|
||||
for item_elem in tree.iter("item"):
|
||||
title = item_elem.find("title")
|
||||
link = item_elem.find("link")
|
||||
title_text = title.text.strip() if title is not None and title.text else ""
|
||||
link_text = link.text.strip() if link is not None and link.text else ""
|
||||
if title_text and link_text:
|
||||
# Catégorie basée sur mots-clés dans le titre
|
||||
title_lower = title_text.lower()
|
||||
if any(w in title_lower for w in ["ai ", "gpt", "llm", "model", "machine learning", "deep learning", "neural"]):
|
||||
cat = "🔬"
|
||||
elif any(w in title_lower for w in ["security", "hack", "vuln", "cyber", "breach", "ransom"]):
|
||||
cat = "🛡️"
|
||||
elif any(w in title_lower for w in ["lawsuit", "regulation", "eu ", "ftc", "privacy", "copyright"]):
|
||||
cat = "🏛️"
|
||||
else:
|
||||
cat = default_cat
|
||||
items.append((title_text[:120], link_text, source, cat))
|
||||
|
||||
# Atom feed
|
||||
if not items or all(i[2] != source for i in items[-5:]):
|
||||
for entry in tree.iter("{http://www.w3.org/2005/Atom}entry"):
|
||||
title = entry.find("{http://www.w3.org/2005/Atom}title")
|
||||
link = entry.find("{http://www.w3.org/2005/Atom}link")
|
||||
title_text = title.text.strip() if title is not None and title.text else ""
|
||||
link_text = link.get("href", "").strip() if link is not None else ""
|
||||
if title_text and link_text:
|
||||
title_lower = title_text.lower()
|
||||
if any(w in title_lower for w in ["ai ", "gpt", "llm", "model"]):
|
||||
cat = "🔬"
|
||||
elif any(w in title_lower for w in ["security", "hack", "vuln"]):
|
||||
cat = "🛡️"
|
||||
else:
|
||||
cat = default_cat
|
||||
items.append((title_text[:120], link_text, source, cat))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Dédupliquer par titre (similarité simple)
|
||||
seen = set()
|
||||
unique = []
|
||||
for title, url, source, cat in items:
|
||||
key = title[:50].lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append((title, url, source, cat))
|
||||
|
||||
return unique[:10] # Max 10 articles
|
||||
|
||||
|
||||
def get_honcho_advanced():
|
||||
"""Query Honcho API for detailed workspace statistics."""
|
||||
try:
|
||||
import urllib.request as ur
|
||||
BASE = "http://localhost:8000/v3"
|
||||
WS = "hermes"
|
||||
|
||||
def post_json(path, data=None):
|
||||
body = json.dumps(data or {}).encode()
|
||||
req = ur.Request(f"{BASE}{path}", data=body, method="POST",
|
||||
headers={"Content-Type": "application/json"})
|
||||
resp = ur.urlopen(req, timeout=5)
|
||||
return json.loads(resp.read())
|
||||
|
||||
stats = {}
|
||||
|
||||
# Health
|
||||
try:
|
||||
req = ur.urlopen("http://localhost:8000/health", timeout=3)
|
||||
health = json.loads(req.read())
|
||||
stats["api"] = health.get("status") == "ok"
|
||||
ver = health.get("version", "")
|
||||
if not ver:
|
||||
# Try /v3/workspaces/{WS} for version info
|
||||
try:
|
||||
ws_data = post_json(f"/workspaces/{WS}")
|
||||
ver = ws_data.get("version", "")
|
||||
except Exception:
|
||||
pass
|
||||
stats["version"] = ver if ver else "?"
|
||||
except Exception:
|
||||
stats["api"] = False
|
||||
stats["version"] = "?"
|
||||
|
||||
# Peers
|
||||
try:
|
||||
data = post_json(f"/workspaces/{WS}/peers/list")
|
||||
items = data.get("items", data)
|
||||
stats["peers"] = len(items) if isinstance(items, list) else "?"
|
||||
except Exception:
|
||||
stats["peers"] = "?"
|
||||
|
||||
# Sessions
|
||||
try:
|
||||
data = post_json(f"/workspaces/{WS}/sessions/list")
|
||||
items = data.get("items", data)
|
||||
stats["sessions"] = len(items) if isinstance(items, list) else "?"
|
||||
|
||||
# Sessions actives (7 derniers jours)
|
||||
cutoff = (datetime.now() - timedelta(days=7)).isoformat()
|
||||
recent = 0
|
||||
if isinstance(items, list):
|
||||
for s in items:
|
||||
created = s.get("created_at", s.get("created", ""))
|
||||
if created and created > cutoff:
|
||||
recent += 1
|
||||
stats["sessions_recent"] = recent
|
||||
except Exception:
|
||||
stats["sessions"] = "?"
|
||||
stats["sessions_recent"] = "?"
|
||||
|
||||
# Conclusions
|
||||
try:
|
||||
data = post_json(f"/workspaces/{WS}/conclusions/list")
|
||||
items = data.get("items", data)
|
||||
stats["conclusions"] = len(items) if isinstance(items, list) else "?"
|
||||
except Exception:
|
||||
stats["conclusions"] = "?"
|
||||
|
||||
# DB size
|
||||
try:
|
||||
out, _ = shell("du -sh /home/bruno/workspace/honcho/ 2>/dev/null | cut -f1")
|
||||
stats["db_size"] = out if out else "?"
|
||||
except Exception:
|
||||
stats["db_size"] = "?"
|
||||
|
||||
return stats
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_system_advanced():
|
||||
"""Get advanced system metrics (temperature, load, I/O)."""
|
||||
metrics = {}
|
||||
|
||||
# Température CPU (Raspberry Pi)
|
||||
try:
|
||||
out, rc = shell("vcgencmd measure_temp 2>/dev/null")
|
||||
if rc == 0 and out:
|
||||
metrics["cpu_temp"] = out.replace("temp=", "").replace("'C", "°C")
|
||||
else:
|
||||
metrics["cpu_temp"] = "?"
|
||||
except Exception:
|
||||
metrics["cpu_temp"] = "?"
|
||||
|
||||
# Load average
|
||||
try:
|
||||
out, rc = shell("cat /proc/loadavg 2>/dev/null | awk '{print $1, $2, $3}'")
|
||||
metrics["load"] = out if rc == 0 and out else "?"
|
||||
except Exception:
|
||||
metrics["load"] = "?"
|
||||
|
||||
# Disk I/O
|
||||
try:
|
||||
out, rc = shell("iostat -d sda 2>/dev/null | awk 'NR==4{print $3, $4}' || iostat -d mmcblk0 2>/dev/null | awk 'NR==4{print $3, $4}'")
|
||||
if rc == 0 and out:
|
||||
metrics["disk_io"] = out
|
||||
else:
|
||||
# Fallback: /proc/diskstats
|
||||
out2, _ = shell("awk '{if($3==\"mmcblk0\"||$3==\"sda\") printf \"%.1f MB/s\", $7*512/1024/1024}' /proc/diskstats 2>/dev/null")
|
||||
metrics["disk_io"] = out2 if out2 else "?"
|
||||
except Exception:
|
||||
metrics["disk_io"] = "?"
|
||||
|
||||
# Swap
|
||||
try:
|
||||
out, rc = shell("free -m | awk 'NR==3{printf \"%.0f/%.0fMB\", $3, $2}'")
|
||||
metrics["swap"] = out if rc == 0 and out else "?"
|
||||
except Exception:
|
||||
metrics["swap"] = "?"
|
||||
|
||||
# Uptime
|
||||
try:
|
||||
out, rc = shell("uptime -p | sed 's/up //'")
|
||||
metrics["uptime"] = out if rc == 0 and out else "?"
|
||||
except Exception:
|
||||
metrics["uptime"] = "?"
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def get_docker_advanced():
|
||||
"""Get detailed Docker container info with ports and health."""
|
||||
try:
|
||||
out, rc = shell("docker ps --format '{{.Names}}\\t{{.Status}}\\t{{.Image}}\\t{{.Ports}}' 2>/dev/null")
|
||||
if rc != 0 or not out:
|
||||
return None, 0
|
||||
|
||||
containers = []
|
||||
for line in out.split("\n"):
|
||||
parts = line.split("\t")
|
||||
if len(parts) >= 2:
|
||||
name = parts[0]
|
||||
status = parts[1]
|
||||
image = parts[2] if len(parts) > 2 else "?"
|
||||
ports = parts[3] if len(parts) > 3 else "—"
|
||||
|
||||
# Health status
|
||||
is_healthy = "(healthy)" in status
|
||||
is_unhealthy = "(unhealthy)" in status
|
||||
is_up = status.startswith("Up")
|
||||
|
||||
if is_up and is_healthy:
|
||||
emoji = "🟢"
|
||||
elif is_up and is_unhealthy:
|
||||
emoji = "🟡"
|
||||
elif is_up:
|
||||
emoji = "🟢" # No healthcheck = assume OK
|
||||
else:
|
||||
emoji = "🔴"
|
||||
|
||||
# Clean ports display
|
||||
if ports and ports != "—" and len(ports) > 20:
|
||||
# Take first port mapping only
|
||||
ports = ports.split(",")[0].strip()
|
||||
|
||||
containers.append((name, emoji, status, image, ports))
|
||||
|
||||
return containers, len(containers)
|
||||
except Exception:
|
||||
return None, 0
|
||||
|
||||
|
||||
def build_markdown(date_id_str, jour, date_complete):
|
||||
"""Build the complete briefing markdown."""
|
||||
|
||||
parts = []
|
||||
|
||||
# ---- Header ----
|
||||
parts.append(f"# 📅 Briefing Matinal\n")
|
||||
parts.append(f"> **{date_complete}** · 🕖 07:00 · `America/Toronto`\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Citation ----
|
||||
quote, author = get_citation()
|
||||
parts.append(f"# 💡 Motivation du jour\n")
|
||||
parts.append(f"> *\"{quote}\"*\n")
|
||||
parts.append(f"> — **{author}**\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Emails ----
|
||||
parts.append(f"# 📧 Emails récents\n")
|
||||
emails, nb_emails = get_emails()
|
||||
if emails:
|
||||
parts.append(f"| # | Expéditeur | Sujet | Résumé |")
|
||||
parts.append(f"|:-:|:-----------|:------|:-------|")
|
||||
parts.extend(emails)
|
||||
parts.append("")
|
||||
else:
|
||||
parts.append("📭 Aucun email non lu.\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Agenda ----
|
||||
parts.append(f"# 📅 Agenda du jour\n")
|
||||
agenda, nb_events = get_agenda(date_id_str)
|
||||
if agenda:
|
||||
parts.append(f"| Heure | Événement | Catégorie |")
|
||||
parts.append(f"|:-----:|:----------|:---------:|")
|
||||
parts.extend(agenda)
|
||||
parts.append("")
|
||||
else:
|
||||
parts.append("📅 Aucun événement aujourd'hui.\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Météo ----
|
||||
parts.append(f"# 🌤️ Météo — Beloeil\n")
|
||||
current_row, forecast_rows = get_meteo()
|
||||
if current_row:
|
||||
parts.append(f"| | Condition | Temp. | Ressenti | Vent | Humidité |")
|
||||
parts.append(f"|:-:|:---------:|:-----:|:--------:|:----:|:--------:|")
|
||||
parts.append(current_row)
|
||||
parts.append("")
|
||||
|
||||
if forecast_rows:
|
||||
parts.append(f"| | 🌡️ Min | 🌡️ Max |")
|
||||
parts.append(f"|:-:|:------:|:------:|")
|
||||
parts.extend(forecast_rows)
|
||||
parts.append("")
|
||||
else:
|
||||
parts.append("🌤️ Météo non disponible.\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- System ----
|
||||
parts.append(f"# 🖥️ HermesBerryPi\n")
|
||||
svc_status, svc_rows, cpu, ram, disk, uptime = get_system()
|
||||
sys_adv = get_system_advanced()
|
||||
parts.append(f"Services — {svc_status}\n")
|
||||
parts.append(f"| Service | Statut |")
|
||||
parts.append(f"|:--------|:------:|")
|
||||
parts.extend(svc_rows)
|
||||
parts.append("")
|
||||
parts.append(f"Ressources système\n")
|
||||
parts.append(f"| CPU | RAM | Disque | Swap | Uptime |")
|
||||
parts.append(f"|:---:|:---:|:------:|:----:|:------:|")
|
||||
parts.append(f"| {cpu} | {ram} | {disk} | {sys_adv.get('swap', '?')} | {sys_adv.get('uptime', uptime)} |")
|
||||
parts.append("")
|
||||
parts.append(f"| 🌡️ Temp. CPU | 📊 Load (1/5/15) | 💾 I/O Disque |")
|
||||
parts.append(f"|:-------------:|:-----------------:|:-------------:|")
|
||||
parts.append(f"| {sys_adv.get('cpu_temp', '?')} | {sys_adv.get('load', '?')} | {sys_adv.get('disk_io', '?')} |")
|
||||
parts.append("")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Docker ----
|
||||
parts.append(f"# 🐳 Docker\n")
|
||||
containers, nb_containers = get_docker_advanced()
|
||||
if containers:
|
||||
stats = get_docker_stats() or []
|
||||
stat_map = {s[0]: s for s in stats}
|
||||
|
||||
up_count = sum(1 for _, e, _, _, _ in containers if e == "🟢")
|
||||
parts.append(f"Services — 🟢 {up_count}/{nb_containers} conteneurs actifs\n")
|
||||
parts.append(f"| Conteneur | Image | Statut | CPU | RAM | Réseau IN | Ports |")
|
||||
parts.append(f"|:----------|:------|:------:|:---:|:---:|:----------:|:------|")
|
||||
for name, emoji, status, image, ports in containers:
|
||||
cpu = mem = net = "—"
|
||||
if name in stat_map:
|
||||
_, cpu, mem, net = stat_map[name]
|
||||
# Shorten image name
|
||||
img_short = image.split("/")[-1] if "/" in image else image
|
||||
if ":" in img_short:
|
||||
img_short = img_short.split(":")[0]
|
||||
parts.append(f"| {name} | {img_short} | {emoji} {status} | {cpu} | {mem} | {net} | {ports} |")
|
||||
parts.append("")
|
||||
else:
|
||||
parts.append("🐳 Docker non disponible.\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Honcho ----
|
||||
parts.append(f"# 🧠 Honcho (mémoire IA)\n")
|
||||
honcho = get_honcho_advanced()
|
||||
if honcho:
|
||||
api_emoji = "🟢" if honcho["api"] else "🔴"
|
||||
parts.append(f"| Métrique | Valeur |")
|
||||
parts.append(f"|:---------|:------:|")
|
||||
parts.append(f"| API | {api_emoji} {'OK' if honcho['api'] else 'HORS-LIGNE'} |")
|
||||
parts.append(f"| Peers | {honcho['peers']} |")
|
||||
parts.append(f"| Sessions (total) | {honcho['sessions']} |")
|
||||
parts.append(f"| Sessions (7j) | {honcho.get('sessions_recent', '?')} |")
|
||||
parts.append(f"| Conclusions | {honcho['conclusions']} |")
|
||||
parts.append(f"| Stockage DB | {honcho.get('db_size', '?')} |")
|
||||
parts.append("")
|
||||
else:
|
||||
parts.append("🧠 Honcho non disponible.\n")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Priorities ----
|
||||
parts.append(f"# 🎯 Priorités du jour\n")
|
||||
priorities = get_priorities()
|
||||
if priorities:
|
||||
parts.append(priorities)
|
||||
else:
|
||||
parts.append("")
|
||||
parts.append("⚠️ Aucune priorité définie — pensez à planifier votre journée !")
|
||||
parts.append("")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Actualités RSS (FreshRSS prioritaire, fallback flux directs) ----
|
||||
parts.append(f"# 📡 Actualités RSS\n")
|
||||
parts.append("")
|
||||
|
||||
# Try FreshRSS first
|
||||
freshrss_news = get_news_freshrss()
|
||||
if freshrss_news:
|
||||
total_articles = sum(len(v) for v in freshrss_news.values())
|
||||
for cat_name, articles in freshrss_news.items():
|
||||
if not articles:
|
||||
continue
|
||||
emoji = articles[0][3] if articles else "📰"
|
||||
parts.append(f"{emoji} {cat_name}\n")
|
||||
parts.append(f"| | Titre | Source |")
|
||||
parts.append(f"|:-:|:------|:------:|")
|
||||
for title, url, source, _ in articles[:4]:
|
||||
parts.append(f"| 📰 | [**{title}**]({url}) | {source} |")
|
||||
parts.append("")
|
||||
parts.append(f"_📡 {total_articles} articles via FreshRSS ({len(freshrss_news)} catégories)_")
|
||||
else:
|
||||
# Fallback to direct RSS feeds
|
||||
news = get_news_rss()
|
||||
if news:
|
||||
# Grouper par catégorie
|
||||
categories = {}
|
||||
for title, url, source, cat in news:
|
||||
if cat not in categories:
|
||||
categories[cat] = []
|
||||
categories[cat].append((title, url, source))
|
||||
|
||||
cat_labels = {"🔬": "Recherche & Produits", "🏛️": "Juridique & Business", "🛡️": "Sécurité & Alignement", "💻": "Tech & Industrie"}
|
||||
for cat, cat_label in cat_labels.items():
|
||||
if cat in categories:
|
||||
parts.append(f"{cat} {cat_label}\n")
|
||||
parts.append(f"| | Titre | Source |")
|
||||
parts.append(f"|:-:|:------|:------:|")
|
||||
for title, url, source in categories[cat][:4]:
|
||||
parts.append(f"| 📰 | [**{title}**]({url}) | {source} |")
|
||||
parts.append("")
|
||||
|
||||
parts.append(f"_📡 {len(news)} articles collectés via flux RSS directs_")
|
||||
else:
|
||||
parts.append("📡 Aucune actualité disponible (FreshRSS et flux RSS inaccessibles).\n")
|
||||
parts.append("")
|
||||
parts.append("---\n")
|
||||
|
||||
# ---- Footer ----
|
||||
parts.append("Bonne journée! ☀️\n")
|
||||
|
||||
return "\n".join(parts), nb_emails, nb_events
|
||||
|
||||
|
||||
def main():
|
||||
date_id_str = sys.argv[1] if len(sys.argv) > 1 else datetime.now().strftime("%Y%m%d")
|
||||
date_id_str, jour, _, date_complete = get_date(date_id_str)
|
||||
|
||||
year = date_id_str[:4]
|
||||
month = date_id_str[4:6]
|
||||
|
||||
# Build markdown
|
||||
md, nb_emails, nb_events = build_markdown(date_id_str, jour, date_complete)
|
||||
|
||||
# Save
|
||||
target_dir = BRIEFING_DIR / year / month
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
filepath = target_dir / f"briefing_{date_id_str}.md"
|
||||
filepath.write_text(md)
|
||||
|
||||
# ObsiGate link
|
||||
rel_path = f"05_Journal_Quotidien/5.1_Quotidien/HermesBerriPi/briefing-matin/{year}/{month}/briefing_{date_id_str}.md"
|
||||
encoded = rel_path.replace("/", "%2F")
|
||||
link = f"{OBSIGATE_BASE}/{encoded}"
|
||||
|
||||
# Output JSON for cron
|
||||
result = {
|
||||
"date_id": date_id_str,
|
||||
"file": str(filepath),
|
||||
"link": link,
|
||||
"emails": nb_emails,
|
||||
"events": nb_events,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""List pending tasks from ALL Google Task lists.
|
||||
|
||||
Usage:
|
||||
python3 tasks_all_lists.py # all non-completed tasks
|
||||
python3 tasks_all_lists.py --all # include completed tasks
|
||||
|
||||
Outputs JSON array of tasks with added 'tasklist' field.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Paths
|
||||
# ----------------------------------------------------------------
|
||||
HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")))
|
||||
TOKEN_PATH = HERMES_HOME / "google_token.json"
|
||||
CLIENT_SECRET_PATH = HERMES_HOME / "google_client_secret.json"
|
||||
|
||||
SCOPES = [
|
||||
"https://www.googleapis.com/auth/tasks",
|
||||
"https://www.googleapis.com/auth/calendar",
|
||||
"https://www.googleapis.com/auth/gmail.readonly",
|
||||
"https://www.googleapis.com/auth/gmail.send",
|
||||
"https://www.googleapis.com/auth/gmail.modify",
|
||||
"https://www.googleapis.com/auth/drive",
|
||||
"https://www.googleapis.com/auth/contacts.readonly",
|
||||
"https://www.googleapis.com/auth/spreadsheets",
|
||||
"https://www.googleapis.com/auth/documents",
|
||||
]
|
||||
|
||||
|
||||
def get_credentials():
|
||||
"""Load and refresh credentials."""
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
if not TOKEN_PATH.exists():
|
||||
print("[]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH), SCOPES)
|
||||
if creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
# Save refreshed token
|
||||
TOKEN_PATH.write_text(
|
||||
json.dumps(
|
||||
{**json.loads(creds.to_json()), "type": "authorized_user"},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return creds
|
||||
|
||||
|
||||
def get_all_tasks(include_completed=False):
|
||||
"""
|
||||
Discover all task lists and get pending tasks from each.
|
||||
Returns list of tasks with 'tasklist' field added.
|
||||
"""
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
creds = get_credentials()
|
||||
service = build("tasks", "v1", credentials=creds)
|
||||
|
||||
# 1. Discover all task lists
|
||||
tl_result = service.tasklists().list(maxResults=20).execute()
|
||||
tasklists = tl_result.get("items", [])
|
||||
|
||||
all_tasks = []
|
||||
for tl in tasklists:
|
||||
tl_id = tl["id"]
|
||||
tl_name = tl.get("title", tl_id)
|
||||
|
||||
try:
|
||||
kwargs = {
|
||||
"tasklist": tl_id,
|
||||
"maxResults": 100,
|
||||
}
|
||||
if not include_completed:
|
||||
kwargs["showCompleted"] = False
|
||||
kwargs["showHidden"] = False
|
||||
|
||||
result = service.tasks().list(**kwargs).execute()
|
||||
except Exception:
|
||||
continue # skip inaccessible lists
|
||||
|
||||
for task in result.get("items", []):
|
||||
# Skip completed unless requested
|
||||
if task.get("status") == "completed" and not include_completed:
|
||||
continue
|
||||
|
||||
due = task.get("due", "")
|
||||
# Format due date nicely
|
||||
if due:
|
||||
try:
|
||||
dt = datetime.fromisoformat(due.replace("Z", "+00:00"))
|
||||
due_display = dt.strftime("%Y-%m-%d")
|
||||
# Add overdue marker
|
||||
if dt < datetime.now(timezone.utc) and task.get("status") != "completed":
|
||||
due_display = f"⚠️ {due_display}"
|
||||
except Exception:
|
||||
due_display = due[:10] if len(due) >= 10 else due
|
||||
else:
|
||||
due_display = ""
|
||||
|
||||
all_tasks.append({
|
||||
"id": task["id"],
|
||||
"title": task.get("title", "(no title)"),
|
||||
"notes": task.get("notes", ""),
|
||||
"status": task.get("status", "needsAction"),
|
||||
"due": due_display,
|
||||
"tasklist": tl_name,
|
||||
})
|
||||
|
||||
# Sort: tasks with due dates first, then by due date ascending
|
||||
def sort_key(t):
|
||||
has_due = 0 if t["due"] else 1
|
||||
due_str = t["due"].replace("⚠️ ", "") if t["due"] else "9999"
|
||||
return (has_due, due_str)
|
||||
|
||||
all_tasks.sort(key=sort_key)
|
||||
return all_tasks
|
||||
|
||||
|
||||
def main():
|
||||
include_completed = "--all" in sys.argv
|
||||
tasks = get_all_tasks(include_completed=include_completed)
|
||||
print(json.dumps(tasks, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
"""Accept all tracked changes in a DOCX file using LibreOffice.
|
||||
|
||||
Requires LibreOffice (soffice) to be installed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from office.soffice import get_soffice_env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LIBREOFFICE_PROFILE = "/tmp/libreoffice_docx_profile"
|
||||
MACRO_DIR = f"{LIBREOFFICE_PROFILE}/user/basic/Standard"
|
||||
|
||||
ACCEPT_CHANGES_MACRO = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE script:module PUBLIC "-//OpenOffice.org//DTD OfficeDocument 1.0//EN" "module.dtd">
|
||||
<script:module xmlns:script="http://openoffice.org/2000/script" script:name="Module1" script:language="StarBasic">
|
||||
Sub AcceptAllTrackedChanges()
|
||||
Dim document As Object
|
||||
Dim dispatcher As Object
|
||||
|
||||
document = ThisComponent.CurrentController.Frame
|
||||
dispatcher = createUnoService("com.sun.star.frame.DispatchHelper")
|
||||
|
||||
dispatcher.executeDispatch(document, ".uno:AcceptAllTrackedChanges", "", 0, Array())
|
||||
ThisComponent.store()
|
||||
ThisComponent.close(True)
|
||||
End Sub
|
||||
</script:module>"""
|
||||
|
||||
|
||||
def accept_changes(
|
||||
input_file: str,
|
||||
output_file: str,
|
||||
) -> tuple[None, str]:
|
||||
input_path = Path(input_file)
|
||||
output_path = Path(output_file)
|
||||
|
||||
if not input_path.exists():
|
||||
return None, f"Error: Input file not found: {input_file}"
|
||||
|
||||
if not input_path.suffix.lower() == ".docx":
|
||||
return None, f"Error: Input file is not a DOCX file: {input_file}"
|
||||
|
||||
try:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(input_path, output_path)
|
||||
except Exception as e:
|
||||
return None, f"Error: Failed to copy input file to output location: {e}"
|
||||
|
||||
if not _setup_libreoffice_macro():
|
||||
return None, "Error: Failed to setup LibreOffice macro"
|
||||
|
||||
cmd = [
|
||||
"soffice",
|
||||
"--headless",
|
||||
f"-env:UserInstallation=file://{LIBREOFFICE_PROFILE}",
|
||||
"--norestore",
|
||||
"vnd.sun.star.script:Standard.Module1.AcceptAllTrackedChanges?language=Basic&location=application",
|
||||
str(output_path.absolute()),
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
env=get_soffice_env(),
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return (
|
||||
None,
|
||||
f"Successfully accepted all tracked changes: {input_file} -> {output_file}",
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return None, f"Error: LibreOffice failed: {result.stderr}"
|
||||
|
||||
return (
|
||||
None,
|
||||
f"Successfully accepted all tracked changes: {input_file} -> {output_file}",
|
||||
)
|
||||
|
||||
|
||||
def _setup_libreoffice_macro() -> bool:
|
||||
macro_dir = Path(MACRO_DIR)
|
||||
macro_file = macro_dir / "Module1.xba"
|
||||
|
||||
if macro_file.exists() and "AcceptAllTrackedChanges" in macro_file.read_text():
|
||||
return True
|
||||
|
||||
if not macro_dir.exists():
|
||||
subprocess.run(
|
||||
[
|
||||
"soffice",
|
||||
"--headless",
|
||||
f"-env:UserInstallation=file://{LIBREOFFICE_PROFILE}",
|
||||
"--terminate_after_init",
|
||||
],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
env=get_soffice_env(),
|
||||
)
|
||||
macro_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
macro_file.write_text(ACCEPT_CHANGES_MACRO)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to setup LibreOffice macro: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Accept all tracked changes in a DOCX file"
|
||||
)
|
||||
parser.add_argument("input_file", help="Input DOCX file with tracked changes")
|
||||
parser.add_argument(
|
||||
"output_file", help="Output DOCX file (clean, no tracked changes)"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
_, message = accept_changes(args.input_file, args.output_file)
|
||||
print(message)
|
||||
|
||||
if "Error" in message:
|
||||
raise SystemExit(1)
|
||||
Executable
+368
@@ -0,0 +1,368 @@
|
||||
"""Add comments to a DOCX document.
|
||||
|
||||
Accepts either an unpacked directory OR a .docx/.dotx file directly.
|
||||
|
||||
Usage:
|
||||
# Against an unpacked directory (writes satellite files in place)
|
||||
python comment.py unpacked/ "Comment text"
|
||||
python comment.py unpacked/ "Reply text" --parent 0
|
||||
|
||||
# Against a .docx directly (extracts, writes satellite files, rezips)
|
||||
python comment.py contract.docx "This cap is too low" -o annotated.docx
|
||||
python comment.py contract.docx "Comment" --id 5 # explicit ID
|
||||
|
||||
The comment ID is auto-assigned (max existing + 1) unless --id is given.
|
||||
Plain text is XML-escaped automatically; if you pass already-escaped text
|
||||
(e.g. &, ’) use --raw to skip escaping.
|
||||
|
||||
After running, add markers to word/document.xml so the comment is visible:
|
||||
<w:commentRangeStart w:id="N"/>
|
||||
... commented content ...
|
||||
<w:commentRangeEnd w:id="N"/>
|
||||
<w:r><w:rPr><w:rStyle w:val="CommentReference"/></w:rPr><w:commentReference w:id="N"/></w:r>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
from xml.parsers.expat import ExpatError
|
||||
from xml.sax.saxutils import escape as xml_escape
|
||||
|
||||
from office.helpers import opc_target, rezip as _rezip, safe_extract as _safe_extract
|
||||
|
||||
TEMPLATE_DIR = Path(__file__).parent / "templates"
|
||||
NS = {
|
||||
"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
|
||||
"w14": "http://schemas.microsoft.com/office/word/2010/wordml",
|
||||
"w15": "http://schemas.microsoft.com/office/word/2012/wordml",
|
||||
"w16cid": "http://schemas.microsoft.com/office/word/2016/wordml/cid",
|
||||
"w16cex": "http://schemas.microsoft.com/office/word/2018/wordml/cex",
|
||||
}
|
||||
|
||||
COMMENT_XML = """\
|
||||
<w:comment w:id="{id}" w:author="{author}" w:date="{date}" w:initials="{initials}">
|
||||
<w:p w14:paraId="{para_id}" w14:textId="77777777">
|
||||
<w:r>
|
||||
<w:rPr><w:rStyle w:val="CommentReference"/></w:rPr>
|
||||
<w:annotationRef/>
|
||||
</w:r>
|
||||
<w:r>
|
||||
<w:rPr>
|
||||
<w:color w:val="000000"/>
|
||||
<w:sz w:val="20"/>
|
||||
<w:szCs w:val="20"/>
|
||||
</w:rPr>
|
||||
<w:t xml:space="preserve">{text}</w:t>
|
||||
</w:r>
|
||||
</w:p>
|
||||
</w:comment>"""
|
||||
|
||||
COMMENT_MARKER_TEMPLATE = """
|
||||
Add to word/document.xml (markers must be direct children of w:p, never inside w:r):
|
||||
<w:commentRangeStart w:id="{cid}"/>
|
||||
<w:r>...</w:r>
|
||||
<w:commentRangeEnd w:id="{cid}"/>
|
||||
<w:r><w:rPr><w:rStyle w:val="CommentReference"/></w:rPr><w:commentReference w:id="{cid}"/></w:r>"""
|
||||
|
||||
REPLY_MARKER_TEMPLATE = """
|
||||
Nest markers inside parent {pid}'s markers (direct children of w:p, never inside w:r):
|
||||
<w:commentRangeStart w:id="{pid}"/><w:commentRangeStart w:id="{cid}"/>
|
||||
<w:r>...</w:r>
|
||||
<w:commentRangeEnd w:id="{cid}"/><w:commentRangeEnd w:id="{pid}"/>
|
||||
<w:r><w:rPr><w:rStyle w:val="CommentReference"/></w:rPr><w:commentReference w:id="{pid}"/></w:r>
|
||||
<w:r><w:rPr><w:rStyle w:val="CommentReference"/></w:rPr><w:commentReference w:id="{cid}"/></w:r>"""
|
||||
|
||||
SMART_QUOTE_ENTITIES = {
|
||||
"“": "“",
|
||||
"”": "”",
|
||||
"‘": "‘",
|
||||
"’": "’",
|
||||
}
|
||||
|
||||
|
||||
def _generate_hex_id() -> str:
|
||||
return f"{random.randint(0, 0x7FFFFFFE):08X}"
|
||||
|
||||
|
||||
def _encode_smart_quotes(text: str) -> str:
|
||||
for char, entity in SMART_QUOTE_ENTITIES.items():
|
||||
text = text.replace(char, entity)
|
||||
return text
|
||||
|
||||
|
||||
def _append_xml(xml_path: Path, root_tag: str, content: str) -> None:
|
||||
dom = defusedxml.minidom.parseString(xml_path.read_text(encoding="utf-8"))
|
||||
root = dom.getElementsByTagName(root_tag)[0]
|
||||
ns_attrs = " ".join(f'xmlns:{k}="{v}"' for k, v in NS.items())
|
||||
wrapper_dom = defusedxml.minidom.parseString(f"<root {ns_attrs}>{content}</root>")
|
||||
for child in wrapper_dom.documentElement.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
root.appendChild(dom.importNode(child, True))
|
||||
output = _encode_smart_quotes(dom.toxml(encoding="UTF-8").decode("utf-8"))
|
||||
xml_path.write_text(output, encoding="utf-8")
|
||||
|
||||
|
||||
def _find_para_id(comments_path: Path, comment_id: int) -> str | None:
|
||||
dom = defusedxml.minidom.parseString(comments_path.read_text(encoding="utf-8"))
|
||||
for c in dom.getElementsByTagName("w:comment"):
|
||||
if c.getAttribute("w:id") == str(comment_id):
|
||||
for p in c.getElementsByTagName("w:p"):
|
||||
if pid := p.getAttribute("w14:paraId"):
|
||||
return pid
|
||||
return None
|
||||
|
||||
|
||||
def _next_comment_id(comments_path: Path) -> int:
|
||||
if not comments_path.exists():
|
||||
return 0
|
||||
dom = defusedxml.minidom.parseString(comments_path.read_text(encoding="utf-8"))
|
||||
ids = []
|
||||
for c in dom.getElementsByTagName("w:comment"):
|
||||
try:
|
||||
ids.append(int(c.getAttribute("w:id")))
|
||||
except ValueError:
|
||||
pass
|
||||
return (max(ids) + 1) if ids else 0
|
||||
|
||||
|
||||
def _get_next_rid(rels_path: Path) -> int:
|
||||
dom = defusedxml.minidom.parseString(rels_path.read_text(encoding="utf-8"))
|
||||
max_rid = 0
|
||||
for rel in dom.getElementsByTagName("Relationship"):
|
||||
rid = rel.getAttribute("Id")
|
||||
if rid and rid.startswith("rId"):
|
||||
try:
|
||||
max_rid = max(max_rid, int(rid[3:]))
|
||||
except ValueError:
|
||||
pass
|
||||
return max_rid + 1
|
||||
|
||||
|
||||
def _has_relationship(rels_path: Path, target: str) -> bool:
|
||||
dom = defusedxml.minidom.parseString(rels_path.read_text(encoding="utf-8"))
|
||||
return any(
|
||||
rel.getAttribute("Target") == target
|
||||
for rel in dom.getElementsByTagName("Relationship")
|
||||
)
|
||||
|
||||
|
||||
def _has_content_type(ct_path: Path, part_name: str) -> bool:
|
||||
dom = defusedxml.minidom.parseString(ct_path.read_text(encoding="utf-8"))
|
||||
return any(
|
||||
o.getAttribute("PartName") == part_name
|
||||
for o in dom.getElementsByTagName("Override")
|
||||
)
|
||||
|
||||
|
||||
_COMMENT_RELS = [
|
||||
("http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments", "comments.xml"),
|
||||
("http://schemas.microsoft.com/office/2011/relationships/commentsExtended", "commentsExtended.xml"),
|
||||
("http://schemas.microsoft.com/office/2016/09/relationships/commentsIds", "commentsIds.xml"),
|
||||
("http://schemas.microsoft.com/office/2018/08/relationships/commentsExtensible", "commentsExtensible.xml"),
|
||||
]
|
||||
_COMMENT_OVERRIDES = [
|
||||
("/word/comments.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml"),
|
||||
("/word/commentsExtended.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtended+xml"),
|
||||
("/word/commentsIds.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsIds+xml"),
|
||||
("/word/commentsExtensible.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtensible+xml"),
|
||||
]
|
||||
|
||||
|
||||
def _ensure_comment_relationships(unpacked_dir: Path) -> None:
|
||||
rels_path = unpacked_dir / "word" / "_rels" / "document.xml.rels"
|
||||
if not rels_path.exists():
|
||||
return
|
||||
dom = defusedxml.minidom.parseString(rels_path.read_text(encoding="utf-8"))
|
||||
root = dom.documentElement
|
||||
comment_types = {rel_type for rel_type, _ in _COMMENT_RELS}
|
||||
existing = set()
|
||||
for rel in dom.getElementsByTagName("Relationship"):
|
||||
if rel.getAttribute("Type") not in comment_types:
|
||||
continue
|
||||
part = opc_target(
|
||||
rel.getAttribute("Target"),
|
||||
"word/document.xml",
|
||||
rel.getAttribute("TargetMode"),
|
||||
)
|
||||
if part is not None:
|
||||
existing.add(part)
|
||||
next_rid = _get_next_rid(rels_path)
|
||||
changed = False
|
||||
for rel_type, target in _COMMENT_RELS:
|
||||
if opc_target(target, "word/document.xml") in existing:
|
||||
continue
|
||||
rel = dom.createElement("Relationship")
|
||||
rel.setAttribute("Id", f"rId{next_rid}")
|
||||
rel.setAttribute("Type", rel_type)
|
||||
rel.setAttribute("Target", target)
|
||||
root.appendChild(rel)
|
||||
next_rid += 1
|
||||
changed = True
|
||||
if changed:
|
||||
rels_path.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
|
||||
|
||||
def _ensure_comment_content_types(unpacked_dir: Path) -> None:
|
||||
ct_path = unpacked_dir / "[Content_Types].xml"
|
||||
if not ct_path.exists():
|
||||
return
|
||||
dom = defusedxml.minidom.parseString(ct_path.read_text(encoding="utf-8"))
|
||||
root = dom.documentElement
|
||||
existing = {
|
||||
o.getAttribute("PartName")
|
||||
for o in dom.getElementsByTagName("Override")
|
||||
}
|
||||
changed = False
|
||||
for part_name, content_type in _COMMENT_OVERRIDES:
|
||||
if part_name in existing:
|
||||
continue
|
||||
override = dom.createElement("Override")
|
||||
override.setAttribute("PartName", part_name)
|
||||
override.setAttribute("ContentType", content_type)
|
||||
root.appendChild(override)
|
||||
changed = True
|
||||
if changed:
|
||||
ct_path.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
|
||||
|
||||
def add_comment(
|
||||
unpacked_dir: Path | str,
|
||||
text: str,
|
||||
comment_id: int | None = None,
|
||||
author: str = "Claude",
|
||||
initials: str = "C",
|
||||
parent_id: int | None = None,
|
||||
raw: bool = False,
|
||||
) -> tuple[int, str, str]:
|
||||
unpacked_dir = Path(unpacked_dir)
|
||||
if not raw:
|
||||
text = xml_escape(text)
|
||||
author = xml_escape(author, {'"': """})
|
||||
initials = xml_escape(initials, {'"': """})
|
||||
word = unpacked_dir / "word"
|
||||
if not word.exists():
|
||||
raise FileNotFoundError(f"{word} not found (not an unpacked .docx?)")
|
||||
|
||||
comments = word / "comments.xml"
|
||||
if comment_id is None:
|
||||
comment_id = _next_comment_id(comments)
|
||||
|
||||
parent_para = None
|
||||
if parent_id is not None:
|
||||
parent_para = _find_para_id(comments, parent_id) if comments.exists() else None
|
||||
if not parent_para:
|
||||
raise ValueError(f"parent comment {parent_id} not found")
|
||||
|
||||
para_id, durable_id = _generate_hex_id(), _generate_hex_id()
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
if not comments.exists():
|
||||
shutil.copy(TEMPLATE_DIR / "comments.xml", comments)
|
||||
_ensure_comment_relationships(unpacked_dir)
|
||||
_ensure_comment_content_types(unpacked_dir)
|
||||
_append_xml(
|
||||
comments,
|
||||
"w:comments",
|
||||
COMMENT_XML.format(
|
||||
id=comment_id, author=author, date=ts, initials=initials,
|
||||
para_id=para_id, text=text,
|
||||
),
|
||||
)
|
||||
|
||||
ext = word / "commentsExtended.xml"
|
||||
if not ext.exists():
|
||||
shutil.copy(TEMPLATE_DIR / "commentsExtended.xml", ext)
|
||||
if parent_para is not None:
|
||||
_append_xml(
|
||||
ext, "w15:commentsEx",
|
||||
f'<w15:commentEx w15:paraId="{para_id}" w15:paraIdParent="{parent_para}" w15:done="0"/>',
|
||||
)
|
||||
else:
|
||||
_append_xml(
|
||||
ext, "w15:commentsEx",
|
||||
f'<w15:commentEx w15:paraId="{para_id}" w15:done="0"/>',
|
||||
)
|
||||
|
||||
ids = word / "commentsIds.xml"
|
||||
if not ids.exists():
|
||||
shutil.copy(TEMPLATE_DIR / "commentsIds.xml", ids)
|
||||
_append_xml(
|
||||
ids, "w16cid:commentsIds",
|
||||
f'<w16cid:commentId w16cid:paraId="{para_id}" w16cid:durableId="{durable_id}"/>',
|
||||
)
|
||||
|
||||
extensible = word / "commentsExtensible.xml"
|
||||
if not extensible.exists():
|
||||
shutil.copy(TEMPLATE_DIR / "commentsExtensible.xml", extensible)
|
||||
_append_xml(
|
||||
extensible, "w16cex:commentsExtensible",
|
||||
f'<w16cex:commentExtensible w16cex:durableId="{durable_id}" w16cex:dateUtc="{ts}"/>',
|
||||
)
|
||||
|
||||
action = "reply" if parent_id is not None else "comment"
|
||||
return comment_id, para_id, f"Added {action} id={comment_id} (paraId={para_id})"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description="Add a comment to a DOCX (directory or .docx file).")
|
||||
p.add_argument("input", help="Unpacked DOCX directory OR a .docx/.dotx file")
|
||||
p.add_argument("text", help="Comment text (plain text; XML-escaped automatically)")
|
||||
p.add_argument("--raw", action="store_true",
|
||||
help="Treat text as pre-escaped XML (skip automatic escaping)")
|
||||
p.add_argument("--id", type=int, dest="comment_id",
|
||||
help="Comment ID (default: auto-assign as max existing + 1)")
|
||||
p.add_argument("--author", default="Claude", help="Author name")
|
||||
p.add_argument("--initials", default="C", help="Author initials")
|
||||
p.add_argument("--parent", type=int, help="Parent comment ID (makes this a reply)")
|
||||
p.add_argument("-o", "--output",
|
||||
help="Output .docx path (only used when input is a .docx; default: overwrite input)")
|
||||
args = p.parse_args()
|
||||
|
||||
src = Path(args.input)
|
||||
|
||||
try:
|
||||
if src.is_dir():
|
||||
if args.output:
|
||||
print("Warning: --output ignored for directory input", file=sys.stderr)
|
||||
cid, _, msg = add_comment(
|
||||
src, args.text, comment_id=args.comment_id,
|
||||
author=args.author, initials=args.initials,
|
||||
parent_id=args.parent, raw=args.raw,
|
||||
)
|
||||
print(msg)
|
||||
elif src.is_file() and src.suffix.lower() in (".docx", ".dotx"):
|
||||
out = Path(args.output) if args.output else src
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
with zipfile.ZipFile(src) as zf:
|
||||
_safe_extract(zf, tmp_path)
|
||||
cid, _, msg = add_comment(
|
||||
tmp_path, args.text, comment_id=args.comment_id,
|
||||
author=args.author, initials=args.initials,
|
||||
parent_id=args.parent, raw=args.raw,
|
||||
)
|
||||
_rezip(tmp_path, out)
|
||||
print(msg)
|
||||
print(f"Wrote {out} (comment defined; add markers to word/document.xml to make it visible)")
|
||||
else:
|
||||
print(f"Error: {src} is neither a directory nor a .docx/.dotx file", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except (FileNotFoundError, ValueError, zipfile.BadZipFile, ExpatError) as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.parent is not None:
|
||||
print(REPLY_MARKER_TEMPLATE.format(pid=args.parent, cid=cid))
|
||||
else:
|
||||
print(COMMENT_MARKER_TEMPLATE.format(cid=cid))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
"""Merge adjacent identically-formatted runs in a DOCX.
|
||||
|
||||
Word fragments paragraph text across many <w:r> elements (revision ids,
|
||||
spell-check markers, editing history), which makes find-and-replace on
|
||||
word/document.xml unreliable — the string you're looking for is split
|
||||
across runs. This coalesces adjacent runs whose formatting (<w:rPr>) is
|
||||
identical, strips rsid attributes and proofErr markers, and consolidates the
|
||||
text elements — <w:t>, and <w:delText> for text inside a tracked deletion.
|
||||
|
||||
Rendering is unchanged. The text you search is what Word draws, which is not
|
||||
always the bytes in the file: an element without xml:space="preserve" has its
|
||||
edge whitespace trimmed before it reaches the page, so `<w:t>Hello </w:t>`
|
||||
followed by `<w:t>world</w:t>` reads "Helloworld" and merges to exactly that.
|
||||
|
||||
Runs in two different <w:ins>/<w:del> wrappers are never merged: that would
|
||||
rewrite tracked-change structure, collapsing separate revisions into one.
|
||||
|
||||
Only word/document.xml is processed (not headers, footers, or footnotes).
|
||||
|
||||
Usage:
|
||||
python merge_runs.py unpacked/ # after unzip, before editing
|
||||
python merge_runs.py document.docx # rewrite in place
|
||||
python merge_runs.py document.docx -o out.docx
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
|
||||
from office.helpers import XML_SPACE, rendered_text, rezip, safe_extract
|
||||
|
||||
WORDML_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
|
||||
|
||||
def merge_runs(input_dir: str) -> tuple[int, str]:
|
||||
doc_xml = Path(input_dir) / "word" / "document.xml"
|
||||
|
||||
if not doc_xml.exists():
|
||||
return 0, f"Error: {doc_xml} not found"
|
||||
|
||||
try:
|
||||
dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8"))
|
||||
root = dom.documentElement
|
||||
run_names = _run_tag_names(root)
|
||||
|
||||
_remove_elements(root, "proofErr")
|
||||
|
||||
runs = _find_runs(root, run_names)
|
||||
_strip_rsid_attrs(runs)
|
||||
|
||||
merge_count = 0
|
||||
for container in {run.parentNode for run in runs}:
|
||||
merge_count += _merge_runs_in(container, run_names)
|
||||
|
||||
doc_xml.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
return merge_count, f"Merged {merge_count} runs"
|
||||
|
||||
except Exception as e:
|
||||
return 0, f"Error: {e}"
|
||||
|
||||
|
||||
|
||||
|
||||
def _is_element(node, tag: str) -> bool:
|
||||
name = node.localName or node.tagName
|
||||
return name == tag or name.endswith(f":{tag}")
|
||||
|
||||
|
||||
def _run_tag_names(root) -> set[str]:
|
||||
names = set()
|
||||
for attr in root.attributes.values():
|
||||
if attr.value == WORDML_NS:
|
||||
if attr.name == "xmlns":
|
||||
names.add("r")
|
||||
elif attr.name.startswith("xmlns:"):
|
||||
names.add(attr.name.split(":", 1)[1] + ":r")
|
||||
return names or {"w:r", "r"}
|
||||
|
||||
|
||||
def _find_elements(root, tag: str) -> list:
|
||||
results = []
|
||||
|
||||
def traverse(node):
|
||||
if node.nodeType == node.ELEMENT_NODE:
|
||||
if _is_element(node, tag):
|
||||
results.append(node)
|
||||
for child in node.childNodes:
|
||||
traverse(child)
|
||||
|
||||
traverse(root)
|
||||
return results
|
||||
|
||||
|
||||
def _find_runs(root, run_names: set[str]) -> list:
|
||||
return [e for e in _find_elements(root, "r") if _is_run(e, run_names)]
|
||||
|
||||
|
||||
def _get_child(parent, tag: str):
|
||||
return next(iter(_get_children(parent, tag)), None)
|
||||
|
||||
|
||||
def _get_children(parent, tag: str) -> list:
|
||||
return [
|
||||
child
|
||||
for child in parent.childNodes
|
||||
if child.nodeType == child.ELEMENT_NODE and _is_element(child, tag)
|
||||
]
|
||||
|
||||
|
||||
def _is_adjacent(elem1, elem2) -> bool:
|
||||
node = elem1.nextSibling
|
||||
while node:
|
||||
if node == elem2:
|
||||
return True
|
||||
if node.nodeType == node.ELEMENT_NODE:
|
||||
return False
|
||||
if node.nodeType == node.TEXT_NODE and node.data.strip(XML_SPACE):
|
||||
return False
|
||||
node = node.nextSibling
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def _remove_elements(root, tag: str):
|
||||
for elem in _find_elements(root, tag):
|
||||
if elem.parentNode:
|
||||
elem.parentNode.removeChild(elem)
|
||||
|
||||
|
||||
def _strip_rsid_attrs(runs: list):
|
||||
for run in runs:
|
||||
for attr in list(run.attributes.values()):
|
||||
if "rsid" in attr.name.lower():
|
||||
run.removeAttribute(attr.name)
|
||||
|
||||
|
||||
|
||||
|
||||
def _merge_runs_in(container, run_names: set[str]) -> int:
|
||||
merge_count = 0
|
||||
run = _first_child_run(container, run_names)
|
||||
|
||||
while run:
|
||||
while True:
|
||||
next_elem = _next_element_sibling(run)
|
||||
if next_elem and _is_run(next_elem, run_names) and _can_merge(run, next_elem):
|
||||
_merge_run_content(run, next_elem)
|
||||
container.removeChild(next_elem)
|
||||
merge_count += 1
|
||||
else:
|
||||
break
|
||||
|
||||
_consolidate_text(run)
|
||||
run = _next_sibling_run(run, run_names)
|
||||
|
||||
return merge_count
|
||||
|
||||
|
||||
def _first_child_run(container, run_names: set[str]):
|
||||
for child in container.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE and _is_run(child, run_names):
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def _next_element_sibling(node):
|
||||
sibling = node.nextSibling
|
||||
while sibling:
|
||||
if sibling.nodeType == sibling.ELEMENT_NODE:
|
||||
return sibling
|
||||
sibling = sibling.nextSibling
|
||||
return None
|
||||
|
||||
|
||||
def _next_sibling_run(node, run_names: set[str]):
|
||||
sibling = node.nextSibling
|
||||
while sibling:
|
||||
if sibling.nodeType == sibling.ELEMENT_NODE:
|
||||
if _is_run(sibling, run_names):
|
||||
return sibling
|
||||
sibling = sibling.nextSibling
|
||||
return None
|
||||
|
||||
|
||||
def _is_run(node, run_names: set[str]) -> bool:
|
||||
return node.tagName in run_names
|
||||
|
||||
|
||||
def _can_merge(run1, run2) -> bool:
|
||||
rpr1 = _get_child(run1, "rPr")
|
||||
rpr2 = _get_child(run2, "rPr")
|
||||
|
||||
if (rpr1 is None) != (rpr2 is None):
|
||||
return False
|
||||
if rpr1 is None:
|
||||
return True
|
||||
return rpr1.toxml() == rpr2.toxml()
|
||||
|
||||
|
||||
def _merge_run_content(target, source):
|
||||
for child in list(source.childNodes):
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name != "rPr" and not name.endswith(":rPr"):
|
||||
target.appendChild(child)
|
||||
|
||||
|
||||
def _element_text(elem) -> str:
|
||||
return "".join(
|
||||
child.data
|
||||
for child in elem.childNodes
|
||||
if child.nodeType in (child.TEXT_NODE, child.CDATA_SECTION_NODE)
|
||||
)
|
||||
|
||||
|
||||
def _has_preserve(elem) -> bool:
|
||||
return elem.getAttribute("xml:space") == "preserve"
|
||||
|
||||
|
||||
def _rendered_text(elem) -> str:
|
||||
return rendered_text(_element_text(elem), _has_preserve(elem))
|
||||
|
||||
|
||||
def _consolidate_text(run):
|
||||
for tag in ("t", "delText"):
|
||||
_consolidate_text_elements(run, tag)
|
||||
|
||||
|
||||
def _consolidate_text_elements(run, tag: str):
|
||||
t_elements = _get_children(run, tag)
|
||||
|
||||
for i in range(len(t_elements) - 1, 0, -1):
|
||||
curr, prev = t_elements[i], t_elements[i - 1]
|
||||
|
||||
if _is_adjacent(prev, curr):
|
||||
merged = _rendered_text(prev) + _rendered_text(curr)
|
||||
had_preserve = _has_preserve(prev) or _has_preserve(curr)
|
||||
|
||||
new_text = run.ownerDocument.createTextNode(merged)
|
||||
for node in list(prev.childNodes):
|
||||
if node.nodeType in (node.TEXT_NODE, node.CDATA_SECTION_NODE):
|
||||
prev.removeChild(node)
|
||||
else:
|
||||
run.insertBefore(node, curr)
|
||||
prev.appendChild(new_text)
|
||||
for node in list(curr.childNodes):
|
||||
if node.nodeType not in (node.TEXT_NODE, node.CDATA_SECTION_NODE):
|
||||
run.insertBefore(node, curr)
|
||||
|
||||
if merged != merged.strip(XML_SPACE) or had_preserve:
|
||||
prev.setAttribute("xml:space", "preserve")
|
||||
elif prev.hasAttribute("xml:space"):
|
||||
prev.removeAttribute("xml:space")
|
||||
|
||||
run.removeChild(curr)
|
||||
|
||||
|
||||
|
||||
|
||||
def _merge_or_die(path: Path) -> str:
|
||||
_, msg = merge_runs(str(path))
|
||||
if msg.startswith("Error"):
|
||||
print(msg, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return msg
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Merge adjacent identically-formatted runs in a DOCX (directory or .docx file)."
|
||||
)
|
||||
p.add_argument("input", help="Unpacked DOCX directory OR a .docx/.dotx file")
|
||||
p.add_argument(
|
||||
"-o", "--output",
|
||||
help="Output .docx path (only valid when input is a .docx; default: overwrite input)",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
src = Path(args.input)
|
||||
|
||||
try:
|
||||
if src.is_dir():
|
||||
if args.output:
|
||||
p.error("--output is only valid for .docx input; directory input is modified in place")
|
||||
print(_merge_or_die(src))
|
||||
elif src.is_file() and src.suffix.lower() in (".docx", ".dotx"):
|
||||
out = Path(args.output) if args.output else src
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
with zipfile.ZipFile(src) as zf:
|
||||
safe_extract(zf, tmp_path)
|
||||
msg = _merge_or_die(tmp_path)
|
||||
rezip(tmp_path, out)
|
||||
print(f"{msg}; wrote {out}")
|
||||
else:
|
||||
print(f"Error: {src} is neither a directory nor a .docx/.dotx file", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except (OSError, ValueError, zipfile.BadZipFile) as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,111 @@
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import stat
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
OOXML_FAMILY = {
|
||||
".docx": "docx",
|
||||
".dotx": "docx",
|
||||
".pptx": "pptx",
|
||||
".potx": "pptx",
|
||||
".xlsx": "xlsx",
|
||||
".xltx": "xlsx",
|
||||
}
|
||||
|
||||
_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:")
|
||||
|
||||
SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide"
|
||||
|
||||
|
||||
def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None:
|
||||
if not target:
|
||||
return None
|
||||
if target_mode.lower() == "external":
|
||||
return None
|
||||
if _SCHEME_RE.match(target):
|
||||
return None
|
||||
|
||||
target = urllib.parse.unquote(target)
|
||||
|
||||
if "\\" in target:
|
||||
raise ValueError(f"relationship target is not a POSIX part name: {target!r}")
|
||||
|
||||
if target.startswith("/"):
|
||||
joined = target.lstrip("/")
|
||||
else:
|
||||
joined = posixpath.join(posixpath.dirname(source_part), target)
|
||||
|
||||
parts: list[str] = []
|
||||
for segment in posixpath.normpath(joined).split("/"):
|
||||
if segment in ("", "."):
|
||||
continue
|
||||
if segment == "..":
|
||||
if not parts:
|
||||
raise ValueError(f"relationship target escapes the package: {target!r}")
|
||||
parts.pop()
|
||||
else:
|
||||
parts.append(segment)
|
||||
|
||||
if not parts:
|
||||
raise ValueError(f"relationship target resolves to nothing: {target!r}")
|
||||
return "/".join(parts)
|
||||
|
||||
|
||||
def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str:
|
||||
owner_dir = rels_file.parent.parent.relative_to(unpacked_dir)
|
||||
return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./")
|
||||
|
||||
|
||||
def part_text(data: bytes) -> str:
|
||||
return data.decode("utf-8", "surrogateescape")
|
||||
|
||||
|
||||
XML_SPACE = " \t\r\n"
|
||||
|
||||
|
||||
def rendered_text(text: str, preserve: bool) -> str:
|
||||
return text if preserve else text.strip(XML_SPACE)
|
||||
|
||||
|
||||
def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None:
|
||||
dest = dest.resolve()
|
||||
for m in zf.infolist():
|
||||
if stat.S_ISLNK(m.external_attr >> 16):
|
||||
raise ValueError(f"symlink archive entry not allowed: {m.filename!r}")
|
||||
target = (dest / m.filename).resolve()
|
||||
if not target.is_relative_to(dest):
|
||||
raise ValueError(f"unsafe archive entry: {m.filename!r}")
|
||||
zf.extract(m, dest)
|
||||
|
||||
|
||||
def rezip(src_dir: Path, out_path: Path) -> None:
|
||||
files = sorted(p for p in src_dir.rglob("*") if p.is_file())
|
||||
ct = src_dir / "[Content_Types].xml"
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
prefix=out_path.name + ".", suffix=".tmp", dir=out_path.parent
|
||||
)
|
||||
tmp_out = Path(tmp_name)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
with zipfile.ZipFile(fh, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
if ct.exists():
|
||||
zf.write(ct, ct.relative_to(src_dir), compress_type=zipfile.ZIP_STORED)
|
||||
for f in files:
|
||||
if f == ct:
|
||||
continue
|
||||
zf.write(f, f.relative_to(src_dir))
|
||||
if out_path.exists():
|
||||
mode = out_path.stat().st_mode & 0o777
|
||||
else:
|
||||
umask = os.umask(0)
|
||||
os.umask(umask)
|
||||
mode = 0o666 & ~umask
|
||||
os.chmod(tmp_out, mode)
|
||||
os.replace(tmp_out, out_path)
|
||||
finally:
|
||||
if tmp_out.exists():
|
||||
tmp_out.unlink()
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Find chart XML that PowerPoint refuses but the schema accepts.
|
||||
|
||||
Detection only: for either fault more than one repair is valid, and only the
|
||||
author knows which was meant.
|
||||
"""
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Mapping
|
||||
|
||||
from . import part_text
|
||||
|
||||
|
||||
_CHART_PART_RE = re.compile(r"ppt/charts/chart\d+\.xml")
|
||||
|
||||
_GROUPING_RE = re.compile(r"""<c:grouping\b[^>]*?\bval=["'](\w+)["']""")
|
||||
_DLBL_POS_RE = re.compile(r"""<c:dLblPos\b[^>]*?\bval=["'](\w+)["']""")
|
||||
|
||||
def _strip_ext_lst(text: str) -> str:
|
||||
out, cursor = [], 0
|
||||
for lo, hi in _ext_lst_spans(text):
|
||||
out.append(text[cursor:lo])
|
||||
cursor = hi
|
||||
out.append(text[cursor:])
|
||||
return "".join(out)
|
||||
|
||||
_BAR_GROUP_RE = re.compile(r"<c:(bar3DChart|barChart)\b[^>]*(?<!/)>.*?</c:\1\s*>", re.DOTALL)
|
||||
|
||||
STACKED_GROUPINGS = frozenset({"stacked", "percentStacked"})
|
||||
ILLEGAL_ON_STACKED = frozenset({"outEnd"})
|
||||
LEGAL_ON_STACKED = ("ctr", "inEnd", "inBase")
|
||||
|
||||
|
||||
def _check_stacked_label_positions(part: str, xml: str) -> list[str]:
|
||||
problems: list[str] = []
|
||||
for match in _BAR_GROUP_RE.finditer(xml):
|
||||
block = _strip_ext_lst(match.group(0))
|
||||
group = match.group(1)
|
||||
|
||||
grouping = _GROUPING_RE.search(block)
|
||||
if grouping is None or grouping.group(1) not in STACKED_GROUPINGS:
|
||||
continue
|
||||
|
||||
bad = [p for p in _DLBL_POS_RE.findall(block) if p in ILLEGAL_ON_STACKED]
|
||||
for pos in sorted(set(bad)):
|
||||
problems.append(
|
||||
f'{part}: {bad.count(pos)} data label(s) use dLblPos="{pos}" on a '
|
||||
f"{grouping.group(1)} {group}; PowerPoint allows only "
|
||||
f"{', '.join(LEGAL_ON_STACKED)} there"
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
|
||||
_ANY_CHART_GROUP_RE = re.compile(r"<c:(\w+Chart)\b[^>]*(?<!/)>.*?</c:\1\s*>", re.DOTALL)
|
||||
|
||||
_AXID_RE = re.compile(
|
||||
r"""\s*<c:axId\b[^>]*?\bval=["'](-?\d+)["']\s*(?:/>|>\s*</c:axId\s*>)"""
|
||||
)
|
||||
|
||||
_AXIS_DECL_RE = re.compile(
|
||||
r"""<c:(catAx|valAx|serAx|dateAx)\b[^>]*(?<!/)>\s*<c:axId\b[^>]*?\bval=["'](-?\d+)["']"""
|
||||
)
|
||||
|
||||
AXID_LIMIT = {
|
||||
"barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2,
|
||||
"bubbleChart": 2, "radarChart": 2, "stockChart": 2,
|
||||
"bar3DChart": 3, "line3DChart": 3, "area3DChart": 3,
|
||||
"surfaceChart": 3, "surface3DChart": 3,
|
||||
}
|
||||
|
||||
AXID_MINIMUM = {
|
||||
"barChart": 2, "lineChart": 2, "areaChart": 2, "scatterChart": 2,
|
||||
"bubbleChart": 2, "radarChart": 2, "stockChart": 2,
|
||||
"bar3DChart": 2, "area3DChart": 2, "surfaceChart": 2,
|
||||
"line3DChart": 3, "surface3DChart": 3,
|
||||
}
|
||||
|
||||
|
||||
def _declared_axes(xml: str) -> dict[str, list[str]]:
|
||||
axes: dict[str, list[str]] = {}
|
||||
for kind, axid in _AXIS_DECL_RE.findall(xml):
|
||||
axes.setdefault(kind, []).append(axid)
|
||||
return axes
|
||||
|
||||
|
||||
def _canonical_ids(axes: dict[str, list[str]], limit: int) -> list[str] | None:
|
||||
category = axes.get("catAx", []) + axes.get("dateAx", [])
|
||||
value = axes.get("valAx", [])
|
||||
series = axes.get("serAx", [])
|
||||
if len(category) != 1 or len(value) != 1 or len(series) > 1:
|
||||
return None
|
||||
ids = [category[0], value[0]]
|
||||
if limit >= 3 and series:
|
||||
ids.append(series[0])
|
||||
return ids
|
||||
|
||||
|
||||
def _undeclared_axes(kind: str, block: str, axes: dict[str, list[str]]) -> list[str] | None:
|
||||
if kind not in AXID_LIMIT:
|
||||
return None
|
||||
ids = _AXID_RE.findall(block)
|
||||
declared = {i for group in axes.values() for i in group}
|
||||
if len([i for i in ids if i in declared]) >= 2:
|
||||
return None
|
||||
return ids
|
||||
|
||||
|
||||
def _check_chart_axis_references(part: str, xml: str) -> list[str]:
|
||||
axes = _declared_axes(xml)
|
||||
problems: list[str] = []
|
||||
declared = {i for group in axes.values() for i in group}
|
||||
for match in _ANY_CHART_GROUP_RE.finditer(xml):
|
||||
kind, block = match.group(1), match.group(0)
|
||||
ids = _undeclared_axes(kind, block, axes)
|
||||
if ids is None:
|
||||
continue
|
||||
if not ids:
|
||||
problems.append(
|
||||
f"{part}: <c:{kind}> declares no <c:axId> this part can resolve; a chart "
|
||||
f"group needs {AXID_MINIMUM[kind]}, and PowerPoint discards one with fewer"
|
||||
)
|
||||
continue
|
||||
dead = [i for i in ids if i not in declared]
|
||||
canonical = _canonical_ids(axes, AXID_LIMIT[kind])
|
||||
if canonical is not None and len(canonical) >= AXID_MINIMUM[kind]:
|
||||
hint = f"Fix: point them at the axes this part declares ({', '.join(canonical)})"
|
||||
else:
|
||||
hint = ("Fix: the part declares several axes of a kind -- declare the "
|
||||
"secondary axes the series expects, or drop them")
|
||||
detail = (f"of which {', '.join(dead)} name no declared axis"
|
||||
if dead else f"only {len(ids)} of which this part declares")
|
||||
problems.append(
|
||||
f"{part}: <c:{kind}> references axId {', '.join(ids)}, {detail}, "
|
||||
f"leaving fewer than two live axes; PowerPoint discards the chart. {hint}"
|
||||
)
|
||||
return problems
|
||||
|
||||
|
||||
def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
|
||||
spans: list[tuple[int, int]] = []
|
||||
depth = 0
|
||||
start = 0
|
||||
for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text):
|
||||
closing, self_closing = match.group(1), match.group(2)
|
||||
if self_closing:
|
||||
continue
|
||||
if closing:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
spans.append((start, match.end()))
|
||||
else:
|
||||
if depth == 0:
|
||||
start = match.start()
|
||||
depth += 1
|
||||
return spans
|
||||
|
||||
|
||||
CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references)
|
||||
|
||||
|
||||
def find_chart_problems(files: Mapping[str, bytes]) -> list[str]:
|
||||
problems: list[str] = []
|
||||
for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)):
|
||||
xml = part_text(files[part])
|
||||
for check in CHART_CHECKS:
|
||||
problems.extend(check(part, xml))
|
||||
return problems
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Pick the slide-XML schema errors PowerPoint refuses the file over.
|
||||
|
||||
A denylist over lxml's messages, so an unrecognised error class is a miss rather
|
||||
than a false alarm.
|
||||
"""
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
SLIDE_PART_RE = re.compile(
|
||||
r"ppt/(slides|slideLayouts|slideMasters|notesSlides|notesMasters|handoutMasters)"
|
||||
r"/[^/]+\.xml"
|
||||
)
|
||||
|
||||
FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(
|
||||
re.compile(r"\}tableStyleId': This element is not expected"),
|
||||
"two <a:tableStyleId> in one <a:tblPr> (the schema allows one)",
|
||||
),
|
||||
(
|
||||
re.compile(r"\}srgbClr', attribute 'val'"),
|
||||
"a colour that is not six hex digits",
|
||||
),
|
||||
(
|
||||
re.compile(r"\}txBody': Missing child element"),
|
||||
"a <p:txBody> with no children",
|
||||
),
|
||||
(
|
||||
re.compile(r"\}miter', attribute 'lim'"),
|
||||
'a line join with lim="NaN"',
|
||||
),
|
||||
(
|
||||
re.compile(r"\}uLnTx': This element is not expected"),
|
||||
"<a:uLnTx> in a position the schema forbids",
|
||||
),
|
||||
(
|
||||
re.compile(r"\}overrideClrMapping': This element is not expected"),
|
||||
"<p:overrideClrMapping> in a position the schema forbids",
|
||||
),
|
||||
(
|
||||
re.compile(r"\}nvGrpSpPr': Missing child element"),
|
||||
"a <p:nvGrpSpPr> with no children",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def is_schema_verdict(error: str) -> bool:
|
||||
return error.startswith("Element ")
|
||||
|
||||
|
||||
def fatal_slide_errors(errors: set[str]) -> list[str]:
|
||||
out = []
|
||||
for error in sorted(errors):
|
||||
for pattern, meaning in FATAL_SLIDE_ERRORS:
|
||||
if pattern.search(error):
|
||||
out.append(f"{meaning}: {error}")
|
||||
break
|
||||
return out
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Find masters sharing a theme part in the way PowerPoint refuses to open.
|
||||
|
||||
Reports only; the fix is to move <p:notesMasterIdLst> back to directly after
|
||||
<p:sldIdLst> in ppt/presentation.xml.
|
||||
"""
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import posixpath
|
||||
import re
|
||||
from typing import Mapping
|
||||
|
||||
from . import part_text
|
||||
|
||||
THEME_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme"
|
||||
|
||||
_MASTER_RE = re.compile(
|
||||
r"^ppt/(?P<group>slideMasters|notesMasters|handoutMasters)/"
|
||||
r"(?:slide|notes|handout)Master(?P<num>\d+)\.xml$"
|
||||
)
|
||||
_GROUP_ORDER = {"slideMasters": 0, "notesMasters": 1, "handoutMasters": 2}
|
||||
|
||||
_RELATIONSHIP_RE = re.compile(
|
||||
r"<Relationship\b[^>]*?(?:/>|>.*?</Relationship\s*>)", re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def _sort_key(name: str) -> tuple[int, int]:
|
||||
m = _MASTER_RE.match(name)
|
||||
assert m is not None
|
||||
return (_GROUP_ORDER[m.group("group")], int(m.group("num")))
|
||||
|
||||
|
||||
def _rels_path(part: str) -> str:
|
||||
directory, base = posixpath.split(part)
|
||||
return f"{directory}/_rels/{base}.rels"
|
||||
|
||||
|
||||
def _resolve(rels_path: str, target: str) -> str:
|
||||
if target.startswith("/"):
|
||||
return target.lstrip("/")
|
||||
part_dir = posixpath.dirname(posixpath.dirname(rels_path))
|
||||
return posixpath.normpath(posixpath.join(part_dir, target))
|
||||
|
||||
|
||||
def _theme_rel(files: Mapping[str, bytes], master: str):
|
||||
rels_path = _rels_path(master)
|
||||
rels = files.get(rels_path)
|
||||
if rels is None:
|
||||
return None
|
||||
for element in _RELATIONSHIP_RE.findall(part_text(rels)):
|
||||
if f'Type="{THEME_REL_TYPE}"' not in element:
|
||||
continue
|
||||
target = re.search(r'\bTarget="([^"]+)"', element)
|
||||
if target is None:
|
||||
continue
|
||||
return rels_path, element, _resolve(rels_path, target.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def _masters(files: Mapping[str, bytes]) -> list[str]:
|
||||
return sorted((n for n in files if _MASTER_RE.match(n)), key=_sort_key)
|
||||
|
||||
|
||||
_PRESENTATION = "ppt/presentation.xml"
|
||||
_NOTES_MASTERS = "ppt/notesMasters/"
|
||||
_IGNORABLE_RE = re.compile(r"<!--.*?-->|<\?.*?\?>", re.DOTALL)
|
||||
_AFTER_SLDIDLST_RE = re.compile(
|
||||
r"<p:sldIdLst\b(?:[^>]*/>|[^>]*>.*?</p:sldIdLst\s*>)\s*(<[^>\s/]+)", re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def _notes_master_share_is_inert(files: Mapping[str, bytes]) -> bool:
|
||||
data = files.get(_PRESENTATION)
|
||||
if data is None:
|
||||
return False
|
||||
match = _AFTER_SLDIDLST_RE.search(_IGNORABLE_RE.sub("", part_text(data)))
|
||||
return match is not None and match.group(1) == "<p:notesMasterIdLst"
|
||||
|
||||
|
||||
def _shares(files: Mapping[str, bytes]):
|
||||
owner: dict[str, str] = {}
|
||||
for master in _masters(files):
|
||||
found = _theme_rel(files, master)
|
||||
if found is None:
|
||||
continue
|
||||
rels_path, element, theme = found
|
||||
if theme not in files:
|
||||
continue
|
||||
if theme in owner:
|
||||
yield master, rels_path, element, theme, owner[theme]
|
||||
else:
|
||||
owner[theme] = master
|
||||
|
||||
|
||||
def _is_inert(master: str, inert_notes: bool) -> bool:
|
||||
return inert_notes and master.startswith(_NOTES_MASTERS)
|
||||
|
||||
|
||||
def find_shared_master_themes(files: Mapping[str, bytes]) -> list[str]:
|
||||
return [
|
||||
f"{master} shares {theme} with {first}"
|
||||
for master, _, _, theme, first in _shares(files)
|
||||
]
|
||||
|
||||
|
||||
def live_shared_master_themes(files: Mapping[str, bytes]) -> list[str]:
|
||||
inert_notes = _notes_master_share_is_inert(files)
|
||||
return [
|
||||
f"{master} shares {theme} with {first}"
|
||||
for master, _, _, theme, first in _shares(files)
|
||||
if not _is_inert(master, inert_notes)
|
||||
]
|
||||
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
Helper for running LibreOffice (soffice) in environments where AF_UNIX
|
||||
sockets may be blocked (e.g., sandboxed VMs). Detects the restriction
|
||||
at runtime and applies an LD_PRELOAD shim if needed.
|
||||
|
||||
Usage:
|
||||
from office.soffice import run_soffice
|
||||
|
||||
result = run_soffice(["--headless", "--convert-to", "pdf", "input.docx"])
|
||||
|
||||
Call soffice through run_soffice, not through subprocess with get_soffice_env():
|
||||
the env dict carries the shim but names no user profile, and a non-root sandbox
|
||||
cannot bootstrap the default one -- soffice aborts with "User installation could
|
||||
not be completed" and converts nothing. get_soffice_env() stays public for the
|
||||
callers that build their own argv (they must pass -env:UserInstallation too).
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_soffice_env() -> dict:
|
||||
env = os.environ.copy()
|
||||
env["SAL_USE_VCLPLUGIN"] = "svp"
|
||||
|
||||
if _needs_shim():
|
||||
shim = _ensure_shim()
|
||||
env["LD_PRELOAD"] = str(shim)
|
||||
|
||||
return env
|
||||
|
||||
|
||||
def run_soffice(args: Iterable[str], **kwargs) -> subprocess.CompletedProcess:
|
||||
args = list(args)
|
||||
with contextlib.ExitStack() as stack:
|
||||
if not any(str(a).startswith("-env:UserInstallation") for a in args):
|
||||
profile = stack.enter_context(
|
||||
tempfile.TemporaryDirectory(prefix="lo_profile_", ignore_cleanup_errors=True)
|
||||
)
|
||||
args = [f"-env:UserInstallation={Path(profile).as_uri()}"] + args
|
||||
return subprocess.run(["soffice"] + args, env=get_soffice_env(), **kwargs)
|
||||
|
||||
|
||||
|
||||
_SHIM_SO = Path(tempfile.gettempdir()) / "lo_socket_shim.so"
|
||||
|
||||
|
||||
def _needs_shim() -> bool:
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.close()
|
||||
return False
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
|
||||
def _ensure_shim() -> Path:
|
||||
if _SHIM_SO.exists():
|
||||
return _SHIM_SO
|
||||
|
||||
src = Path(tempfile.gettempdir()) / "lo_socket_shim.c"
|
||||
src.write_text(_SHIM_SOURCE)
|
||||
subprocess.run(
|
||||
["gcc", "-shared", "-fPIC", "-o", str(_SHIM_SO), str(src), "-ldl"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
src.unlink()
|
||||
return _SHIM_SO
|
||||
|
||||
|
||||
|
||||
_SHIM_SOURCE = r"""
|
||||
#define _GNU_SOURCE
|
||||
#include <dlfcn.h>
|
||||
#include <errno.h>
|
||||
#include <signal.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int (*real_socket)(int, int, int);
|
||||
static int (*real_socketpair)(int, int, int, int[2]);
|
||||
static int (*real_listen)(int, int);
|
||||
static int (*real_accept)(int, struct sockaddr *, socklen_t *);
|
||||
static int (*real_close)(int);
|
||||
static int (*real_read)(int, void *, size_t);
|
||||
|
||||
/* Per-FD bookkeeping (FDs >= 1024 are passed through unshimmed). */
|
||||
static int is_shimmed[1024];
|
||||
static int peer_of[1024];
|
||||
static int wake_r[1024]; /* accept() blocks reading this */
|
||||
static int wake_w[1024]; /* close() writes to this */
|
||||
static int listener_fd = -1; /* FD that received listen() */
|
||||
|
||||
__attribute__((constructor))
|
||||
static void init(void) {
|
||||
real_socket = dlsym(RTLD_NEXT, "socket");
|
||||
real_socketpair = dlsym(RTLD_NEXT, "socketpair");
|
||||
real_listen = dlsym(RTLD_NEXT, "listen");
|
||||
real_accept = dlsym(RTLD_NEXT, "accept");
|
||||
real_close = dlsym(RTLD_NEXT, "close");
|
||||
real_read = dlsym(RTLD_NEXT, "read");
|
||||
for (int i = 0; i < 1024; i++) {
|
||||
peer_of[i] = -1;
|
||||
wake_r[i] = -1;
|
||||
wake_w[i] = -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- socket ---------------------------------------------------------- */
|
||||
int socket(int domain, int type, int protocol) {
|
||||
if (domain == AF_UNIX) {
|
||||
int fd = real_socket(domain, type, protocol);
|
||||
if (fd >= 0) return fd;
|
||||
/* socket(AF_UNIX) blocked – fall back to socketpair(). */
|
||||
int sv[2];
|
||||
if (real_socketpair(domain, type, protocol, sv) == 0) {
|
||||
if (sv[0] >= 0 && sv[0] < 1024) {
|
||||
is_shimmed[sv[0]] = 1;
|
||||
peer_of[sv[0]] = sv[1];
|
||||
int wp[2];
|
||||
if (pipe(wp) == 0) {
|
||||
wake_r[sv[0]] = wp[0];
|
||||
wake_w[sv[0]] = wp[1];
|
||||
}
|
||||
}
|
||||
return sv[0];
|
||||
}
|
||||
errno = EPERM;
|
||||
return -1;
|
||||
}
|
||||
return real_socket(domain, type, protocol);
|
||||
}
|
||||
|
||||
/* ---- listen ---------------------------------------------------------- */
|
||||
int listen(int sockfd, int backlog) {
|
||||
if (sockfd >= 0 && sockfd < 1024 && is_shimmed[sockfd]) {
|
||||
listener_fd = sockfd;
|
||||
return 0;
|
||||
}
|
||||
return real_listen(sockfd, backlog);
|
||||
}
|
||||
|
||||
/* ---- accept ---------------------------------------------------------- */
|
||||
int accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) {
|
||||
if (sockfd >= 0 && sockfd < 1024 && is_shimmed[sockfd]) {
|
||||
/* Block until close() writes to the wake pipe. */
|
||||
if (wake_r[sockfd] >= 0) {
|
||||
char buf;
|
||||
real_read(wake_r[sockfd], &buf, 1);
|
||||
}
|
||||
errno = ECONNABORTED;
|
||||
return -1;
|
||||
}
|
||||
return real_accept(sockfd, addr, addrlen);
|
||||
}
|
||||
|
||||
/* ---- close ----------------------------------------------------------- */
|
||||
int close(int fd) {
|
||||
if (fd >= 0 && fd < 1024 && is_shimmed[fd]) {
|
||||
int was_listener = (fd == listener_fd);
|
||||
is_shimmed[fd] = 0;
|
||||
|
||||
if (wake_w[fd] >= 0) { /* unblock accept() */
|
||||
char c = 0;
|
||||
write(wake_w[fd], &c, 1);
|
||||
real_close(wake_w[fd]);
|
||||
wake_w[fd] = -1;
|
||||
}
|
||||
if (wake_r[fd] >= 0) { real_close(wake_r[fd]); wake_r[fd] = -1; }
|
||||
if (peer_of[fd] >= 0) { real_close(peer_of[fd]); peer_of[fd] = -1; }
|
||||
|
||||
if (was_listener)
|
||||
_exit(0); /* conversion done – exit */
|
||||
}
|
||||
return real_close(fd);
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
result = run_soffice(sys.argv[1:])
|
||||
sys.exit(result.returncode)
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
Command line tool to validate Office document XML files against XSD schemas and tracked changes.
|
||||
|
||||
Usage:
|
||||
python validate.py <path> [--original <original_file>] [--auto-repair] [--author NAME]
|
||||
|
||||
The first argument can be either:
|
||||
- An unpacked directory containing the Office document XML files
|
||||
- A packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx template) which will be unpacked to a temp directory
|
||||
|
||||
Auto-repair fixes:
|
||||
- paraId/durableId values that exceed OOXML limits
|
||||
- Missing xml:space="preserve" on w:t elements with whitespace
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
from defusedxml.common import DefusedXmlException
|
||||
|
||||
from helpers import OOXML_FAMILY, rezip, safe_extract
|
||||
from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator
|
||||
|
||||
WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
|
||||
|
||||
def _fail(message: str):
|
||||
print(f"Error: {message}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def _has_tracked_changes(unpacked_dir: Path) -> bool:
|
||||
document = unpacked_dir / "word" / "document.xml"
|
||||
if not document.is_file():
|
||||
return False
|
||||
try:
|
||||
root = ET.parse(document).getroot()
|
||||
except (ET.ParseError, DefusedXmlException):
|
||||
return False
|
||||
tracked = {f"{{{WORD_NS}}}ins", f"{{{WORD_NS}}}del"}
|
||||
return any(elem.tag in tracked for elem in root.iter())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Validate Office document XML files")
|
||||
parser.add_argument(
|
||||
"path",
|
||||
help="Path to unpacked directory or packed Office file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--original",
|
||||
required=False,
|
||||
default=None,
|
||||
help="Path to original file (.docx/.pptx/.xlsx or .dotx/.potx/.xltx). If omitted, all XSD errors are reported and redlining validation is skipped.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose output",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--auto-repair",
|
||||
action="store_true",
|
||||
help="Automatically repair common issues (hex IDs, whitespace preservation). "
|
||||
"Modifies the input in place: repairs to a packed file are written back to it.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--author",
|
||||
default=None,
|
||||
help="The name you are redlining under. Passing it turns on the "
|
||||
"tracked-change check: any text differing from --original without a "
|
||||
"<w:ins>/<w:del> recording it is reported. Untracked edits carry no "
|
||||
"author, so the check covers them whoever made them — the name marks "
|
||||
"the run as redlining work and is not used to filter. Requires "
|
||||
"--original; docx only.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.author is not None and not args.original:
|
||||
_fail("--author requires --original")
|
||||
|
||||
path = Path(args.path)
|
||||
if not path.exists():
|
||||
_fail(f"{path} does not exist")
|
||||
|
||||
original_file = None
|
||||
if args.original:
|
||||
original_file = Path(args.original)
|
||||
if not original_file.is_file():
|
||||
_fail(f"{original_file} is not a file")
|
||||
if original_file.suffix.lower() not in OOXML_FAMILY:
|
||||
_fail(f"{original_file} must be one of: {', '.join(sorted(OOXML_FAMILY))}")
|
||||
|
||||
family = OOXML_FAMILY.get((original_file or path).suffix.lower())
|
||||
if family is None:
|
||||
_fail(
|
||||
f"Cannot determine file type from {path}. Use --original or provide one of: {', '.join(sorted(OOXML_FAMILY))}."
|
||||
)
|
||||
|
||||
if args.author is not None and family != "docx":
|
||||
_fail(f"--author only applies to docx files, not {family}")
|
||||
|
||||
packed_file = None
|
||||
temp_dir_ctx = None
|
||||
if path.is_file() and path.suffix.lower() in OOXML_FAMILY:
|
||||
packed_file = path
|
||||
temp_dir_ctx = tempfile.TemporaryDirectory()
|
||||
unpacked_dir = Path(temp_dir_ctx.name)
|
||||
try:
|
||||
with zipfile.ZipFile(path, "r") as zf:
|
||||
safe_extract(zf, unpacked_dir)
|
||||
except (zipfile.BadZipFile, ValueError, OSError) as e:
|
||||
_fail(f"cannot unpack {path}: {e}")
|
||||
else:
|
||||
if not path.is_dir():
|
||||
_fail(f"{path} is not a directory or Office file")
|
||||
unpacked_dir = path
|
||||
|
||||
match family:
|
||||
case "docx":
|
||||
validators = [
|
||||
DOCXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose),
|
||||
]
|
||||
if args.author is not None:
|
||||
validators.append(
|
||||
RedliningValidator(unpacked_dir, original_file, verbose=args.verbose)
|
||||
)
|
||||
elif original_file and _has_tracked_changes(unpacked_dir):
|
||||
print(
|
||||
"Note: this document has tracked changes; they were not "
|
||||
"checked against the original (pass --author to check)."
|
||||
)
|
||||
case "pptx":
|
||||
validators = [
|
||||
PPTXSchemaValidator(unpacked_dir, original_file, verbose=args.verbose),
|
||||
]
|
||||
case "xlsx":
|
||||
exts = ", ".join(k for k, v in sorted(OOXML_FAMILY.items()) if v == "xlsx")
|
||||
print(
|
||||
f"No XSD schema validation is performed for xlsx-family files ({exts}). "
|
||||
"For formula-error checking, use scripts/recalc.py instead."
|
||||
)
|
||||
sys.exit(0)
|
||||
case _:
|
||||
print(f"Error: Validation not supported for file type {family}")
|
||||
sys.exit(1)
|
||||
|
||||
if args.auto_repair:
|
||||
total_repairs = sum(v.repair() for v in validators)
|
||||
if total_repairs:
|
||||
print(f"Auto-repaired {total_repairs} issue(s)")
|
||||
if packed_file is not None:
|
||||
rezip(unpacked_dir, packed_file)
|
||||
print(f"Wrote repaired file to {packed_file}")
|
||||
|
||||
success = all([v.validate() for v in validators])
|
||||
|
||||
if temp_dir_ctx is not None:
|
||||
temp_dir_ctx.cleanup()
|
||||
|
||||
if success:
|
||||
print("All validations PASSED!")
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
Validation modules for Word document processing.
|
||||
"""
|
||||
|
||||
from .base import BaseSchemaValidator
|
||||
from .docx import DOCXSchemaValidator
|
||||
from .pptx import PPTXSchemaValidator
|
||||
from .redlining import RedliningValidator
|
||||
|
||||
__all__ = [
|
||||
"BaseSchemaValidator",
|
||||
"DOCXSchemaValidator",
|
||||
"PPTXSchemaValidator",
|
||||
"RedliningValidator",
|
||||
]
|
||||
@@ -0,0 +1,875 @@
|
||||
"""
|
||||
Base validator with common validation logic for document files.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
from functools import lru_cache
|
||||
|
||||
import lxml.etree
|
||||
|
||||
from helpers import safe_extract
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _load_schema(schema_path: str):
|
||||
with open(schema_path, "rb") as xsd_file:
|
||||
xsd_doc = lxml.etree.parse(
|
||||
xsd_file, parser=lxml.etree.XMLParser(), base_url=schema_path
|
||||
)
|
||||
return lxml.etree.XMLSchema(xsd_doc)
|
||||
|
||||
class BaseSchemaValidator:
|
||||
|
||||
IGNORED_VALIDATION_ERRORS = [
|
||||
"hyphenationZone",
|
||||
"purl.org/dc/terms",
|
||||
]
|
||||
|
||||
UNIQUE_ID_REQUIREMENTS = {
|
||||
"comment": ("id", "file"),
|
||||
"commentrangestart": ("id", "file"),
|
||||
"commentrangeend": ("id", "file"),
|
||||
"bookmarkstart": ("id", "file"),
|
||||
"bookmarkend": ("id", "file"),
|
||||
"sldid": ("id", "file"),
|
||||
"sldmasterid": ("id", "global"),
|
||||
"sldlayoutid": ("id", "global"),
|
||||
"cm": ("authorid", "file"),
|
||||
"sheet": ("sheetid", "file"),
|
||||
"definedname": ("id", "file"),
|
||||
"cxnsp": ("id", "file"),
|
||||
"sp": ("id", "file"),
|
||||
"pic": ("id", "file"),
|
||||
"grpsp": ("id", "file"),
|
||||
}
|
||||
|
||||
EXCLUDED_ID_CONTAINERS = {
|
||||
"sectionlst",
|
||||
}
|
||||
|
||||
ELEMENT_RELATIONSHIP_TYPES = {}
|
||||
|
||||
SCHEMA_MAPPINGS = {
|
||||
"word": "ISO-IEC29500-4_2016/wml.xsd",
|
||||
"ppt": "ISO-IEC29500-4_2016/pml.xsd",
|
||||
"xl": "ISO-IEC29500-4_2016/sml.xsd",
|
||||
"[Content_Types].xml": "ecma/fourth-edition/opc-contentTypes.xsd",
|
||||
"app.xml": "ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd",
|
||||
"core.xml": "ecma/fourth-edition/opc-coreProperties.xsd",
|
||||
"custom.xml": "ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd",
|
||||
".rels": "ecma/fourth-edition/opc-relationships.xsd",
|
||||
"people.xml": "microsoft/wml-2012.xsd",
|
||||
"commentsIds.xml": "microsoft/wml-cid-2016.xsd",
|
||||
"commentsExtensible.xml": "microsoft/wml-cex-2018.xsd",
|
||||
"commentsExtended.xml": "microsoft/wml-2012.xsd",
|
||||
"chart": "ISO-IEC29500-4_2016/dml-chart.xsd",
|
||||
"theme": "ISO-IEC29500-4_2016/dml-main.xsd",
|
||||
"drawing": "ISO-IEC29500-4_2016/dml-main.xsd",
|
||||
}
|
||||
|
||||
MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006"
|
||||
XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
|
||||
|
||||
PACKAGE_RELATIONSHIPS_NAMESPACE = (
|
||||
"http://schemas.openxmlformats.org/package/2006/relationships"
|
||||
)
|
||||
OFFICE_RELATIONSHIPS_NAMESPACE = (
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
||||
)
|
||||
CONTENT_TYPES_NAMESPACE = (
|
||||
"http://schemas.openxmlformats.org/package/2006/content-types"
|
||||
)
|
||||
|
||||
MAIN_CONTENT_FOLDERS = {"word", "ppt", "xl"}
|
||||
|
||||
OOXML_NAMESPACES = {
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/math",
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/relationships",
|
||||
"http://schemas.openxmlformats.org/schemaLibrary/2006/main",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/main",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/chart",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/chartDrawing",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/diagram",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/picture",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing",
|
||||
"http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
|
||||
"http://schemas.openxmlformats.org/wordprocessingml/2006/main",
|
||||
"http://schemas.openxmlformats.org/presentationml/2006/main",
|
||||
"http://schemas.openxmlformats.org/spreadsheetml/2006/main",
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/sharedTypes",
|
||||
"http://www.w3.org/XML/1998/namespace",
|
||||
}
|
||||
|
||||
def __init__(self, unpacked_dir, original_file=None, verbose=False):
|
||||
self.unpacked_dir = Path(unpacked_dir).resolve()
|
||||
self.original_file = Path(original_file) if original_file else None
|
||||
self.verbose = verbose
|
||||
|
||||
self.schemas_dir = Path(__file__).parent.parent / "schemas"
|
||||
|
||||
patterns = ["*.xml", "*.rels"]
|
||||
self.xml_files = [
|
||||
f for pattern in patterns for f in self.unpacked_dir.rglob(pattern)
|
||||
]
|
||||
|
||||
if not self.xml_files:
|
||||
print(f"Warning: No XML files found in {self.unpacked_dir}")
|
||||
|
||||
def validate(self):
|
||||
raise NotImplementedError("Subclasses must implement the validate method")
|
||||
|
||||
def repair(self) -> int:
|
||||
return self.repair_whitespace_preservation()
|
||||
|
||||
def repair_whitespace_preservation(self) -> int:
|
||||
repairs = 0
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
content = xml_file.read_text(encoding="utf-8")
|
||||
dom = defusedxml.minidom.parseString(content)
|
||||
pending = []
|
||||
|
||||
for elem in dom.getElementsByTagName("*"):
|
||||
local_name = elem.tagName.rsplit(":", 1)[-1]
|
||||
if local_name in ("t", "delText", "instrText", "delInstrText"):
|
||||
text = "".join(
|
||||
child.data
|
||||
for child in elem.childNodes
|
||||
if child.nodeType in (child.TEXT_NODE, child.CDATA_SECTION_NODE)
|
||||
)
|
||||
ws = (" ", "\t", "\n", "\r")
|
||||
if text and (text.startswith(ws) or text.endswith(ws)):
|
||||
if elem.getAttribute("xml:space") != "preserve":
|
||||
elem.setAttribute("xml:space", "preserve")
|
||||
text_preview = repr(text[:30]) + "..." if len(text) > 30 else repr(text)
|
||||
pending.append(f" Repaired: {xml_file.name}: Added xml:space='preserve' to {elem.tagName}: {text_preview}")
|
||||
|
||||
if pending:
|
||||
xml_file.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
for message in pending:
|
||||
print(message)
|
||||
repairs += len(pending)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return repairs
|
||||
|
||||
def validate_xml(self):
|
||||
errors = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
lxml.etree.parse(str(xml_file))
|
||||
except lxml.etree.XMLSyntaxError as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {e.lineno}: {e.msg}"
|
||||
)
|
||||
except Exception as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Unexpected error: {str(e)}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} XML violations:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All XML files are well-formed")
|
||||
return True
|
||||
|
||||
def validate_namespaces(self):
|
||||
errors = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
declared = set(root.nsmap.keys()) - {None}
|
||||
|
||||
for attr_val in [
|
||||
v for k, v in root.attrib.items() if k.endswith("Ignorable")
|
||||
]:
|
||||
undeclared = set(attr_val.split()) - declared
|
||||
errors.extend(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Namespace '{ns}' in Ignorable but not declared"
|
||||
for ns in undeclared
|
||||
)
|
||||
except lxml.etree.XMLSyntaxError:
|
||||
continue
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - {len(errors)} namespace issues:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
if self.verbose:
|
||||
print("PASSED - All namespace prefixes properly declared")
|
||||
return True
|
||||
|
||||
def validate_unique_ids(self):
|
||||
errors = []
|
||||
global_ids = {}
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
file_ids = {}
|
||||
|
||||
mc_elements = root.xpath(
|
||||
".//mc:AlternateContent", namespaces={"mc": self.MC_NAMESPACE}
|
||||
)
|
||||
for elem in mc_elements:
|
||||
elem.getparent().remove(elem)
|
||||
|
||||
for elem in root.iter():
|
||||
if not hasattr(elem, "tag") or callable(elem.tag):
|
||||
continue
|
||||
tag = (
|
||||
elem.tag.split("}")[-1].lower()
|
||||
if "}" in elem.tag
|
||||
else elem.tag.lower()
|
||||
)
|
||||
|
||||
if tag in self.UNIQUE_ID_REQUIREMENTS:
|
||||
in_excluded_container = any(
|
||||
ancestor.tag.split("}")[-1].lower() in self.EXCLUDED_ID_CONTAINERS
|
||||
for ancestor in elem.iterancestors()
|
||||
)
|
||||
if in_excluded_container:
|
||||
continue
|
||||
|
||||
attr_name, scope = self.UNIQUE_ID_REQUIREMENTS[tag]
|
||||
|
||||
id_value = None
|
||||
for attr, value in elem.attrib.items():
|
||||
attr_local = (
|
||||
attr.split("}")[-1].lower()
|
||||
if "}" in attr
|
||||
else attr.lower()
|
||||
)
|
||||
if attr_local == attr_name:
|
||||
id_value = value
|
||||
break
|
||||
|
||||
if id_value is not None:
|
||||
if scope == "global":
|
||||
if id_value in global_ids:
|
||||
prev_file, prev_line, prev_tag = global_ids[
|
||||
id_value
|
||||
]
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {elem.sourceline}: Global ID '{id_value}' in <{tag}> "
|
||||
f"already used in {prev_file} at line {prev_line} in <{prev_tag}>"
|
||||
)
|
||||
else:
|
||||
global_ids[id_value] = (
|
||||
xml_file.relative_to(self.unpacked_dir),
|
||||
elem.sourceline,
|
||||
tag,
|
||||
)
|
||||
elif scope == "file":
|
||||
key = (tag, attr_name)
|
||||
if key not in file_ids:
|
||||
file_ids[key] = {}
|
||||
|
||||
if id_value in file_ids[key]:
|
||||
prev_line = file_ids[key][id_value]
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {elem.sourceline}: Duplicate {attr_name}='{id_value}' in <{tag}> "
|
||||
f"(first occurrence at line {prev_line})"
|
||||
)
|
||||
else:
|
||||
file_ids[key][id_value] = elem.sourceline
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} ID uniqueness violations:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All required IDs are unique")
|
||||
return True
|
||||
|
||||
def validate_file_references(self):
|
||||
errors = []
|
||||
|
||||
rels_files = list(self.unpacked_dir.rglob("*.rels"))
|
||||
|
||||
if not rels_files:
|
||||
if self.verbose:
|
||||
print("PASSED - No .rels files found")
|
||||
return True
|
||||
|
||||
all_files = []
|
||||
for file_path in self.unpacked_dir.rglob("*"):
|
||||
if (
|
||||
file_path.is_file()
|
||||
and file_path.name != "[Content_Types].xml"
|
||||
and not file_path.name.endswith(".rels")
|
||||
):
|
||||
all_files.append(file_path.resolve())
|
||||
|
||||
all_referenced_files = set()
|
||||
|
||||
if self.verbose:
|
||||
print(
|
||||
f"Found {len(rels_files)} .rels files and {len(all_files)} target files"
|
||||
)
|
||||
|
||||
for rels_file in rels_files:
|
||||
try:
|
||||
rels_root = lxml.etree.parse(str(rels_file)).getroot()
|
||||
|
||||
rels_dir = rels_file.parent
|
||||
|
||||
referenced_files = set()
|
||||
broken_refs = []
|
||||
|
||||
for rel in rels_root.findall(
|
||||
".//ns:Relationship",
|
||||
namespaces={"ns": self.PACKAGE_RELATIONSHIPS_NAMESPACE},
|
||||
):
|
||||
target = rel.get("Target")
|
||||
if rel.get("TargetMode") == "External":
|
||||
continue
|
||||
if target and not target.startswith(
|
||||
("http", "mailto:")
|
||||
):
|
||||
if target.startswith("/"):
|
||||
target_path = self.unpacked_dir / target.lstrip("/")
|
||||
elif rels_file.name == ".rels":
|
||||
target_path = self.unpacked_dir / target
|
||||
else:
|
||||
base_dir = rels_dir.parent
|
||||
target_path = base_dir / target
|
||||
|
||||
try:
|
||||
target_path = target_path.resolve()
|
||||
if target_path.exists() and target_path.is_file():
|
||||
referenced_files.add(target_path)
|
||||
all_referenced_files.add(target_path)
|
||||
else:
|
||||
broken_refs.append((target, rel.sourceline))
|
||||
except (OSError, ValueError):
|
||||
broken_refs.append((target, rel.sourceline))
|
||||
|
||||
if broken_refs:
|
||||
rel_path = rels_file.relative_to(self.unpacked_dir)
|
||||
for broken_ref, line_num in broken_refs:
|
||||
errors.append(
|
||||
f" {rel_path}: Line {line_num}: Broken reference to {broken_ref}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
rel_path = rels_file.relative_to(self.unpacked_dir)
|
||||
errors.append(f" Error parsing {rel_path}: {e}")
|
||||
|
||||
unreferenced_files = set(all_files) - all_referenced_files
|
||||
|
||||
if unreferenced_files:
|
||||
for unref_file in sorted(unreferenced_files):
|
||||
unref_rel_path = unref_file.relative_to(self.unpacked_dir)
|
||||
errors.append(f" Unreferenced file: {unref_rel_path}")
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} relationship validation errors:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
print(
|
||||
"CRITICAL: These errors will cause the document to appear corrupt. "
|
||||
+ "Broken references MUST be fixed, "
|
||||
+ "and unreferenced files MUST be referenced or removed."
|
||||
)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print(
|
||||
"PASSED - All references are valid and all files are properly referenced"
|
||||
)
|
||||
return True
|
||||
|
||||
def validate_all_relationship_ids(self):
|
||||
import lxml.etree
|
||||
|
||||
errors = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
if xml_file.suffix == ".rels":
|
||||
continue
|
||||
|
||||
rels_dir = xml_file.parent / "_rels"
|
||||
rels_file = rels_dir / f"{xml_file.name}.rels"
|
||||
|
||||
if not rels_file.exists():
|
||||
continue
|
||||
|
||||
try:
|
||||
rels_root = lxml.etree.parse(str(rels_file)).getroot()
|
||||
rid_to_type = {}
|
||||
|
||||
for rel in rels_root.findall(
|
||||
f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship"
|
||||
):
|
||||
rid = rel.get("Id")
|
||||
rel_type = rel.get("Type", "")
|
||||
if rid:
|
||||
if rid in rid_to_type:
|
||||
rels_rel_path = rels_file.relative_to(self.unpacked_dir)
|
||||
errors.append(
|
||||
f" {rels_rel_path}: Line {rel.sourceline}: "
|
||||
f"Duplicate relationship ID '{rid}' (IDs must be unique)"
|
||||
)
|
||||
type_name = (
|
||||
rel_type.split("/")[-1] if "/" in rel_type else rel_type
|
||||
)
|
||||
rid_to_type[rid] = type_name
|
||||
|
||||
xml_root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
|
||||
r_ns = self.OFFICE_RELATIONSHIPS_NAMESPACE
|
||||
rid_attrs_to_check = ["id", "embed", "link"]
|
||||
for elem in xml_root.iter():
|
||||
if not hasattr(elem, "tag") or callable(elem.tag):
|
||||
continue
|
||||
for attr_name in rid_attrs_to_check:
|
||||
rid_attr = elem.get(f"{{{r_ns}}}{attr_name}")
|
||||
if not rid_attr:
|
||||
continue
|
||||
xml_rel_path = xml_file.relative_to(self.unpacked_dir)
|
||||
elem_name = (
|
||||
elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag
|
||||
)
|
||||
|
||||
if rid_attr not in rid_to_type:
|
||||
errors.append(
|
||||
f" {xml_rel_path}: Line {elem.sourceline}: "
|
||||
f"<{elem_name}> r:{attr_name} references non-existent relationship '{rid_attr}' "
|
||||
f"(valid IDs: {', '.join(sorted(rid_to_type.keys())[:5])}{'...' if len(rid_to_type) > 5 else ''})"
|
||||
)
|
||||
elif attr_name == "id" and self.ELEMENT_RELATIONSHIP_TYPES:
|
||||
expected_type = self._get_expected_relationship_type(
|
||||
elem_name
|
||||
)
|
||||
if expected_type:
|
||||
actual_type = rid_to_type[rid_attr]
|
||||
if expected_type not in actual_type.lower():
|
||||
errors.append(
|
||||
f" {xml_rel_path}: Line {elem.sourceline}: "
|
||||
f"<{elem_name}> references '{rid_attr}' which points to '{actual_type}' "
|
||||
f"but should point to a '{expected_type}' relationship"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
xml_rel_path = xml_file.relative_to(self.unpacked_dir)
|
||||
errors.append(f" Error processing {xml_rel_path}: {e}")
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} relationship ID reference errors:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
print("\nThese ID mismatches will cause the document to appear corrupt!")
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All relationship ID references are valid")
|
||||
return True
|
||||
|
||||
def _get_expected_relationship_type(self, element_name):
|
||||
elem_lower = element_name.lower()
|
||||
|
||||
if elem_lower in self.ELEMENT_RELATIONSHIP_TYPES:
|
||||
return self.ELEMENT_RELATIONSHIP_TYPES[elem_lower]
|
||||
|
||||
if elem_lower.endswith("id") and len(elem_lower) > 2:
|
||||
prefix = elem_lower[:-2]
|
||||
if prefix.endswith("master"):
|
||||
return prefix.lower()
|
||||
elif prefix.endswith("layout"):
|
||||
return prefix.lower()
|
||||
else:
|
||||
if prefix == "sld":
|
||||
return "slide"
|
||||
return prefix.lower()
|
||||
|
||||
if elem_lower.endswith("reference") and len(elem_lower) > 9:
|
||||
prefix = elem_lower[:-9]
|
||||
return prefix.lower()
|
||||
|
||||
return None
|
||||
|
||||
def validate_content_types(self):
|
||||
errors = []
|
||||
|
||||
content_types_file = self.unpacked_dir / "[Content_Types].xml"
|
||||
if not content_types_file.exists():
|
||||
print("FAILED - [Content_Types].xml file not found")
|
||||
return False
|
||||
|
||||
try:
|
||||
root = lxml.etree.parse(str(content_types_file)).getroot()
|
||||
declared_parts = set()
|
||||
declared_extensions = set()
|
||||
|
||||
for override in root.findall(
|
||||
f".//{{{self.CONTENT_TYPES_NAMESPACE}}}Override"
|
||||
):
|
||||
part_name = override.get("PartName")
|
||||
if part_name is not None:
|
||||
declared_parts.add(part_name.lstrip("/"))
|
||||
|
||||
for default in root.findall(
|
||||
f".//{{{self.CONTENT_TYPES_NAMESPACE}}}Default"
|
||||
):
|
||||
extension = default.get("Extension")
|
||||
if extension is not None:
|
||||
declared_extensions.add(extension.lower())
|
||||
|
||||
declarable_roots = {
|
||||
"sld",
|
||||
"sldLayout",
|
||||
"sldMaster",
|
||||
"presentation",
|
||||
"document",
|
||||
"workbook",
|
||||
"worksheet",
|
||||
"theme",
|
||||
}
|
||||
|
||||
media_extensions = {
|
||||
"png": "image/png",
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"gif": "image/gif",
|
||||
"bmp": "image/bmp",
|
||||
"tiff": "image/tiff",
|
||||
"wmf": "image/x-wmf",
|
||||
"emf": "image/x-emf",
|
||||
}
|
||||
|
||||
all_files = list(self.unpacked_dir.rglob("*"))
|
||||
all_files = [f for f in all_files if f.is_file()]
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
path_str = str(xml_file.relative_to(self.unpacked_dir)).replace(
|
||||
"\\", "/"
|
||||
)
|
||||
|
||||
if any(
|
||||
skip in path_str
|
||||
for skip in [".rels", "[Content_Types]", "docProps/", "_rels/"]
|
||||
):
|
||||
continue
|
||||
|
||||
try:
|
||||
root_tag = lxml.etree.parse(str(xml_file)).getroot().tag
|
||||
root_name = root_tag.split("}")[-1] if "}" in root_tag else root_tag
|
||||
|
||||
if root_name in declarable_roots and path_str not in declared_parts:
|
||||
errors.append(
|
||||
f" {path_str}: File with <{root_name}> root not declared in [Content_Types].xml"
|
||||
)
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for file_path in all_files:
|
||||
if file_path.suffix.lower() in {".xml", ".rels"}:
|
||||
continue
|
||||
if file_path.name == "[Content_Types].xml":
|
||||
continue
|
||||
if "_rels" in file_path.parts or "docProps" in file_path.parts:
|
||||
continue
|
||||
|
||||
extension = file_path.suffix.lstrip(".").lower()
|
||||
if extension and extension not in declared_extensions:
|
||||
if extension in media_extensions:
|
||||
relative_path = file_path.relative_to(self.unpacked_dir)
|
||||
errors.append(
|
||||
f' {relative_path}: File with extension \'{extension}\' not declared in [Content_Types].xml - should add: <Default Extension="{extension}" ContentType="{media_extensions[extension]}"/>'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
errors.append(f" Error parsing [Content_Types].xml: {e}")
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} content type declaration errors:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print(
|
||||
"PASSED - All content files are properly declared in [Content_Types].xml"
|
||||
)
|
||||
return True
|
||||
|
||||
def validate_file_against_xsd(self, xml_file, verbose=False):
|
||||
xml_file = Path(xml_file).resolve()
|
||||
unpacked_dir = self.unpacked_dir.resolve()
|
||||
|
||||
is_valid, current_errors = self._validate_single_file_xsd(
|
||||
xml_file, unpacked_dir
|
||||
)
|
||||
|
||||
if is_valid is None:
|
||||
return None, set()
|
||||
elif is_valid:
|
||||
return True, set()
|
||||
|
||||
original_errors = self._get_original_file_errors(xml_file)
|
||||
|
||||
assert current_errors is not None
|
||||
new_errors = current_errors - original_errors
|
||||
|
||||
new_errors = {
|
||||
e for e in new_errors
|
||||
if not any(pattern in e for pattern in self.IGNORED_VALIDATION_ERRORS)
|
||||
}
|
||||
|
||||
if new_errors:
|
||||
if verbose:
|
||||
relative_path = xml_file.relative_to(unpacked_dir)
|
||||
print(f"FAILED - {relative_path}: {len(new_errors)} new error(s)")
|
||||
for error in list(new_errors)[:3]:
|
||||
truncated = error[:250] + "..." if len(error) > 250 else error
|
||||
print(f" - {truncated}")
|
||||
return False, new_errors
|
||||
else:
|
||||
if verbose:
|
||||
print(
|
||||
f"PASSED - No new errors (original had {len(current_errors)} errors)"
|
||||
)
|
||||
return True, set()
|
||||
|
||||
def validate_against_xsd(self):
|
||||
new_errors = []
|
||||
original_error_count = 0
|
||||
valid_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
relative_path = str(xml_file.relative_to(self.unpacked_dir))
|
||||
is_valid, new_file_errors = self.validate_file_against_xsd(
|
||||
xml_file, verbose=False
|
||||
)
|
||||
|
||||
if is_valid is None:
|
||||
skipped_count += 1
|
||||
continue
|
||||
elif is_valid and not new_file_errors:
|
||||
valid_count += 1
|
||||
continue
|
||||
elif is_valid:
|
||||
original_error_count += 1
|
||||
valid_count += 1
|
||||
continue
|
||||
|
||||
new_errors.append(f" {relative_path}: {len(new_file_errors)} new error(s)")
|
||||
for error in list(new_file_errors)[:3]:
|
||||
new_errors.append(
|
||||
f" - {error[:250]}..." if len(error) > 250 else f" - {error}"
|
||||
)
|
||||
|
||||
if self.verbose:
|
||||
print(f"Validated {len(self.xml_files)} files:")
|
||||
print(f" - Valid: {valid_count}")
|
||||
print(f" - Skipped (no schema): {skipped_count}")
|
||||
if original_error_count:
|
||||
print(f" - With original errors (ignored): {original_error_count}")
|
||||
print(
|
||||
f" - With NEW errors: {len(new_errors) > 0 and len([e for e in new_errors if not e.startswith(' ')]) or 0}"
|
||||
)
|
||||
|
||||
if new_errors:
|
||||
print("\nFAILED - Found NEW validation errors:")
|
||||
for error in new_errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("\nPASSED - No new XSD validation errors introduced")
|
||||
return True
|
||||
|
||||
def _get_schema_path(self, xml_file):
|
||||
if xml_file.name in self.SCHEMA_MAPPINGS:
|
||||
return self.schemas_dir / self.SCHEMA_MAPPINGS[xml_file.name]
|
||||
|
||||
if xml_file.suffix == ".rels":
|
||||
return self.schemas_dir / self.SCHEMA_MAPPINGS[".rels"]
|
||||
|
||||
if "charts/" in str(xml_file) and xml_file.name.startswith("chart"):
|
||||
return self.schemas_dir / self.SCHEMA_MAPPINGS["chart"]
|
||||
|
||||
if "theme/" in str(xml_file) and xml_file.name.startswith("theme"):
|
||||
return self.schemas_dir / self.SCHEMA_MAPPINGS["theme"]
|
||||
|
||||
if xml_file.parent.name in self.MAIN_CONTENT_FOLDERS:
|
||||
return self.schemas_dir / self.SCHEMA_MAPPINGS[xml_file.parent.name]
|
||||
|
||||
return None
|
||||
|
||||
def _clean_ignorable_namespaces(self, xml_doc):
|
||||
xml_string = lxml.etree.tostring(xml_doc, encoding="unicode")
|
||||
xml_copy = lxml.etree.fromstring(xml_string)
|
||||
|
||||
for elem in xml_copy.iter():
|
||||
attrs_to_remove = []
|
||||
|
||||
for attr in elem.attrib:
|
||||
if "{" in attr:
|
||||
ns = attr.split("}")[0][1:]
|
||||
if ns not in self.OOXML_NAMESPACES:
|
||||
attrs_to_remove.append(attr)
|
||||
|
||||
for attr in attrs_to_remove:
|
||||
del elem.attrib[attr]
|
||||
|
||||
self._remove_ignorable_elements(xml_copy)
|
||||
|
||||
return lxml.etree.ElementTree(xml_copy)
|
||||
|
||||
def _remove_ignorable_elements(self, root):
|
||||
elements_to_remove = []
|
||||
|
||||
for elem in list(root):
|
||||
if not hasattr(elem, "tag") or callable(elem.tag):
|
||||
continue
|
||||
|
||||
tag_str = str(elem.tag)
|
||||
if tag_str.startswith("{"):
|
||||
ns = tag_str.split("}")[0][1:]
|
||||
if ns not in self.OOXML_NAMESPACES:
|
||||
elements_to_remove.append(elem)
|
||||
continue
|
||||
|
||||
self._remove_ignorable_elements(elem)
|
||||
|
||||
for elem in elements_to_remove:
|
||||
root.remove(elem)
|
||||
|
||||
def _preprocess_for_mc_ignorable(self, xml_doc):
|
||||
root = xml_doc.getroot()
|
||||
|
||||
if f"{{{self.MC_NAMESPACE}}}Ignorable" in root.attrib:
|
||||
del root.attrib[f"{{{self.MC_NAMESPACE}}}Ignorable"]
|
||||
|
||||
return xml_doc
|
||||
|
||||
def _preprocess_for_schema(self, xml_doc, relative_path):
|
||||
return xml_doc
|
||||
|
||||
def _validate_single_file_xsd(self, xml_file, base_path, schema_path=None):
|
||||
schema_path = schema_path or self._get_schema_path(xml_file)
|
||||
if not schema_path:
|
||||
return None, None
|
||||
|
||||
try:
|
||||
schema = _load_schema(str(schema_path))
|
||||
|
||||
with open(xml_file, "r") as f:
|
||||
xml_doc = lxml.etree.parse(f)
|
||||
|
||||
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
|
||||
xml_doc = self._preprocess_for_mc_ignorable(xml_doc)
|
||||
|
||||
relative_path = xml_file.relative_to(base_path)
|
||||
if (
|
||||
relative_path.parts
|
||||
and relative_path.parts[0] in self.MAIN_CONTENT_FOLDERS
|
||||
):
|
||||
xml_doc = self._clean_ignorable_namespaces(xml_doc)
|
||||
|
||||
xml_doc = self._preprocess_for_schema(xml_doc, relative_path)
|
||||
|
||||
if schema.validate(xml_doc):
|
||||
return True, set()
|
||||
else:
|
||||
errors = set()
|
||||
for error in schema.error_log:
|
||||
errors.add(error.message)
|
||||
return False, errors
|
||||
|
||||
except Exception as e:
|
||||
return False, {str(e)}
|
||||
|
||||
def _get_original_file_errors(self, xml_file, schema_path=None):
|
||||
if self.original_file is None:
|
||||
return set()
|
||||
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
xml_file = Path(xml_file).resolve()
|
||||
unpacked_dir = self.unpacked_dir.resolve()
|
||||
relative_path = xml_file.relative_to(unpacked_dir)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(self.original_file, "r") as zip_ref:
|
||||
safe_extract(zip_ref, temp_path)
|
||||
except (zipfile.BadZipFile, ValueError, OSError):
|
||||
return set()
|
||||
|
||||
original_xml_file = temp_path / relative_path
|
||||
|
||||
if not original_xml_file.exists():
|
||||
return set()
|
||||
|
||||
is_valid, errors = self._validate_single_file_xsd(
|
||||
original_xml_file, temp_path, schema_path=schema_path
|
||||
)
|
||||
return errors if errors else set()
|
||||
|
||||
def _remove_template_tags_from_text_nodes(self, xml_doc):
|
||||
warnings = []
|
||||
template_pattern = re.compile(r"\{\{[^}]*\}\}")
|
||||
|
||||
xml_string = lxml.etree.tostring(xml_doc, encoding="unicode")
|
||||
xml_copy = lxml.etree.fromstring(xml_string)
|
||||
|
||||
def process_text_content(text, content_type):
|
||||
if not text:
|
||||
return text
|
||||
matches = list(template_pattern.finditer(text))
|
||||
if matches:
|
||||
for match in matches:
|
||||
warnings.append(
|
||||
f"Found template tag in {content_type}: {match.group()}"
|
||||
)
|
||||
return template_pattern.sub("", text)
|
||||
return text
|
||||
|
||||
for elem in xml_copy.iter():
|
||||
if not hasattr(elem, "tag") or callable(elem.tag):
|
||||
continue
|
||||
tag_str = str(elem.tag)
|
||||
if tag_str.endswith("}t") or tag_str == "t":
|
||||
continue
|
||||
|
||||
elem.text = process_text_content(elem.text, "text content")
|
||||
elem.tail = process_text_content(elem.tail, "tail content")
|
||||
|
||||
return lxml.etree.ElementTree(xml_copy), warnings
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise RuntimeError("This module should not be run directly.")
|
||||
@@ -0,0 +1,466 @@
|
||||
"""
|
||||
Validator for Word document XML files against XSD schemas.
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
import lxml.etree
|
||||
|
||||
from helpers import safe_extract
|
||||
|
||||
from .base import BaseSchemaValidator
|
||||
|
||||
|
||||
class DOCXSchemaValidator(BaseSchemaValidator):
|
||||
|
||||
WORD_2006_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
W14_NAMESPACE = "http://schemas.microsoft.com/office/word/2010/wordml"
|
||||
W16CID_NAMESPACE = "http://schemas.microsoft.com/office/word/2016/wordml/cid"
|
||||
|
||||
ELEMENT_RELATIONSHIP_TYPES = {}
|
||||
|
||||
def validate(self):
|
||||
if not self.validate_xml():
|
||||
return False
|
||||
|
||||
all_valid = True
|
||||
if not self.validate_namespaces():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_unique_ids():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_file_references():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_content_types():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_against_xsd():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_whitespace_preservation():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_deletions():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_insertions():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_all_relationship_ids():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_id_constraints():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_comment_markers():
|
||||
all_valid = False
|
||||
|
||||
self.compare_paragraph_counts()
|
||||
|
||||
return all_valid
|
||||
|
||||
def validate_whitespace_preservation(self):
|
||||
errors = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
if xml_file.name != "document.xml":
|
||||
continue
|
||||
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
|
||||
for elem in root.iter(f"{{{self.WORD_2006_NAMESPACE}}}t"):
|
||||
if elem.text:
|
||||
text = elem.text
|
||||
if re.search(r"^[ \t\n\r]", text) or re.search(
|
||||
r"[ \t\n\r]$", text
|
||||
):
|
||||
xml_space_attr = f"{{{self.XML_NAMESPACE}}}space"
|
||||
if (
|
||||
xml_space_attr not in elem.attrib
|
||||
or elem.attrib[xml_space_attr] != "preserve"
|
||||
):
|
||||
text_preview = (
|
||||
repr(text)[:50] + "..."
|
||||
if len(repr(text)) > 50
|
||||
else repr(text)
|
||||
)
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {elem.sourceline}: w:t element with whitespace missing xml:space='preserve': {text_preview}"
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} whitespace preservation violations:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All whitespace is properly preserved")
|
||||
return True
|
||||
|
||||
def validate_deletions(self):
|
||||
errors = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
if xml_file.name != "document.xml":
|
||||
continue
|
||||
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
namespaces = {"w": self.WORD_2006_NAMESPACE}
|
||||
|
||||
for t_elem in root.xpath(".//w:del//w:t", namespaces=namespaces):
|
||||
if t_elem.text:
|
||||
text_preview = (
|
||||
repr(t_elem.text)[:50] + "..."
|
||||
if len(repr(t_elem.text)) > 50
|
||||
else repr(t_elem.text)
|
||||
)
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {t_elem.sourceline}: <w:t> found within <w:del>: {text_preview}"
|
||||
)
|
||||
|
||||
for instr_elem in root.xpath(
|
||||
".//w:del//w:instrText", namespaces=namespaces
|
||||
):
|
||||
text_preview = (
|
||||
repr(instr_elem.text or "")[:50] + "..."
|
||||
if len(repr(instr_elem.text or "")) > 50
|
||||
else repr(instr_elem.text or "")
|
||||
)
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {instr_elem.sourceline}: <w:instrText> found within <w:del> (use <w:delInstrText>): {text_preview}"
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} deletion validation violations:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - No w:t elements found within w:del elements")
|
||||
return True
|
||||
|
||||
def count_paragraphs_in_unpacked(self):
|
||||
count = 0
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
if xml_file.name != "document.xml":
|
||||
continue
|
||||
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
paragraphs = root.findall(f".//{{{self.WORD_2006_NAMESPACE}}}p")
|
||||
count = len(paragraphs)
|
||||
except Exception as e:
|
||||
print(f"Error counting paragraphs in unpacked document: {e}")
|
||||
|
||||
return count
|
||||
|
||||
def count_paragraphs_in_original(self):
|
||||
original = self.original_file
|
||||
if original is None:
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with zipfile.ZipFile(original, "r") as zip_ref:
|
||||
safe_extract(zip_ref, Path(temp_dir))
|
||||
|
||||
doc_xml_path = temp_dir + "/word/document.xml"
|
||||
root = lxml.etree.parse(doc_xml_path).getroot()
|
||||
|
||||
paragraphs = root.findall(f".//{{{self.WORD_2006_NAMESPACE}}}p")
|
||||
count = len(paragraphs)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error counting paragraphs in original document: {e}")
|
||||
|
||||
return count
|
||||
|
||||
def validate_insertions(self):
|
||||
errors = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
if xml_file.name != "document.xml":
|
||||
continue
|
||||
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
namespaces = {"w": self.WORD_2006_NAMESPACE}
|
||||
|
||||
invalid_elements = root.xpath(
|
||||
".//w:ins//w:delText[not(ancestor::w:del)]", namespaces=namespaces
|
||||
)
|
||||
|
||||
for elem in invalid_elements:
|
||||
text_preview = (
|
||||
repr(elem.text or "")[:50] + "..."
|
||||
if len(repr(elem.text or "")) > 50
|
||||
else repr(elem.text or "")
|
||||
)
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {elem.sourceline}: <w:delText> within <w:ins>: {text_preview}"
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} insertion validation violations:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - No w:delText elements within w:ins elements")
|
||||
return True
|
||||
|
||||
def compare_paragraph_counts(self):
|
||||
new_count = self.count_paragraphs_in_unpacked()
|
||||
if self.original_file is None:
|
||||
print(f"\nParagraphs: {new_count}")
|
||||
return
|
||||
|
||||
original_count = self.count_paragraphs_in_original()
|
||||
diff = new_count - original_count
|
||||
diff_str = f"+{diff}" if diff > 0 else str(diff)
|
||||
print(f"\nParagraphs: {original_count} → {new_count} ({diff_str})")
|
||||
|
||||
def _parse_id_value(self, val: str, base: int = 16) -> int:
|
||||
return int(val, base)
|
||||
|
||||
def validate_id_constraints(self):
|
||||
errors = []
|
||||
para_id_attr = f"{{{self.W14_NAMESPACE}}}paraId"
|
||||
durable_id_attr = f"{{{self.W16CID_NAMESPACE}}}durableId"
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
for elem in lxml.etree.parse(str(xml_file)).iter():
|
||||
if val := elem.get(para_id_attr):
|
||||
try:
|
||||
if self._parse_id_value(val, base=16) >= 0x80000000:
|
||||
errors.append(
|
||||
f" {xml_file.name}:{elem.sourceline}: paraId={val} >= 0x80000000"
|
||||
)
|
||||
except ValueError:
|
||||
errors.append(
|
||||
f" {xml_file.name}:{elem.sourceline}: "
|
||||
f"paraId={val} is not valid hex"
|
||||
)
|
||||
|
||||
if val := elem.get(durable_id_attr):
|
||||
if xml_file.name == "numbering.xml":
|
||||
try:
|
||||
if self._parse_id_value(val, base=10) >= 0x7FFFFFFF:
|
||||
errors.append(
|
||||
f" {xml_file.name}:{elem.sourceline}: "
|
||||
f"durableId={val} >= 0x7FFFFFFF"
|
||||
)
|
||||
except ValueError:
|
||||
errors.append(
|
||||
f" {xml_file.name}:{elem.sourceline}: "
|
||||
f"durableId={val} must be decimal in numbering.xml"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
if self._parse_id_value(val, base=16) >= 0x7FFFFFFF:
|
||||
errors.append(
|
||||
f" {xml_file.name}:{elem.sourceline}: "
|
||||
f"durableId={val} >= 0x7FFFFFFF"
|
||||
)
|
||||
except ValueError:
|
||||
errors.append(
|
||||
f" {xml_file.name}:{elem.sourceline}: "
|
||||
f"durableId={val} is not valid hex"
|
||||
)
|
||||
except lxml.etree.XMLSyntaxError:
|
||||
continue
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - {len(errors)} ID constraint violations:")
|
||||
for e in errors:
|
||||
print(e)
|
||||
elif self.verbose:
|
||||
print("PASSED - All paraId/durableId values within constraints")
|
||||
return not errors
|
||||
|
||||
def validate_comment_markers(self):
|
||||
errors = []
|
||||
|
||||
document_xml = None
|
||||
comments_xml = None
|
||||
for xml_file in self.xml_files:
|
||||
if xml_file.name == "document.xml" and "word" in str(xml_file):
|
||||
document_xml = xml_file
|
||||
elif xml_file.name == "comments.xml":
|
||||
comments_xml = xml_file
|
||||
|
||||
if not document_xml:
|
||||
if self.verbose:
|
||||
print("PASSED - No document.xml found (skipping comment validation)")
|
||||
return True
|
||||
|
||||
try:
|
||||
doc_root = lxml.etree.parse(str(document_xml)).getroot()
|
||||
namespaces = {"w": self.WORD_2006_NAMESPACE}
|
||||
|
||||
range_starts = {
|
||||
elem.get(f"{{{self.WORD_2006_NAMESPACE}}}id")
|
||||
for elem in doc_root.xpath(
|
||||
".//w:commentRangeStart", namespaces=namespaces
|
||||
)
|
||||
}
|
||||
range_ends = {
|
||||
elem.get(f"{{{self.WORD_2006_NAMESPACE}}}id")
|
||||
for elem in doc_root.xpath(
|
||||
".//w:commentRangeEnd", namespaces=namespaces
|
||||
)
|
||||
}
|
||||
references = {
|
||||
elem.get(f"{{{self.WORD_2006_NAMESPACE}}}id")
|
||||
for elem in doc_root.xpath(
|
||||
".//w:commentReference", namespaces=namespaces
|
||||
)
|
||||
}
|
||||
|
||||
orphaned_ends = range_ends - range_starts
|
||||
for comment_id in sorted(
|
||||
orphaned_ends, key=lambda x: int(x) if x and x.isdigit() else 0
|
||||
):
|
||||
errors.append(
|
||||
f' document.xml: commentRangeEnd id="{comment_id}" has no matching commentRangeStart'
|
||||
)
|
||||
|
||||
orphaned_starts = range_starts - range_ends
|
||||
for comment_id in sorted(
|
||||
orphaned_starts, key=lambda x: int(x) if x and x.isdigit() else 0
|
||||
):
|
||||
errors.append(
|
||||
f' document.xml: commentRangeStart id="{comment_id}" has no matching commentRangeEnd'
|
||||
)
|
||||
|
||||
comment_ids = set()
|
||||
if comments_xml and comments_xml.exists():
|
||||
comments_root = lxml.etree.parse(str(comments_xml)).getroot()
|
||||
comment_ids = {
|
||||
elem.get(f"{{{self.WORD_2006_NAMESPACE}}}id")
|
||||
for elem in comments_root.xpath(
|
||||
".//w:comment", namespaces=namespaces
|
||||
)
|
||||
}
|
||||
|
||||
marker_ids = range_starts | range_ends | references
|
||||
invalid_refs = marker_ids - comment_ids
|
||||
for comment_id in sorted(
|
||||
invalid_refs, key=lambda x: int(x) if x and x.isdigit() else 0
|
||||
):
|
||||
if comment_id:
|
||||
errors.append(
|
||||
f' document.xml: marker id="{comment_id}" references non-existent comment'
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(f" Error parsing XML: {e}")
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - {len(errors)} comment marker violations:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All comment markers properly paired")
|
||||
return True
|
||||
|
||||
def repair(self) -> int:
|
||||
repairs = super().repair()
|
||||
repairs += self.repair_durableId()
|
||||
return repairs
|
||||
|
||||
def repair_durableId(self) -> int:
|
||||
DURABLE_ID_ATTRS = ("w16cid:durableId", "w16cex:durableId")
|
||||
repairs = 0
|
||||
renames: dict = {}
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
content = xml_file.read_text(encoding="utf-8")
|
||||
dom = defusedxml.minidom.parseString(content)
|
||||
is_numbering = xml_file.name == "numbering.xml"
|
||||
base = 10 if is_numbering else 16
|
||||
pending = []
|
||||
seen_in_file = set()
|
||||
modified = False
|
||||
|
||||
for elem in dom.getElementsByTagName("*"):
|
||||
for attr_name in DURABLE_ID_ATTRS:
|
||||
if not elem.hasAttribute(attr_name):
|
||||
continue
|
||||
|
||||
durable_id = elem.getAttribute(attr_name)
|
||||
try:
|
||||
key = self._parse_id_value(durable_id, base=base)
|
||||
needs_repair = key >= 0x7FFFFFFF
|
||||
except ValueError:
|
||||
key = durable_id
|
||||
needs_repair = True
|
||||
|
||||
if needs_repair:
|
||||
if key in seen_in_file:
|
||||
value = random.randint(1, 0x7FFFFFFE)
|
||||
else:
|
||||
seen_in_file.add(key)
|
||||
if key not in renames:
|
||||
renames[key] = random.randint(1, 0x7FFFFFFE)
|
||||
value = renames[key]
|
||||
new_id = str(value) if is_numbering else f"{value:08X}"
|
||||
|
||||
elem.setAttribute(attr_name, new_id)
|
||||
pending.append(
|
||||
f" Repaired: {xml_file.name}: durableId {durable_id} → {new_id}"
|
||||
)
|
||||
modified = True
|
||||
|
||||
if modified:
|
||||
xml_file.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
for message in pending:
|
||||
print(message)
|
||||
repairs += len(pending)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return repairs
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise RuntimeError("This module should not be run directly.")
|
||||
@@ -0,0 +1,441 @@
|
||||
"""
|
||||
Validator for PowerPoint presentation XML files against XSD schemas.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from helpers import opc_target, rels_source_part, safe_extract
|
||||
|
||||
from .base import BaseSchemaValidator
|
||||
|
||||
|
||||
class PPTXSchemaValidator(BaseSchemaValidator):
|
||||
|
||||
PRESENTATIONML_NAMESPACE = (
|
||||
"http://schemas.openxmlformats.org/presentationml/2006/main"
|
||||
)
|
||||
|
||||
ELEMENT_RELATIONSHIP_TYPES = {
|
||||
"sldid": "slide",
|
||||
"sldmasterid": "slidemaster",
|
||||
"notesmasterid": "notesmaster",
|
||||
"sldlayoutid": "slidelayout",
|
||||
"themeid": "theme",
|
||||
"tablestyleid": "tablestyles",
|
||||
}
|
||||
|
||||
def validate(self):
|
||||
if not self.validate_xml():
|
||||
return False
|
||||
|
||||
all_valid = True
|
||||
if not self.validate_namespaces():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_unique_ids():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_uuid_ids():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_file_references():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_slide_layout_ids():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_content_types():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_against_xsd():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_notes_slide_references():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_all_relationship_ids():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_no_duplicate_slide_layouts():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_master_theme_uniqueness():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_charts():
|
||||
all_valid = False
|
||||
|
||||
if not self.validate_slides():
|
||||
all_valid = False
|
||||
|
||||
return all_valid
|
||||
|
||||
def _package_map(self) -> dict:
|
||||
wanted = []
|
||||
wanted += list(self.unpacked_dir.glob("[[]Content_Types[]].xml"))
|
||||
wanted += list(self.unpacked_dir.glob("ppt/presentation.xml"))
|
||||
wanted += list(self.unpacked_dir.glob("ppt/theme/*.xml"))
|
||||
wanted += list(self.unpacked_dir.glob("ppt/theme/_rels/*.rels"))
|
||||
wanted += list(self.unpacked_dir.glob("ppt/charts/chart*.xml"))
|
||||
for group in ("slideMasters", "notesMasters", "handoutMasters"):
|
||||
wanted += list(self.unpacked_dir.glob(f"ppt/{group}/*.xml"))
|
||||
wanted += list(self.unpacked_dir.glob(f"ppt/{group}/_rels/*.rels"))
|
||||
return {
|
||||
p.relative_to(self.unpacked_dir).as_posix(): p.read_bytes()
|
||||
for p in wanted
|
||||
if p.is_file()
|
||||
}
|
||||
|
||||
def validate_master_theme_uniqueness(self):
|
||||
from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes
|
||||
|
||||
shared = live_shared_master_themes(self._package_map())
|
||||
if shared:
|
||||
print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:")
|
||||
for message in shared:
|
||||
print(f" {message}")
|
||||
if any(m.startswith(_NOTES_MASTERS) for m in shared):
|
||||
print(" Fix: in ppt/presentation.xml, move <p:notesMasterIdLst> back to "
|
||||
"directly after <p:sldIdLst>. PowerPoint reads that happily.")
|
||||
else:
|
||||
print(" Fix: give each master its own theme part.")
|
||||
return False
|
||||
|
||||
if self.verbose:
|
||||
print("PASSED - No master shares a theme part in a way PowerPoint refuses")
|
||||
return True
|
||||
|
||||
def validate_charts(self):
|
||||
from helpers.pptx_chart import find_chart_problems
|
||||
|
||||
problems = find_chart_problems(self._package_map())
|
||||
if problems:
|
||||
print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:")
|
||||
for message in problems:
|
||||
print(f" {message}")
|
||||
return False
|
||||
|
||||
if self.verbose:
|
||||
print("PASSED - Charts satisfy the constraints PowerPoint enforces")
|
||||
return True
|
||||
|
||||
def _original_slide_defects(self, schema) -> set[str]:
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
from helpers.pptx_slide import SLIDE_PART_RE, fatal_slide_errors
|
||||
|
||||
if self.original_file is None:
|
||||
return set()
|
||||
|
||||
found: set[str] = set()
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
try:
|
||||
with zipfile.ZipFile(self.original_file, "r") as zf:
|
||||
safe_extract(zf, temp_path)
|
||||
except (zipfile.BadZipFile, ValueError, OSError):
|
||||
return set()
|
||||
|
||||
for part in sorted(temp_path.rglob("*.xml")):
|
||||
relative = part.relative_to(temp_path).as_posix()
|
||||
if not SLIDE_PART_RE.fullmatch(relative):
|
||||
continue
|
||||
ok, errors = self._validate_single_file_xsd(
|
||||
part.resolve(), temp_path.resolve(), schema_path=schema
|
||||
)
|
||||
if ok is None or ok or not errors:
|
||||
continue
|
||||
found |= set(fatal_slide_errors(set(errors)))
|
||||
return found
|
||||
|
||||
def validate_slides(self):
|
||||
from helpers.pptx_slide import (
|
||||
SLIDE_PART_RE,
|
||||
fatal_slide_errors,
|
||||
is_schema_verdict,
|
||||
)
|
||||
|
||||
schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"]
|
||||
inherited = self._original_slide_defects(schema)
|
||||
problems: list[str] = []
|
||||
broken: list[str] = []
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
relative = xml_file.relative_to(self.unpacked_dir).as_posix()
|
||||
if not SLIDE_PART_RE.fullmatch(relative):
|
||||
continue
|
||||
ok, errors = self._validate_single_file_xsd(
|
||||
xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema
|
||||
)
|
||||
if ok is None or not errors:
|
||||
continue
|
||||
|
||||
unreadable = [f"{relative}: {e}" for e in errors if not is_schema_verdict(e)]
|
||||
if unreadable:
|
||||
broken.extend(unreadable)
|
||||
continue
|
||||
if ok:
|
||||
continue
|
||||
|
||||
for message in fatal_slide_errors(set(errors)):
|
||||
if message in inherited:
|
||||
continue
|
||||
problems.append(f"{relative}: {message}")
|
||||
|
||||
if broken:
|
||||
print(f"FAILED - Could not check {len(broken)} slide part(s):")
|
||||
for message in sorted(broken):
|
||||
print(f" {message[:240]}")
|
||||
|
||||
if problems:
|
||||
print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:")
|
||||
for message in sorted(problems):
|
||||
print(f" {message[:240]}")
|
||||
|
||||
if broken or problems:
|
||||
return False
|
||||
|
||||
if self.verbose:
|
||||
print("PASSED - Slide XML has none of the defects PowerPoint refuses")
|
||||
return True
|
||||
|
||||
def _get_schema_path(self, xml_file):
|
||||
if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"):
|
||||
return None
|
||||
return super()._get_schema_path(xml_file)
|
||||
|
||||
def _preprocess_for_schema(self, xml_doc, relative_path):
|
||||
if relative_path.as_posix() != "ppt/presentation.xml":
|
||||
return xml_doc
|
||||
|
||||
root = xml_doc.getroot()
|
||||
ns = f"{{{self.PRESENTATIONML_NAMESPACE}}}"
|
||||
notes = root.find(f"{ns}notesMasterIdLst")
|
||||
slides = root.find(f"{ns}sldIdLst")
|
||||
if notes is None or slides is None:
|
||||
return xml_doc
|
||||
|
||||
children = list(root)
|
||||
if children.index(notes) < children.index(slides):
|
||||
return xml_doc
|
||||
|
||||
root.remove(notes)
|
||||
root.insert(list(root).index(slides), notes)
|
||||
return xml_doc
|
||||
|
||||
def validate_uuid_ids(self):
|
||||
import lxml.etree
|
||||
|
||||
errors = []
|
||||
uuid_pattern = re.compile(
|
||||
r"^[\{\(]?[0-9A-Fa-f]{8}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{12}[\}\)]?$"
|
||||
)
|
||||
|
||||
for xml_file in self.xml_files:
|
||||
try:
|
||||
root = lxml.etree.parse(str(xml_file)).getroot()
|
||||
|
||||
for elem in root.iter():
|
||||
for attr, value in elem.attrib.items():
|
||||
attr_name = attr.split("}")[-1].lower()
|
||||
if attr_name == "id" or attr_name.endswith("id"):
|
||||
if self._looks_like_uuid(value):
|
||||
if not uuid_pattern.match(value):
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {elem.sourceline}: ID '{value}' appears to be a UUID but contains invalid hex characters"
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} UUID ID validation errors:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All UUID-like IDs contain valid hex values")
|
||||
return True
|
||||
|
||||
def _looks_like_uuid(self, value):
|
||||
clean_value = value.strip("{}()").replace("-", "")
|
||||
return len(clean_value) == 32 and all(c.isalnum() for c in clean_value)
|
||||
|
||||
def validate_slide_layout_ids(self):
|
||||
import lxml.etree
|
||||
|
||||
errors = []
|
||||
|
||||
slide_masters = list(self.unpacked_dir.glob("ppt/slideMasters/*.xml"))
|
||||
|
||||
if not slide_masters:
|
||||
if self.verbose:
|
||||
print("PASSED - No slide masters found")
|
||||
return True
|
||||
|
||||
for slide_master in slide_masters:
|
||||
try:
|
||||
root = lxml.etree.parse(str(slide_master)).getroot()
|
||||
|
||||
rels_file = slide_master.parent / "_rels" / f"{slide_master.name}.rels"
|
||||
|
||||
if not rels_file.exists():
|
||||
errors.append(
|
||||
f" {slide_master.relative_to(self.unpacked_dir)}: "
|
||||
f"Missing relationships file: {rels_file.relative_to(self.unpacked_dir)}"
|
||||
)
|
||||
continue
|
||||
|
||||
rels_root = lxml.etree.parse(str(rels_file)).getroot()
|
||||
|
||||
valid_layout_rids = set()
|
||||
for rel in rels_root.findall(
|
||||
f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship"
|
||||
):
|
||||
rel_type = rel.get("Type", "")
|
||||
if "slideLayout" in rel_type:
|
||||
valid_layout_rids.add(rel.get("Id"))
|
||||
|
||||
for sld_layout_id in root.findall(
|
||||
f".//{{{self.PRESENTATIONML_NAMESPACE}}}sldLayoutId"
|
||||
):
|
||||
r_id = sld_layout_id.get(
|
||||
f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
|
||||
)
|
||||
layout_id = sld_layout_id.get("id")
|
||||
|
||||
if r_id and r_id not in valid_layout_rids:
|
||||
errors.append(
|
||||
f" {slide_master.relative_to(self.unpacked_dir)}: "
|
||||
f"Line {sld_layout_id.sourceline}: sldLayoutId with id='{layout_id}' "
|
||||
f"references r:id='{r_id}' which is not found in slide layout relationships"
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {slide_master.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print(f"FAILED - Found {len(errors)} slide layout ID validation errors:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
print(
|
||||
"Remove invalid references or add missing slide layouts to the relationships file."
|
||||
)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All slide layout IDs reference valid slide layouts")
|
||||
return True
|
||||
|
||||
def validate_no_duplicate_slide_layouts(self):
|
||||
import lxml.etree
|
||||
|
||||
errors = []
|
||||
slide_rels_files = list(self.unpacked_dir.glob("ppt/slides/_rels/*.xml.rels"))
|
||||
|
||||
for rels_file in slide_rels_files:
|
||||
try:
|
||||
root = lxml.etree.parse(str(rels_file)).getroot()
|
||||
|
||||
layout_rels = [
|
||||
rel
|
||||
for rel in root.findall(
|
||||
f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship"
|
||||
)
|
||||
if "slideLayout" in rel.get("Type", "")
|
||||
]
|
||||
|
||||
if len(layout_rels) > 1:
|
||||
errors.append(
|
||||
f" {rels_file.relative_to(self.unpacked_dir)}: has {len(layout_rels)} slideLayout references"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
errors.append(
|
||||
f" {rels_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print("FAILED - Found slides with duplicate slideLayout references:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All slides have exactly one slideLayout reference")
|
||||
return True
|
||||
|
||||
def validate_notes_slide_references(self):
|
||||
import lxml.etree
|
||||
|
||||
errors = []
|
||||
notes_slide_references = {}
|
||||
|
||||
slide_rels_files = list(self.unpacked_dir.glob("ppt/slides/_rels/*.xml.rels"))
|
||||
|
||||
if not slide_rels_files:
|
||||
if self.verbose:
|
||||
print("PASSED - No slide relationship files found")
|
||||
return True
|
||||
|
||||
for rels_file in slide_rels_files:
|
||||
try:
|
||||
root = lxml.etree.parse(str(rels_file)).getroot()
|
||||
|
||||
for rel in root.findall(
|
||||
f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship"
|
||||
):
|
||||
rel_type = rel.get("Type", "")
|
||||
if "notesSlide" in rel_type:
|
||||
part = opc_target(
|
||||
rel.get("Target", ""),
|
||||
rels_source_part(rels_file, self.unpacked_dir),
|
||||
rel.get("TargetMode", ""),
|
||||
)
|
||||
if part:
|
||||
slide_name = rels_file.stem.replace(
|
||||
".xml", ""
|
||||
)
|
||||
|
||||
notes_slide_references.setdefault(part, []).append(
|
||||
(slide_name, rels_file)
|
||||
)
|
||||
|
||||
except (lxml.etree.XMLSyntaxError, Exception) as e:
|
||||
errors.append(
|
||||
f" {rels_file.relative_to(self.unpacked_dir)}: Error: {e}"
|
||||
)
|
||||
|
||||
for target, references in notes_slide_references.items():
|
||||
if len(references) > 1:
|
||||
slide_names = [ref[0] for ref in references]
|
||||
errors.append(
|
||||
f" Notes slide '{target}' is referenced by multiple slides: {', '.join(slide_names)}"
|
||||
)
|
||||
for slide_name, rels_file in references:
|
||||
errors.append(f" - {rels_file.relative_to(self.unpacked_dir)}")
|
||||
|
||||
if errors:
|
||||
print(
|
||||
f"FAILED - Found {len([e for e in errors if not e.startswith(' ')])} notes slide reference validation errors:"
|
||||
)
|
||||
for error in errors:
|
||||
print(error)
|
||||
print("Each slide may optionally have its own slide file.")
|
||||
return False
|
||||
else:
|
||||
if self.verbose:
|
||||
print("PASSED - All notes slide references are unique")
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise RuntimeError("This module should not be run directly.")
|
||||
@@ -0,0 +1,299 @@
|
||||
"""
|
||||
Validator for tracked changes in Word documents.
|
||||
|
||||
Detects untracked edits in word/document.xml: text that differs from the
|
||||
original without a <w:ins>/<w:del> wrapper recording it. The tracked changes
|
||||
that are new relative to the original are undone, and the result is compared
|
||||
against the original; whatever text still differs was edited without being
|
||||
tracked.
|
||||
|
||||
Only the document body is compared. Headers, footers, footnotes and endnotes
|
||||
are separate parts and are not checked.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
from defusedxml.common import DefusedXmlException
|
||||
|
||||
from helpers import rendered_text, safe_extract
|
||||
|
||||
|
||||
class RedliningValidator:
|
||||
|
||||
def __init__(self, unpacked_dir, original_docx, verbose=False):
|
||||
self.unpacked_dir = Path(unpacked_dir)
|
||||
self.original_docx = Path(original_docx)
|
||||
self.verbose = verbose
|
||||
self.namespaces = {
|
||||
"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
}
|
||||
|
||||
def repair(self) -> int:
|
||||
return 0
|
||||
|
||||
def validate(self):
|
||||
modified_file = self.unpacked_dir / "word" / "document.xml"
|
||||
if not modified_file.exists():
|
||||
print(f"FAILED - Modified document.xml not found at {modified_file}")
|
||||
return False
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(self.original_docx, "r") as zip_ref:
|
||||
safe_extract(zip_ref, temp_path)
|
||||
except Exception as e:
|
||||
print(f"FAILED - Error unpacking original docx: {e}")
|
||||
return False
|
||||
|
||||
original_file = temp_path / "word" / "document.xml"
|
||||
if not original_file.exists():
|
||||
print(
|
||||
f"FAILED - Original document.xml not found in {self.original_docx}"
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
modified_tree = ET.parse(modified_file)
|
||||
modified_root = modified_tree.getroot()
|
||||
original_tree = ET.parse(original_file)
|
||||
original_root = original_tree.getroot()
|
||||
except (ET.ParseError, DefusedXmlException) as e:
|
||||
print(f"FAILED - Error parsing XML files: {e}")
|
||||
return False
|
||||
|
||||
new_changes = self._new_tracked_changes(original_root, modified_root)
|
||||
self._remove_tracked_changes(modified_root, new_changes)
|
||||
|
||||
modified_text = self._extract_text_content(modified_root)
|
||||
original_text = self._extract_text_content(original_root)
|
||||
|
||||
if modified_text != original_text:
|
||||
error_message = self._generate_detailed_diff(
|
||||
original_text, modified_text
|
||||
)
|
||||
print(error_message)
|
||||
return False
|
||||
|
||||
if self.verbose:
|
||||
print(
|
||||
f"PASSED - All {len(new_changes)} change(s) against the original "
|
||||
"are properly tracked"
|
||||
)
|
||||
return True
|
||||
|
||||
def _tracked_change_elements(self, root):
|
||||
ins_tag = f"{{{self.namespaces['w']}}}ins"
|
||||
del_tag = f"{{{self.namespaces['w']}}}del"
|
||||
return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)]
|
||||
|
||||
def _rendered_text(self, elem):
|
||||
preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve"
|
||||
return rendered_text(elem.text or "", preserve)
|
||||
|
||||
def _text_elements(self, elem):
|
||||
w = self.namespaces["w"]
|
||||
return [
|
||||
node
|
||||
for node in elem.iter()
|
||||
if node.tag in (f"{{{w}}}t", f"{{{w}}}delText")
|
||||
]
|
||||
|
||||
def _tracked_change_key(self, elem):
|
||||
w = self.namespaces["w"]
|
||||
text = "".join(self._rendered_text(node) for node in self._text_elements(elem))
|
||||
return (elem.tag, elem.get(f"{{{w}}}author"), elem.get(f"{{{w}}}date"), text)
|
||||
|
||||
def _new_tracked_changes(self, original_root, modified_root):
|
||||
original = self._tracked_change_elements(original_root)
|
||||
modified = self._tracked_change_elements(modified_root)
|
||||
|
||||
pool = {}
|
||||
for elem in original:
|
||||
pool.setdefault(self._tracked_change_key(elem), []).append(elem)
|
||||
|
||||
matched, leftover = set(), []
|
||||
for elem in modified:
|
||||
bucket = pool.get(self._tracked_change_key(elem))
|
||||
if bucket:
|
||||
matched.add(bucket.pop())
|
||||
else:
|
||||
leftover.append(elem)
|
||||
|
||||
def group(elem):
|
||||
return self._tracked_change_key(elem)[:3]
|
||||
|
||||
def text_of(elems):
|
||||
return "".join(self._tracked_change_key(e)[3] for e in elems)
|
||||
|
||||
unmatched_original = {}
|
||||
for elem in original:
|
||||
if elem not in matched:
|
||||
unmatched_original.setdefault(group(elem), []).append(elem)
|
||||
|
||||
by_group = {}
|
||||
for elem in leftover:
|
||||
by_group.setdefault(group(elem), []).append(elem)
|
||||
|
||||
new = set()
|
||||
for key, elems in by_group.items():
|
||||
rebuilt = text_of(elems)
|
||||
if rebuilt and rebuilt == text_of(unmatched_original.get(key, [])):
|
||||
continue
|
||||
new.update(elems)
|
||||
return new
|
||||
|
||||
def _generate_detailed_diff(self, original_text, modified_text):
|
||||
error_parts = [
|
||||
"FAILED - Document text doesn't match after removing the tracked changes",
|
||||
"",
|
||||
"Likely causes:",
|
||||
" 1. Modified text inside another author's <w:ins> or <w:del> tags",
|
||||
" 2. Made edits without proper tracked changes",
|
||||
" 3. Didn't nest <w:del> inside <w:ins> when deleting another's insertion",
|
||||
" 4. Rewrote another author's <w:ins>/<w:del> and changed its text on",
|
||||
" the way. A tracked change from the original is recognised by its",
|
||||
" author, date and text; anything that doesn't reproduce one exactly",
|
||||
" reads as new, and the text it carried is reported missing.",
|
||||
"",
|
||||
"For pre-redlined documents, use correct patterns:",
|
||||
" - To reject another's INSERTION: Nest <w:del> inside their <w:ins>",
|
||||
" - To reject PART of one: nest <w:del> around only the runs you reject.",
|
||||
" Their <w:ins> may be split around it, so long as the pieces keep",
|
||||
" their author and date and still spell out the same text.",
|
||||
" - To restore another's DELETION: Add new <w:ins> AFTER their <w:del>",
|
||||
"",
|
||||
]
|
||||
|
||||
git_diff = self._get_git_word_diff(original_text, modified_text)
|
||||
if git_diff:
|
||||
error_parts.extend(["Differences:", "============", git_diff])
|
||||
else:
|
||||
error_parts.append("Unable to generate word diff (git not available)")
|
||||
|
||||
return "\n".join(error_parts)
|
||||
|
||||
def _get_git_word_diff(self, original_text, modified_text):
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
original_file = temp_path / "original.txt"
|
||||
modified_file = temp_path / "modified.txt"
|
||||
|
||||
original_file.write_text(original_text, encoding="utf-8")
|
||||
modified_file.write_text(modified_text, encoding="utf-8")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--word-diff=plain",
|
||||
"--word-diff-regex=.",
|
||||
"-U0",
|
||||
"--no-index",
|
||||
str(original_file),
|
||||
str(modified_file),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.stdout.strip():
|
||||
lines = result.stdout.split("\n")
|
||||
content_lines = []
|
||||
in_content = False
|
||||
for line in lines:
|
||||
if line.startswith("@@"):
|
||||
in_content = True
|
||||
continue
|
||||
if in_content and line.strip():
|
||||
content_lines.append(line)
|
||||
|
||||
if content_lines:
|
||||
return "\n".join(content_lines)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--word-diff=plain",
|
||||
"-U0",
|
||||
"--no-index",
|
||||
str(original_file),
|
||||
str(modified_file),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.stdout.strip():
|
||||
lines = result.stdout.split("\n")
|
||||
content_lines = []
|
||||
in_content = False
|
||||
for line in lines:
|
||||
if line.startswith("@@"):
|
||||
in_content = True
|
||||
continue
|
||||
if in_content and line.strip():
|
||||
content_lines.append(line)
|
||||
return "\n".join(content_lines)
|
||||
|
||||
except (subprocess.CalledProcessError, FileNotFoundError, Exception):
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _remove_tracked_changes(self, root, targets):
|
||||
ins_tag = f"{{{self.namespaces['w']}}}ins"
|
||||
del_tag = f"{{{self.namespaces['w']}}}del"
|
||||
|
||||
for parent in root.iter():
|
||||
to_remove = []
|
||||
for child in parent:
|
||||
if child.tag == ins_tag and child in targets:
|
||||
to_remove.append(child)
|
||||
for elem in to_remove:
|
||||
parent.remove(elem)
|
||||
|
||||
deltext_tag = f"{{{self.namespaces['w']}}}delText"
|
||||
t_tag = f"{{{self.namespaces['w']}}}t"
|
||||
|
||||
for parent in root.iter():
|
||||
to_process = []
|
||||
for child in parent:
|
||||
if child.tag == del_tag and child in targets:
|
||||
to_process.append((child, list(parent).index(child)))
|
||||
|
||||
for del_elem, del_index in reversed(to_process):
|
||||
for elem in del_elem.iter():
|
||||
if elem.tag == deltext_tag:
|
||||
elem.tag = t_tag
|
||||
|
||||
for child in reversed(list(del_elem)):
|
||||
parent.insert(del_index, child)
|
||||
parent.remove(del_elem)
|
||||
|
||||
def _extract_text_content(self, root):
|
||||
p_tag = f"{{{self.namespaces['w']}}}p"
|
||||
t_tag = f"{{{self.namespaces['w']}}}t"
|
||||
|
||||
paragraphs = []
|
||||
for p_elem in root.findall(f".//{p_tag}"):
|
||||
text_parts = []
|
||||
for t_elem in p_elem.findall(f".//{t_tag}"):
|
||||
text_parts.append(self._rendered_text(t_elem))
|
||||
paragraph_text = "".join(text_parts)
|
||||
if paragraph_text:
|
||||
paragraphs.append(paragraph_text)
|
||||
|
||||
return "\n".join(paragraphs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise RuntimeError("This module should not be run directly.")
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Resolve HERMES_HOME for standalone skill scripts.
|
||||
|
||||
Skill scripts may run outside the Hermes process (e.g. system Python,
|
||||
nix env, CI) where ``hermes_constants`` is not importable. This module
|
||||
provides the same ``get_hermes_home()`` and ``display_hermes_home()``
|
||||
contracts as ``hermes_constants`` without requiring it on ``sys.path``.
|
||||
|
||||
When ``hermes_constants`` IS available it is used directly so that any
|
||||
future enhancements (profile resolution, Docker detection, etc.) are
|
||||
picked up automatically. The fallback path replicates the core logic
|
||||
from ``hermes_constants.py`` using only the stdlib.
|
||||
|
||||
All scripts under ``google-workspace/scripts/`` should import from here
|
||||
instead of duplicating the ``HERMES_HOME = Path(os.getenv(...))`` pattern.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from hermes_constants import display_hermes_home as display_hermes_home
|
||||
from hermes_constants import get_hermes_home as get_hermes_home
|
||||
except (ModuleNotFoundError, ImportError):
|
||||
|
||||
def get_hermes_home() -> Path:
|
||||
"""Return the Hermes home directory (default: ~/.hermes).
|
||||
|
||||
Mirrors ``hermes_constants.get_hermes_home()``."""
|
||||
val = os.environ.get("HERMES_HOME", "").strip()
|
||||
return Path(val) if val else Path.home() / ".hermes"
|
||||
|
||||
def display_hermes_home() -> str:
|
||||
"""Return a user-friendly ``~/``-shortened display string.
|
||||
|
||||
Mirrors ``hermes_constants.display_hermes_home()``."""
|
||||
home = get_hermes_home()
|
||||
try:
|
||||
return "~/" + str(home.relative_to(Path.home()))
|
||||
except ValueError:
|
||||
return str(home)
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract text from documents using pymupdf. Lightweight (~25MB), no models.
|
||||
|
||||
Usage:
|
||||
python extract_pymupdf.py document.pdf
|
||||
python extract_pymupdf.py document.pdf --markdown
|
||||
python extract_pymupdf.py document.pdf --pages 0-4
|
||||
python extract_pymupdf.py document.pdf --images output_dir/
|
||||
python extract_pymupdf.py document.pdf --tables
|
||||
python extract_pymupdf.py document.pdf --metadata
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
|
||||
def extract_text(path, pages=None):
|
||||
import pymupdf
|
||||
doc = pymupdf.open(path)
|
||||
page_range = range(len(doc)) if pages is None else pages
|
||||
for i in page_range:
|
||||
if i < len(doc):
|
||||
print(f"\n--- Page {i+1}/{len(doc)} ---\n")
|
||||
print(doc[i].get_text())
|
||||
|
||||
def extract_markdown(path, pages=None):
|
||||
import pymupdf4llm
|
||||
md = pymupdf4llm.to_markdown(path, pages=pages)
|
||||
print(md)
|
||||
|
||||
def extract_tables(path):
|
||||
import pymupdf
|
||||
doc = pymupdf.open(path)
|
||||
for i, page in enumerate(doc):
|
||||
tables = page.find_tables()
|
||||
for j, table in enumerate(tables.tables):
|
||||
print(f"\n--- Page {i+1}, Table {j+1} ---\n")
|
||||
df = table.to_pandas()
|
||||
print(df.to_markdown(index=False))
|
||||
|
||||
def extract_images(path, output_dir):
|
||||
import pymupdf
|
||||
from pathlib import Path
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
doc = pymupdf.open(path)
|
||||
count = 0
|
||||
for i, page in enumerate(doc):
|
||||
for img_idx, img in enumerate(page.get_images(full=True)):
|
||||
xref = img[0]
|
||||
pix = pymupdf.Pixmap(doc, xref)
|
||||
if pix.n >= 5:
|
||||
pix = pymupdf.Pixmap(pymupdf.csRGB, pix)
|
||||
out_path = f"{output_dir}/page{i+1}_img{img_idx+1}.png"
|
||||
pix.save(out_path)
|
||||
count += 1
|
||||
print(f"Extracted {count} images to {output_dir}/")
|
||||
|
||||
def show_metadata(path):
|
||||
import pymupdf
|
||||
doc = pymupdf.open(path)
|
||||
print(json.dumps({
|
||||
"pages": len(doc),
|
||||
"title": doc.metadata.get("title", ""),
|
||||
"author": doc.metadata.get("author", ""),
|
||||
"subject": doc.metadata.get("subject", ""),
|
||||
"creator": doc.metadata.get("creator", ""),
|
||||
"producer": doc.metadata.get("producer", ""),
|
||||
"format": doc.metadata.get("format", ""),
|
||||
}, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
if not args or args[0] in {"-h", "--help"}:
|
||||
print(__doc__)
|
||||
sys.exit(0)
|
||||
|
||||
path = args[0]
|
||||
pages = None
|
||||
|
||||
if "--pages" in args:
|
||||
idx = args.index("--pages")
|
||||
p = args[idx + 1]
|
||||
if "-" in p:
|
||||
start, end = p.split("-")
|
||||
pages = list(range(int(start), int(end) + 1))
|
||||
else:
|
||||
pages = [int(p)]
|
||||
|
||||
if "--metadata" in args:
|
||||
show_metadata(path)
|
||||
elif "--tables" in args:
|
||||
extract_tables(path)
|
||||
elif "--images" in args:
|
||||
idx = args.index("--images")
|
||||
output_dir = args[idx + 1] if idx + 1 < len(args) else "./images"
|
||||
extract_images(path, output_dir)
|
||||
elif "--markdown" in args:
|
||||
extract_markdown(path, pages=pages)
|
||||
else:
|
||||
extract_text(path, pages=pages)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Add a new slide to an unpacked PPTX directory.
|
||||
|
||||
Usage: python add_slide.py <unpacked_dir> <source>
|
||||
|
||||
The source can be:
|
||||
- A slide file (e.g., slide2.xml) - duplicates the slide
|
||||
- A layout file (e.g., slideLayout2.xml) - creates from layout
|
||||
|
||||
Examples:
|
||||
python add_slide.py unpacked/ slide2.xml
|
||||
# Duplicates slide2, creates slide5.xml
|
||||
|
||||
python add_slide.py unpacked/ slideLayout2.xml
|
||||
# Creates slide5.xml from slideLayout2.xml
|
||||
|
||||
To see available layouts: ls unpacked/ppt/slideLayouts/
|
||||
|
||||
Prints the <p:sldId> element to add to presentation.xml.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_next_slide_number(slides_dir: Path) -> int:
|
||||
existing = [int(m.group(1)) for f in slides_dir.glob("slide*.xml")
|
||||
if (m := re.match(r"slide(\d+)\.xml", f.name))]
|
||||
return max(existing) + 1 if existing else 1
|
||||
|
||||
|
||||
def create_slide_from_layout(unpacked_dir: Path, layout_file: str) -> None:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
rels_dir = slides_dir / "_rels"
|
||||
layouts_dir = unpacked_dir / "ppt" / "slideLayouts"
|
||||
|
||||
layout_path = layouts_dir / layout_file
|
||||
if not layout_path.exists():
|
||||
print(f"Error: {layout_path} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
next_num = get_next_slide_number(slides_dir)
|
||||
dest = f"slide{next_num}.xml"
|
||||
dest_slide = slides_dir / dest
|
||||
dest_rels = rels_dir / f"{dest}.rels"
|
||||
|
||||
slide_xml = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<p:sld xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main">
|
||||
<p:cSld>
|
||||
<p:spTree>
|
||||
<p:nvGrpSpPr>
|
||||
<p:cNvPr id="1" name=""/>
|
||||
<p:cNvGrpSpPr/>
|
||||
<p:nvPr/>
|
||||
</p:nvGrpSpPr>
|
||||
<p:grpSpPr>
|
||||
<a:xfrm>
|
||||
<a:off x="0" y="0"/>
|
||||
<a:ext cx="0" cy="0"/>
|
||||
<a:chOff x="0" y="0"/>
|
||||
<a:chExt cx="0" cy="0"/>
|
||||
</a:xfrm>
|
||||
</p:grpSpPr>
|
||||
</p:spTree>
|
||||
</p:cSld>
|
||||
<p:clrMapOvr>
|
||||
<a:masterClrMapping/>
|
||||
</p:clrMapOvr>
|
||||
</p:sld>'''
|
||||
dest_slide.write_text(slide_xml, encoding="utf-8")
|
||||
|
||||
rels_dir.mkdir(exist_ok=True)
|
||||
rels_xml = f'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" Target="../slideLayouts/{layout_file}"/>
|
||||
</Relationships>'''
|
||||
dest_rels.write_text(rels_xml, encoding="utf-8")
|
||||
|
||||
_add_to_content_types(unpacked_dir, dest)
|
||||
|
||||
rid = _add_to_presentation_rels(unpacked_dir, dest)
|
||||
|
||||
next_slide_id = _get_next_slide_id(unpacked_dir)
|
||||
|
||||
print(f"Created {dest} from {layout_file}")
|
||||
print(f'Add to presentation.xml <p:sldIdLst>: <p:sldId id="{next_slide_id}" r:id="{rid}"/>')
|
||||
|
||||
|
||||
def duplicate_slide(unpacked_dir: Path, source: str) -> None:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
rels_dir = slides_dir / "_rels"
|
||||
|
||||
source_slide = slides_dir / source
|
||||
|
||||
if not source_slide.exists():
|
||||
print(f"Error: {source_slide} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
next_num = get_next_slide_number(slides_dir)
|
||||
dest = f"slide{next_num}.xml"
|
||||
dest_slide = slides_dir / dest
|
||||
|
||||
source_rels = rels_dir / f"{source}.rels"
|
||||
dest_rels = rels_dir / f"{dest}.rels"
|
||||
|
||||
shutil.copy2(source_slide, dest_slide)
|
||||
|
||||
if source_rels.exists():
|
||||
shutil.copy2(source_rels, dest_rels)
|
||||
|
||||
rels_content = dest_rels.read_text(encoding="utf-8")
|
||||
rels_content = re.sub(
|
||||
r'\s*<Relationship[^>]*Type="[^"]*notesSlide"[^>]*/>\s*',
|
||||
"\n",
|
||||
rels_content,
|
||||
)
|
||||
dest_rels.write_text(rels_content, encoding="utf-8")
|
||||
|
||||
_add_to_content_types(unpacked_dir, dest)
|
||||
|
||||
rid = _add_to_presentation_rels(unpacked_dir, dest)
|
||||
|
||||
next_slide_id = _get_next_slide_id(unpacked_dir)
|
||||
|
||||
print(f"Created {dest} from {source}")
|
||||
print(f'Add to presentation.xml <p:sldIdLst>: <p:sldId id="{next_slide_id}" r:id="{rid}"/>')
|
||||
|
||||
|
||||
def _add_to_content_types(unpacked_dir: Path, dest: str) -> None:
|
||||
content_types_path = unpacked_dir / "[Content_Types].xml"
|
||||
content_types = content_types_path.read_text(encoding="utf-8")
|
||||
|
||||
new_override = f'<Override PartName="/ppt/slides/{dest}" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'
|
||||
|
||||
if f"/ppt/slides/{dest}" not in content_types:
|
||||
content_types = content_types.replace("</Types>", f" {new_override}\n</Types>")
|
||||
content_types_path.write_text(content_types, encoding="utf-8")
|
||||
|
||||
|
||||
def _add_to_presentation_rels(unpacked_dir: Path, dest: str) -> str:
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
pres_rels = pres_rels_path.read_text(encoding="utf-8")
|
||||
|
||||
rids = [int(m) for m in re.findall(r'Id="rId(\d+)"', pres_rels)]
|
||||
next_rid = max(rids) + 1 if rids else 1
|
||||
rid = f"rId{next_rid}"
|
||||
|
||||
new_rel = f'<Relationship Id="{rid}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" Target="slides/{dest}"/>'
|
||||
|
||||
if f"slides/{dest}" not in pres_rels:
|
||||
pres_rels = pres_rels.replace("</Relationships>", f" {new_rel}\n</Relationships>")
|
||||
pres_rels_path.write_text(pres_rels, encoding="utf-8")
|
||||
|
||||
return rid
|
||||
|
||||
|
||||
def _get_next_slide_id(unpacked_dir: Path) -> int:
|
||||
pres_path = unpacked_dir / "ppt" / "presentation.xml"
|
||||
pres_content = pres_path.read_text(encoding="utf-8")
|
||||
slide_ids = [int(m) for m in re.findall(r'<p:sldId[^>]*id="(\d+)"', pres_content)]
|
||||
return max(slide_ids) + 1 if slide_ids else 256
|
||||
|
||||
|
||||
def parse_source(source: str) -> tuple[str, str | None]:
|
||||
if source.startswith("slideLayout") and source.endswith(".xml"):
|
||||
return ("layout", source)
|
||||
|
||||
return ("slide", None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python add_slide.py <unpacked_dir> <source>", file=sys.stderr)
|
||||
print("", file=sys.stderr)
|
||||
print("Source can be:", file=sys.stderr)
|
||||
print(" slide2.xml - duplicate an existing slide", file=sys.stderr)
|
||||
print(" slideLayout2.xml - create from a layout template", file=sys.stderr)
|
||||
print("", file=sys.stderr)
|
||||
print("To see available layouts: ls <unpacked_dir>/ppt/slideLayouts/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
unpacked_dir = Path(sys.argv[1])
|
||||
source = sys.argv[2]
|
||||
|
||||
if not unpacked_dir.exists():
|
||||
print(f"Error: {unpacked_dir} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
source_type, layout_file = parse_source(source)
|
||||
|
||||
if source_type == "layout" and layout_file is not None:
|
||||
create_slide_from_layout(unpacked_dir, layout_file)
|
||||
else:
|
||||
duplicate_slide(unpacked_dir, source)
|
||||
@@ -0,0 +1,481 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Google Workspace OAuth2 setup for Hermes Agent.
|
||||
|
||||
Fully non-interactive — designed to be driven by the agent via terminal commands.
|
||||
The agent mediates between this script and the user (works on CLI, Telegram, Discord, etc.)
|
||||
|
||||
Commands:
|
||||
setup.py --check # Is auth valid? Exit 0 = yes, 1 = no
|
||||
setup.py --client-secret /path/to.json # Store OAuth client credentials
|
||||
setup.py --auth-url # Print the OAuth URL for user to visit
|
||||
setup.py --auth-code CODE # Exchange auth code for token
|
||||
setup.py --revoke # Revoke and delete stored token
|
||||
setup.py --install-deps # Install Python dependencies only
|
||||
|
||||
Agent workflow:
|
||||
1. Run --check. If exit 0, auth is good — skip setup.
|
||||
2. Ask user for client_secret.json path. Run --client-secret PATH.
|
||||
3. Run --auth-url. Send the printed URL to the user.
|
||||
4. User opens URL, authorizes, gets redirected to a page with a code.
|
||||
5. User pastes the code. Agent runs --auth-code CODE.
|
||||
6. Run --check to verify. Done.
|
||||
"""
|
||||
|
||||
from __future__ import annotations # allow PEP 604 `X | None` on Python 3.9+
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure sibling modules (_hermes_home) are importable when run standalone.
|
||||
_SCRIPTS_DIR = str(Path(__file__).resolve().parent)
|
||||
if _SCRIPTS_DIR not in sys.path:
|
||||
sys.path.insert(0, _SCRIPTS_DIR)
|
||||
|
||||
from _hermes_home import display_hermes_home, get_hermes_home
|
||||
|
||||
HERMES_HOME = get_hermes_home()
|
||||
TOKEN_PATH = HERMES_HOME / "google_token.json"
|
||||
CLIENT_SECRET_PATH = HERMES_HOME / "google_client_secret.json"
|
||||
PENDING_AUTH_PATH = HERMES_HOME / "google_oauth_pending.json"
|
||||
|
||||
SCOPES = [
|
||||
"https://www.googleapis.com/auth/gmail.readonly",
|
||||
"https://www.googleapis.com/auth/gmail.send",
|
||||
"https://www.googleapis.com/auth/gmail.modify",
|
||||
"https://www.googleapis.com/auth/calendar",
|
||||
"https://www.googleapis.com/auth/drive",
|
||||
"https://www.googleapis.com/auth/contacts.readonly",
|
||||
"https://www.googleapis.com/auth/spreadsheets",
|
||||
"https://www.googleapis.com/auth/documents",
|
||||
]
|
||||
|
||||
REQUIRED_PACKAGES = ["google-api-python-client", "google-auth-oauthlib", "google-auth-httplib2"]
|
||||
|
||||
# OAuth redirect for "out of band" manual code copy flow.
|
||||
# Google deprecated OOB, so we use a localhost redirect and tell the user to
|
||||
# copy the code from the browser's URL bar (or the page body).
|
||||
REDIRECT_URI = "http://localhost:1"
|
||||
|
||||
|
||||
def _normalize_authorized_user_payload(payload: dict) -> dict:
|
||||
normalized = dict(payload)
|
||||
if not normalized.get("type"):
|
||||
normalized["type"] = "authorized_user"
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_token_payload(path: Path = TOKEN_PATH) -> dict:
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _missing_scopes_from_payload(payload: dict) -> list[str]:
|
||||
raw = payload.get("scopes") or payload.get("scope")
|
||||
if not raw:
|
||||
return []
|
||||
granted = {s.strip() for s in (raw.split() if isinstance(raw, str) else raw) if s.strip()}
|
||||
return sorted(scope for scope in SCOPES if scope not in granted)
|
||||
|
||||
|
||||
def _format_missing_scopes(missing_scopes: list[str]) -> str:
|
||||
bullets = "\n".join(f" - {scope}" for scope in missing_scopes)
|
||||
return (
|
||||
"Token is valid but missing required Google Workspace scopes:\n"
|
||||
f"{bullets}\n"
|
||||
"Run the Google Workspace setup again from this same Hermes profile to refresh consent."
|
||||
)
|
||||
|
||||
|
||||
def install_deps():
|
||||
"""Install Google API packages if missing. Returns True on success."""
|
||||
try:
|
||||
import googleapiclient # noqa: F401
|
||||
import google_auth_oauthlib # noqa: F401
|
||||
print("Dependencies already installed.")
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
print("Installing Google API dependencies...")
|
||||
|
||||
# First choice: pip in the current interpreter. Works for most installs.
|
||||
try:
|
||||
subprocess.check_call(
|
||||
[sys.executable, "-m", "pip", "install", "--quiet"] + REQUIRED_PACKAGES,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
print("Dependencies installed.")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
pip_error = e
|
||||
|
||||
# Fallback: the interpreter has no pip (the Hermes Docker image's venv is
|
||||
# built with `uv sync`, which does not bootstrap pip). `uv pip install
|
||||
# --python <interpreter>` installs into that exact interpreter without
|
||||
# needing pip present. Targeting sys.executable keeps us on the venv the
|
||||
# script is actually running under, rather than guessing.
|
||||
uv = shutil.which("uv")
|
||||
if uv:
|
||||
try:
|
||||
subprocess.check_call(
|
||||
[uv, "pip", "install", "--python", sys.executable, "--quiet"]
|
||||
+ REQUIRED_PACKAGES,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
print("Dependencies installed.")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"ERROR: Failed to install dependencies via uv: {e}")
|
||||
print(f"Manually: {uv} pip install --python {sys.executable} {' '.join(REQUIRED_PACKAGES)}")
|
||||
return False
|
||||
|
||||
print(f"ERROR: Failed to install dependencies: {pip_error}")
|
||||
print(
|
||||
"On environments without pip (e.g. Nix, or the Hermes Docker image's "
|
||||
"uv-managed venv), install the optional extra instead:"
|
||||
)
|
||||
print(" pip install 'hermes-agent[google]'")
|
||||
print(f"Or manually: {sys.executable} -m pip install {' '.join(REQUIRED_PACKAGES)}")
|
||||
return False
|
||||
|
||||
|
||||
def _ensure_deps():
|
||||
"""Check deps are available, install if not, exit on failure."""
|
||||
try:
|
||||
import googleapiclient # noqa: F401
|
||||
import google_auth_oauthlib # noqa: F401
|
||||
except ImportError:
|
||||
if not install_deps():
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def check_auth_live():
|
||||
"""Check auth with a real API call to detect disabled_client/account issues."""
|
||||
# quiet=True suppresses the "AUTHENTICATED" print from check_auth so the
|
||||
# final status line reflects the live-call outcome (OK or FAILED).
|
||||
if not check_auth(quiet=True):
|
||||
return False
|
||||
try:
|
||||
from googleapiclient.discovery import build
|
||||
from google.oauth2.credentials import Credentials
|
||||
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH))
|
||||
service = build("calendar", "v3", credentials=creds)
|
||||
service.calendarList().list(maxResults=1).execute()
|
||||
print("LIVE_CHECK_OK: Real API call succeeded.")
|
||||
return True
|
||||
except Exception as e:
|
||||
err_str = str(e).lower()
|
||||
if "disabled_client" in err_str or "invalid_client" in err_str:
|
||||
print(f"LIVE_CHECK_FAILED: OAuth client or account disabled: {e}")
|
||||
print(" 1. Check Google Cloud Console for disabled OAuth client")
|
||||
print(" 2. Check myaccount.google.com for account status")
|
||||
print(" 3. Do NOT retry with a disabled account")
|
||||
else:
|
||||
print(f"LIVE_CHECK_FAILED: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def check_auth(quiet: bool = False):
|
||||
"""Check if stored credentials are valid. Prints status, exits 0 or 1."""
|
||||
if not TOKEN_PATH.exists():
|
||||
print(f"NOT_AUTHENTICATED: No token at {TOKEN_PATH}")
|
||||
return False
|
||||
|
||||
_ensure_deps()
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
try:
|
||||
# Don't pass scopes — user may have authorized only a subset.
|
||||
# Passing scopes forces google-auth to validate them on refresh,
|
||||
# which fails with invalid_scope if the token has fewer scopes
|
||||
# than requested.
|
||||
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH))
|
||||
except Exception as e:
|
||||
print(f"TOKEN_CORRUPT: {e}")
|
||||
return False
|
||||
|
||||
payload = _load_token_payload(TOKEN_PATH)
|
||||
if creds.valid:
|
||||
missing_scopes = _missing_scopes_from_payload(payload)
|
||||
if missing_scopes:
|
||||
print(f"AUTHENTICATED (partial): Token valid but missing {len(missing_scopes)} scopes:")
|
||||
for s in missing_scopes:
|
||||
print(f" - {s}")
|
||||
if not quiet:
|
||||
print(f"AUTHENTICATED: Token valid at {TOKEN_PATH}")
|
||||
return True
|
||||
|
||||
if creds.expired and creds.refresh_token:
|
||||
try:
|
||||
creds.refresh(Request())
|
||||
TOKEN_PATH.write_text(
|
||||
json.dumps(
|
||||
_normalize_authorized_user_payload(json.loads(creds.to_json())),
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
missing_scopes = _missing_scopes_from_payload(_load_token_payload(TOKEN_PATH))
|
||||
if missing_scopes:
|
||||
print(f"AUTHENTICATED (partial): Token refreshed but missing {len(missing_scopes)} scopes:")
|
||||
for s in missing_scopes:
|
||||
print(f" - {s}")
|
||||
if not quiet:
|
||||
print(f"AUTHENTICATED: Token refreshed at {TOKEN_PATH}")
|
||||
return True
|
||||
except Exception as e:
|
||||
err_str = str(e).lower()
|
||||
if "disabled_client" in err_str or "invalid_client" in err_str:
|
||||
print(f"OAUTH_CLIENT_DISABLED: {e}")
|
||||
print(" The OAuth client or Google account has been disabled.")
|
||||
print(" Steps to resolve:")
|
||||
print(" 1. Check your Google Cloud Console — verify the OAuth client is not disabled")
|
||||
print(" 2. Check if your Google account itself has been disabled at myaccount.google.com")
|
||||
print(" 3. If the account is disabled, you can appeal at accounts.google.com/signin/recovery")
|
||||
print(" 4. Do NOT retry API calls with a disabled account — this may worsen the situation")
|
||||
print(" 5. If the OAuth client is disabled, create a new one in Google Cloud Console")
|
||||
elif "token_revoked" in err_str or "invalid_grant" in err_str:
|
||||
print(f"TOKEN_REVOKED: {e}")
|
||||
print(" Re-run setup to re-authenticate.")
|
||||
else:
|
||||
print(f"REFRESH_FAILED: {e}")
|
||||
return False
|
||||
|
||||
print("TOKEN_INVALID: Re-run setup.")
|
||||
return False
|
||||
|
||||
|
||||
def store_client_secret(path: str):
|
||||
"""Copy and validate client_secret.json to Hermes home."""
|
||||
src = Path(path).expanduser().resolve()
|
||||
if not src.exists():
|
||||
print(f"ERROR: File not found: {src}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
data = json.loads(src.read_text())
|
||||
except json.JSONDecodeError:
|
||||
print("ERROR: File is not valid JSON.")
|
||||
sys.exit(1)
|
||||
|
||||
if "installed" not in data and "web" not in data:
|
||||
print("ERROR: Not a Google OAuth client secret file (missing 'installed' key).")
|
||||
print("Download the correct file from: https://console.cloud.google.com/apis/credentials")
|
||||
sys.exit(1)
|
||||
|
||||
CLIENT_SECRET_PATH.write_text(json.dumps(data, indent=2))
|
||||
print(f"OK: Client secret saved to {CLIENT_SECRET_PATH}")
|
||||
|
||||
|
||||
def _save_pending_auth(*, state: str, code_verifier: str):
|
||||
"""Persist the OAuth session bits needed for a later token exchange."""
|
||||
PENDING_AUTH_PATH.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"state": state,
|
||||
"code_verifier": code_verifier,
|
||||
"redirect_uri": REDIRECT_URI,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _load_pending_auth() -> dict:
|
||||
"""Load the pending OAuth session created by get_auth_url()."""
|
||||
if not PENDING_AUTH_PATH.exists():
|
||||
print("ERROR: No pending OAuth session found. Run --auth-url first.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
data = json.loads(PENDING_AUTH_PATH.read_text())
|
||||
except Exception as e:
|
||||
print(f"ERROR: Could not read pending OAuth session: {e}")
|
||||
print("Run --auth-url again to start a fresh OAuth session.")
|
||||
sys.exit(1)
|
||||
|
||||
if not data.get("state") or not data.get("code_verifier"):
|
||||
print("ERROR: Pending OAuth session is missing PKCE data.")
|
||||
print("Run --auth-url again to start a fresh OAuth session.")
|
||||
sys.exit(1)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _extract_code_and_state(code_or_url: str) -> tuple[str, str | None]:
|
||||
"""Accept either a raw auth code or the full redirect URL pasted by the user."""
|
||||
if not code_or_url.startswith("http"):
|
||||
return code_or_url, None
|
||||
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
parsed = urlparse(code_or_url)
|
||||
params = parse_qs(parsed.query)
|
||||
if "code" not in params:
|
||||
print("ERROR: No 'code' parameter found in URL.")
|
||||
sys.exit(1)
|
||||
|
||||
state = params.get("state", [None])[0]
|
||||
return params["code"][0], state
|
||||
|
||||
|
||||
def get_auth_url():
|
||||
"""Print the OAuth authorization URL. User visits this in a browser."""
|
||||
if not CLIENT_SECRET_PATH.exists():
|
||||
print("ERROR: No client secret stored. Run --client-secret first.")
|
||||
sys.exit(1)
|
||||
|
||||
_ensure_deps()
|
||||
from google_auth_oauthlib.flow import Flow
|
||||
|
||||
flow = Flow.from_client_secrets_file(
|
||||
str(CLIENT_SECRET_PATH),
|
||||
scopes=SCOPES,
|
||||
redirect_uri=REDIRECT_URI,
|
||||
autogenerate_code_verifier=True,
|
||||
)
|
||||
auth_url, state = flow.authorization_url(
|
||||
access_type="offline",
|
||||
prompt="consent",
|
||||
)
|
||||
_save_pending_auth(state=state, code_verifier=flow.code_verifier)
|
||||
# Print just the URL so the agent can extract it cleanly
|
||||
print(auth_url)
|
||||
|
||||
|
||||
def exchange_auth_code(code: str):
|
||||
"""Exchange the authorization code for a token and save it."""
|
||||
if not CLIENT_SECRET_PATH.exists():
|
||||
print("ERROR: No client secret stored. Run --client-secret first.")
|
||||
sys.exit(1)
|
||||
|
||||
pending_auth = _load_pending_auth()
|
||||
raw_callback = code
|
||||
code, returned_state = _extract_code_and_state(code)
|
||||
if returned_state and returned_state != pending_auth["state"]:
|
||||
print("ERROR: OAuth state mismatch. Run --auth-url again to start a fresh session.")
|
||||
sys.exit(1)
|
||||
|
||||
_ensure_deps()
|
||||
from google_auth_oauthlib.flow import Flow
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
# Extract granted scopes from the callback URL if the user pasted the full redirect URL.
|
||||
granted_scopes = list(SCOPES)
|
||||
if isinstance(raw_callback, str) and raw_callback.startswith("http"):
|
||||
params = parse_qs(urlparse(raw_callback).query)
|
||||
scope_val = (params.get("scope") or [""])[0].strip()
|
||||
if scope_val:
|
||||
granted_scopes = scope_val.split()
|
||||
|
||||
flow = Flow.from_client_secrets_file(
|
||||
str(CLIENT_SECRET_PATH),
|
||||
scopes=granted_scopes,
|
||||
redirect_uri=pending_auth.get("redirect_uri", REDIRECT_URI),
|
||||
state=pending_auth["state"],
|
||||
code_verifier=pending_auth["code_verifier"],
|
||||
)
|
||||
|
||||
try:
|
||||
# Accept partial scopes — user may deselect some permissions in the consent screen
|
||||
os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1"
|
||||
flow.fetch_token(code=code)
|
||||
except Exception as e:
|
||||
print(f"ERROR: Token exchange failed: {e}")
|
||||
print("The code may have expired. Run --auth-url to get a fresh URL.")
|
||||
sys.exit(1)
|
||||
|
||||
creds = flow.credentials
|
||||
token_payload = _normalize_authorized_user_payload(json.loads(creds.to_json()))
|
||||
|
||||
# Store only the scopes actually granted by the user, not what was requested.
|
||||
# creds.to_json() writes the requested scopes, which causes refresh to fail
|
||||
# with invalid_scope if the user only authorized a subset.
|
||||
actually_granted = list(creds.granted_scopes or []) if hasattr(creds, "granted_scopes") and creds.granted_scopes else []
|
||||
if actually_granted:
|
||||
token_payload["scopes"] = actually_granted
|
||||
elif granted_scopes != SCOPES:
|
||||
# granted_scopes was extracted from the callback URL
|
||||
token_payload["scopes"] = granted_scopes
|
||||
|
||||
missing_scopes = _missing_scopes_from_payload(token_payload)
|
||||
if missing_scopes:
|
||||
print(f"WARNING: Token missing some Google Workspace scopes: {', '.join(missing_scopes)}")
|
||||
print("Some services may not be available.")
|
||||
|
||||
TOKEN_PATH.write_text(json.dumps(token_payload, indent=2))
|
||||
PENDING_AUTH_PATH.unlink(missing_ok=True)
|
||||
print(f"OK: Authenticated. Token saved to {TOKEN_PATH}")
|
||||
print(f"Profile-scoped token location: {display_hermes_home()}/google_token.json")
|
||||
|
||||
|
||||
def revoke():
|
||||
"""Revoke stored token and delete it."""
|
||||
if not TOKEN_PATH.exists():
|
||||
print("No token to revoke.")
|
||||
return
|
||||
|
||||
_ensure_deps()
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
try:
|
||||
creds = Credentials.from_authorized_user_file(str(TOKEN_PATH), SCOPES)
|
||||
if creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
|
||||
import urllib.request
|
||||
urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
f"https://oauth2.googleapis.com/revoke?token={creds.token}",
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
),
|
||||
timeout=15,
|
||||
)
|
||||
print("Token revoked with Google.")
|
||||
except Exception as e:
|
||||
print(f"Remote revocation failed (token may already be invalid): {e}")
|
||||
|
||||
TOKEN_PATH.unlink(missing_ok=True)
|
||||
PENDING_AUTH_PATH.unlink(missing_ok=True)
|
||||
print(f"Deleted {TOKEN_PATH}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Google Workspace OAuth setup for Hermes")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--check", action="store_true", help="Check if auth is valid (exit 0=yes, 1=no)")
|
||||
group.add_argument("--check-live", action="store_true", help="Check auth with a real API call (detects disabled_client)")
|
||||
group.add_argument("--client-secret", metavar="PATH", help="Store OAuth client_secret.json")
|
||||
group.add_argument("--auth-url", action="store_true", help="Print OAuth URL for user to visit")
|
||||
group.add_argument("--auth-code", metavar="CODE", help="Exchange auth code for token")
|
||||
group.add_argument("--revoke", action="store_true", help="Revoke and delete stored token")
|
||||
group.add_argument("--install-deps", action="store_true", help="Install Python dependencies")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.check:
|
||||
sys.exit(0 if check_auth() else 1)
|
||||
if getattr(args, "check_live", False):
|
||||
sys.exit(0 if check_auth_live() else 1)
|
||||
elif args.client_secret:
|
||||
store_client_secret(args.client_secret)
|
||||
elif args.auth_url:
|
||||
get_auth_url()
|
||||
elif args.auth_code:
|
||||
exchange_auth_code(args.auth_code)
|
||||
elif args.revoke:
|
||||
revoke()
|
||||
elif args.install_deps:
|
||||
sys.exit(0 if install_deps() else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Linear GraphQL API CLI — zero dependencies, stdlib only.
|
||||
|
||||
Usage:
|
||||
linear_api.py <command> [args...]
|
||||
|
||||
Commands:
|
||||
whoami Show authenticated user
|
||||
list-teams List all teams
|
||||
list-projects [--team KEY] List projects (optionally filter by team)
|
||||
list-states [--team KEY] List workflow states
|
||||
list-issues [filters] List issues
|
||||
--team KEY Filter by team key (e.g. ENG)
|
||||
--status NAME Filter by workflow state name
|
||||
--assignee NAME Filter by assignee name (exact)
|
||||
--label NAME Filter by label name
|
||||
--limit N Max results (default: 25)
|
||||
get-issue <IDENTIFIER> Full issue details (e.g. ENG-42)
|
||||
search-issues <query> Full-text search across issues
|
||||
create-issue [options] Create a new issue
|
||||
--title TITLE Required
|
||||
--team KEY Required
|
||||
--description DESC
|
||||
--priority 0-4 0=none, 1=urgent, 4=low
|
||||
--label NAME
|
||||
--assignee NAME
|
||||
--parent IDENTIFIER Parent issue ID for sub-issues
|
||||
update-issue <IDENTIFIER> [options] Update existing issue (same options as create)
|
||||
update-status <IDENTIFIER> <STATE> Move issue to workflow state (by state name)
|
||||
add-comment <IDENTIFIER> <body> Add comment to issue
|
||||
|
||||
list-documents [--limit N] List documents (docs, not issues)
|
||||
get-document <SLUG_OR_ID> Fetch a document by slugId (from URL) or UUID
|
||||
search-documents <query> Search documents by title
|
||||
|
||||
raw <graphql_query> [variables_json] Run an arbitrary GraphQL query
|
||||
Use --vars '{"key":"value"}' for variables
|
||||
|
||||
Auth:
|
||||
Set LINEAR_API_KEY environment variable (from Linear Settings -> API).
|
||||
Uses the personal API key header format: `Authorization: <KEY>` (no Bearer prefix).
|
||||
|
||||
Output:
|
||||
JSON to stdout. Errors to stderr with non-zero exit code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
API_URL = "https://api.linear.app/graphql"
|
||||
|
||||
|
||||
def _get_key() -> str:
|
||||
key = os.environ.get("LINEAR_API_KEY", "").strip()
|
||||
if not key:
|
||||
sys.stderr.write(
|
||||
"ERROR: LINEAR_API_KEY not set.\n"
|
||||
"Create one at https://linear.app/settings/api and export it,\n"
|
||||
"or add `LINEAR_API_KEY=lin_api_...` to ~/.hermes/.env\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
return key
|
||||
|
||||
|
||||
def gql(query: str, variables: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Execute a GraphQL query against Linear. Raises on HTTP error or GraphQL errors."""
|
||||
key = _get_key()
|
||||
payload = {"query": query}
|
||||
if variables:
|
||||
payload["variables"] = variables
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
API_URL,
|
||||
data=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": key, # Personal API key — NO `Bearer` prefix
|
||||
"User-Agent": "hermes-agent-linear-skill/1.0",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
body = resp.read().decode("utf-8")
|
||||
except urllib.error.HTTPError as e:
|
||||
sys.stderr.write(f"HTTP {e.code}: {e.read().decode('utf-8', 'replace')}\n")
|
||||
sys.exit(1)
|
||||
except urllib.error.URLError as e:
|
||||
sys.stderr.write(f"Network error: {e}\n")
|
||||
sys.exit(1)
|
||||
|
||||
result = json.loads(body)
|
||||
if "errors" in result and result["errors"]:
|
||||
sys.stderr.write(f"GraphQL errors: {json.dumps(result['errors'], indent=2)}\n")
|
||||
# Still return data if partial success; let caller decide
|
||||
if not result.get("data"):
|
||||
sys.exit(1)
|
||||
return result.get("data", {}) or {}
|
||||
|
||||
|
||||
def emit(obj: Any) -> None:
|
||||
print(json.dumps(obj, indent=2, default=str))
|
||||
|
||||
|
||||
# ---------- Commands ----------
|
||||
|
||||
def cmd_whoami(_args: argparse.Namespace) -> None:
|
||||
q = "query { viewer { id name email displayName } }"
|
||||
emit(gql(q).get("viewer"))
|
||||
|
||||
|
||||
def cmd_list_teams(_args: argparse.Namespace) -> None:
|
||||
q = "query { teams(first: 100) { nodes { id key name description } } }"
|
||||
emit(gql(q).get("teams", {}).get("nodes", []))
|
||||
|
||||
|
||||
def _resolve_team_id(key_or_name: str) -> str | None:
|
||||
"""Map a team key (ENG) or name to UUID."""
|
||||
q = "query { teams(first: 100) { nodes { id key name } } }"
|
||||
teams = gql(q).get("teams", {}).get("nodes", [])
|
||||
kl = key_or_name.lower()
|
||||
for t in teams:
|
||||
if t["key"].lower() == kl or t["name"].lower() == kl:
|
||||
return t["id"]
|
||||
return None
|
||||
|
||||
|
||||
def cmd_list_projects(args: argparse.Namespace) -> None:
|
||||
if args.team:
|
||||
tid = _resolve_team_id(args.team)
|
||||
if not tid:
|
||||
sys.stderr.write(f"Team not found: {args.team}\n")
|
||||
sys.exit(1)
|
||||
q = """query($id: String!) {
|
||||
team(id: $id) { projects(first: 100) { nodes { id name description state } } }
|
||||
}"""
|
||||
data = gql(q, {"id": tid})
|
||||
emit(data.get("team", {}).get("projects", {}).get("nodes", []))
|
||||
else:
|
||||
q = "query { projects(first: 100) { nodes { id name description state } } }"
|
||||
emit(gql(q).get("projects", {}).get("nodes", []))
|
||||
|
||||
|
||||
def cmd_list_states(args: argparse.Namespace) -> None:
|
||||
if args.team:
|
||||
tid = _resolve_team_id(args.team)
|
||||
if not tid:
|
||||
sys.stderr.write(f"Team not found: {args.team}\n")
|
||||
sys.exit(1)
|
||||
q = """query($id: String!) {
|
||||
team(id: $id) { states(first: 100) { nodes { id name type color } } }
|
||||
}"""
|
||||
emit(gql(q, {"id": tid}).get("team", {}).get("states", {}).get("nodes", []))
|
||||
else:
|
||||
q = "query { workflowStates(first: 200) { nodes { id name type team { key } } } }"
|
||||
emit(gql(q).get("workflowStates", {}).get("nodes", []))
|
||||
|
||||
|
||||
def cmd_list_issues(args: argparse.Namespace) -> None:
|
||||
filt: dict[str, Any] = {}
|
||||
if args.team:
|
||||
filt["team"] = {"key": {"eq": args.team}}
|
||||
if args.status:
|
||||
filt["state"] = {"name": {"eq": args.status}}
|
||||
if args.assignee:
|
||||
filt["assignee"] = {"name": {"eq": args.assignee}}
|
||||
if args.label:
|
||||
filt["labels"] = {"name": {"eq": args.label}}
|
||||
|
||||
q = """query($filter: IssueFilter, $first: Int!) {
|
||||
issues(filter: $filter, first: $first, orderBy: updatedAt) {
|
||||
nodes {
|
||||
id identifier title
|
||||
state { name } priority
|
||||
assignee { name }
|
||||
team { key }
|
||||
updatedAt url
|
||||
}
|
||||
}
|
||||
}"""
|
||||
data = gql(q, {"filter": filt or None, "first": args.limit})
|
||||
emit(data.get("issues", {}).get("nodes", []))
|
||||
|
||||
|
||||
def cmd_get_issue(args: argparse.Namespace) -> None:
|
||||
q = """query($id: String!) {
|
||||
issue(id: $id) {
|
||||
id identifier title description
|
||||
state { name type }
|
||||
priority priorityLabel
|
||||
assignee { name email }
|
||||
creator { name }
|
||||
team { key name }
|
||||
project { name }
|
||||
labels { nodes { name } }
|
||||
parent { identifier title }
|
||||
children { nodes { identifier title state { name } } }
|
||||
comments { nodes { user { name } body createdAt } }
|
||||
createdAt updatedAt url
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"id": args.identifier}).get("issue"))
|
||||
|
||||
|
||||
def cmd_search_issues(args: argparse.Namespace) -> None:
|
||||
q = """query($term: String!, $first: Int!) {
|
||||
searchIssues(term: $term, first: $first) {
|
||||
nodes { id identifier title state { name } url }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"term": args.query, "first": args.limit}).get("searchIssues", {}).get("nodes", []))
|
||||
|
||||
|
||||
def cmd_create_issue(args: argparse.Namespace) -> None:
|
||||
tid = _resolve_team_id(args.team)
|
||||
if not tid:
|
||||
sys.stderr.write(f"Team not found: {args.team}\n")
|
||||
sys.exit(1)
|
||||
inp: dict[str, Any] = {"title": args.title, "teamId": tid}
|
||||
if args.description:
|
||||
inp["description"] = args.description
|
||||
if args.priority is not None:
|
||||
inp["priority"] = args.priority
|
||||
if args.parent:
|
||||
inp["parentId"] = args.parent
|
||||
# TODO: label + assignee name->id lookup (omitted for v1 brevity)
|
||||
|
||||
q = """mutation($input: IssueCreateInput!) {
|
||||
issueCreate(input: $input) {
|
||||
success issue { id identifier title url }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"input": inp}).get("issueCreate"))
|
||||
|
||||
|
||||
def cmd_update_issue(args: argparse.Namespace) -> None:
|
||||
inp: dict[str, Any] = {}
|
||||
if args.title:
|
||||
inp["title"] = args.title
|
||||
if args.description:
|
||||
inp["description"] = args.description
|
||||
if args.priority is not None:
|
||||
inp["priority"] = args.priority
|
||||
if not inp:
|
||||
sys.stderr.write("No update fields provided.\n")
|
||||
sys.exit(1)
|
||||
q = """mutation($id: String!, $input: IssueUpdateInput!) {
|
||||
issueUpdate(id: $id, input: $input) {
|
||||
success issue { identifier title url }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"id": args.identifier, "input": inp}).get("issueUpdate"))
|
||||
|
||||
|
||||
def cmd_update_status(args: argparse.Namespace) -> None:
|
||||
# Resolve state name -> id within the issue's team
|
||||
get_q = """query($id: String!) {
|
||||
issue(id: $id) { team { id states(first: 100) { nodes { id name } } } }
|
||||
}"""
|
||||
issue = gql(get_q, {"id": args.identifier}).get("issue")
|
||||
if not issue:
|
||||
sys.stderr.write(f"Issue not found: {args.identifier}\n")
|
||||
sys.exit(1)
|
||||
sl = args.state.lower()
|
||||
match = next((s for s in issue["team"]["states"]["nodes"] if s["name"].lower() == sl), None)
|
||||
if not match:
|
||||
sys.stderr.write(
|
||||
f"State '{args.state}' not found. Available: "
|
||||
f"{[s['name'] for s in issue['team']['states']['nodes']]}\n"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
q = """mutation($id: String!, $stateId: String!) {
|
||||
issueUpdate(id: $id, input: { stateId: $stateId }) {
|
||||
success issue { identifier state { name } url }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"id": args.identifier, "stateId": match["id"]}).get("issueUpdate"))
|
||||
|
||||
|
||||
def cmd_add_comment(args: argparse.Namespace) -> None:
|
||||
q = """mutation($input: CommentCreateInput!) {
|
||||
commentCreate(input: $input) {
|
||||
success comment { id body createdAt }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"input": {"issueId": args.identifier, "body": args.body}}).get("commentCreate"))
|
||||
|
||||
|
||||
# ---- Documents ----
|
||||
|
||||
def cmd_list_documents(args: argparse.Namespace) -> None:
|
||||
q = """query($first: Int!) {
|
||||
documents(first: $first, orderBy: updatedAt) {
|
||||
nodes { id title slugId updatedAt url project { name } creator { name } }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"first": args.limit}).get("documents", {}).get("nodes", []))
|
||||
|
||||
|
||||
def cmd_get_document(args: argparse.Namespace) -> None:
|
||||
"""Fetch a document by slugId (from URL) OR full UUID.
|
||||
|
||||
Linear document URLs look like:
|
||||
https://linear.app/<workspace>/document/<slug>-<shortid>
|
||||
The part we want is the final hex segment (the slugId).
|
||||
"""
|
||||
ref = args.ref
|
||||
# If it looks like a UUID, query by id. Otherwise, assume slugId.
|
||||
is_uuid = len(ref) == 36 and ref.count("-") == 4
|
||||
if is_uuid:
|
||||
q = """query($id: String!) {
|
||||
document(id: $id) {
|
||||
id title content contentState slugId
|
||||
createdAt updatedAt url
|
||||
creator { name } project { name }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"id": ref}).get("document"))
|
||||
else:
|
||||
# Query the collection and filter by slugId — the doc() query only accepts UUIDs.
|
||||
q = """query($slug: String!) {
|
||||
documents(filter: { slugId: { eq: $slug } }, first: 1) {
|
||||
nodes {
|
||||
id title content contentState slugId
|
||||
createdAt updatedAt url
|
||||
creator { name } project { name }
|
||||
}
|
||||
}
|
||||
}"""
|
||||
nodes = gql(q, {"slug": ref}).get("documents", {}).get("nodes", [])
|
||||
emit(nodes[0] if nodes else None)
|
||||
|
||||
|
||||
def cmd_search_documents(args: argparse.Namespace) -> None:
|
||||
# Linear doesn't have a first-class searchDocuments — use title filter as a fallback.
|
||||
q = """query($term: String!, $first: Int!) {
|
||||
documents(filter: { title: { containsIgnoreCase: $term } }, first: $first) {
|
||||
nodes { id title slugId url updatedAt }
|
||||
}
|
||||
}"""
|
||||
emit(gql(q, {"term": args.query, "first": args.limit}).get("documents", {}).get("nodes", []))
|
||||
|
||||
|
||||
def cmd_raw(args: argparse.Namespace) -> None:
|
||||
variables = json.loads(args.vars) if args.vars else None
|
||||
emit(gql(args.query, variables))
|
||||
|
||||
|
||||
# ---------- Arg parsing ----------
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="linear_api.py", description="Linear GraphQL CLI")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("whoami").set_defaults(func=cmd_whoami)
|
||||
sub.add_parser("list-teams").set_defaults(func=cmd_list_teams)
|
||||
|
||||
lp = sub.add_parser("list-projects")
|
||||
lp.add_argument("--team")
|
||||
lp.set_defaults(func=cmd_list_projects)
|
||||
|
||||
ls = sub.add_parser("list-states")
|
||||
ls.add_argument("--team")
|
||||
ls.set_defaults(func=cmd_list_states)
|
||||
|
||||
li = sub.add_parser("list-issues")
|
||||
li.add_argument("--team")
|
||||
li.add_argument("--status")
|
||||
li.add_argument("--assignee")
|
||||
li.add_argument("--label")
|
||||
li.add_argument("--limit", type=int, default=25)
|
||||
li.set_defaults(func=cmd_list_issues)
|
||||
|
||||
gi = sub.add_parser("get-issue")
|
||||
gi.add_argument("identifier")
|
||||
gi.set_defaults(func=cmd_get_issue)
|
||||
|
||||
si = sub.add_parser("search-issues")
|
||||
si.add_argument("query")
|
||||
si.add_argument("--limit", type=int, default=25)
|
||||
si.set_defaults(func=cmd_search_issues)
|
||||
|
||||
ci = sub.add_parser("create-issue")
|
||||
ci.add_argument("--title", required=True)
|
||||
ci.add_argument("--team", required=True)
|
||||
ci.add_argument("--description")
|
||||
ci.add_argument("--priority", type=int, choices=[0, 1, 2, 3, 4])
|
||||
ci.add_argument("--label")
|
||||
ci.add_argument("--assignee")
|
||||
ci.add_argument("--parent")
|
||||
ci.set_defaults(func=cmd_create_issue)
|
||||
|
||||
ui = sub.add_parser("update-issue")
|
||||
ui.add_argument("identifier")
|
||||
ui.add_argument("--title")
|
||||
ui.add_argument("--description")
|
||||
ui.add_argument("--priority", type=int, choices=[0, 1, 2, 3, 4])
|
||||
ui.set_defaults(func=cmd_update_issue)
|
||||
|
||||
us = sub.add_parser("update-status")
|
||||
us.add_argument("identifier")
|
||||
us.add_argument("state")
|
||||
us.set_defaults(func=cmd_update_status)
|
||||
|
||||
ac = sub.add_parser("add-comment")
|
||||
ac.add_argument("identifier")
|
||||
ac.add_argument("body")
|
||||
ac.set_defaults(func=cmd_add_comment)
|
||||
|
||||
ld = sub.add_parser("list-documents")
|
||||
ld.add_argument("--limit", type=int, default=50)
|
||||
ld.set_defaults(func=cmd_list_documents)
|
||||
|
||||
gd = sub.add_parser("get-document")
|
||||
gd.add_argument("ref", help="slugId (hex suffix from URL) or full UUID")
|
||||
gd.set_defaults(func=cmd_get_document)
|
||||
|
||||
sd = sub.add_parser("search-documents")
|
||||
sd.add_argument("query")
|
||||
sd.add_argument("--limit", type=int, default=25)
|
||||
sd.set_defaults(func=cmd_search_documents)
|
||||
|
||||
r = sub.add_parser("raw")
|
||||
r.add_argument("query")
|
||||
r.add_argument("--vars", help="JSON string of variables")
|
||||
r.set_defaults(func=cmd_raw)
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
# minimalist
|
||||
|
||||
极简画风 - Clean black line art, limited spot color, simplified stick-figure characters
|
||||
|
||||
## Overview
|
||||
|
||||
Minimalist cartoon illustration characterized by clean black line art on white background with very limited spot color for emphasis. Characters are simplified to near-stick-figure abstraction, focusing on gesture and concept rather than anatomical detail. Designed for business allegory, quick-read educational content, and concept illustration.
|
||||
|
||||
## Line Work
|
||||
|
||||
- Clean, uniform black lines (1.5-2px)
|
||||
- No hatching, cross-hatching, or shading techniques
|
||||
- Minimal detail — every line serves a purpose
|
||||
- Bold outlines for characters, thinner lines for props/labels
|
||||
- No decorative flourishes or ornamental lines
|
||||
|
||||
## Character Design
|
||||
|
||||
- Highly simplified, stick-figure-like business characters
|
||||
- Circle or oval heads with minimal facial features (dot eyes, simple line mouth)
|
||||
- Body as simple geometric shapes or line constructions
|
||||
- Distinguishing features through props only (tie, hat, briefcase, glasses)
|
||||
- No anatomical detail — expressive through posture and gesture
|
||||
- 4-5 head height proportions (squat, iconic)
|
||||
|
||||
## Background Treatment
|
||||
|
||||
- Mostly blank/white — negative space is a design element
|
||||
- Minimal environmental cues (a line for ground, simple desk outline)
|
||||
- Concept labels and text annotations replace detailed environments
|
||||
- Icons and symbols over realistic rendering
|
||||
- No perspective or spatial depth
|
||||
|
||||
## Color Approach
|
||||
|
||||
- Primarily black and white (90%+ of the image)
|
||||
- 1-2 spot accent colors for emphasis on key concepts
|
||||
- Accent color used sparingly: highlighting key objects, text labels, concept indicators
|
||||
- No gradients, no shading, no color fills on backgrounds
|
||||
- Color draws the eye to the most important element in each panel
|
||||
|
||||
## Default Color Palette
|
||||
|
||||
| Role | Color | Hex |
|
||||
|------|-------|-----|
|
||||
| Primary | Black ink | `#1A1A1A` |
|
||||
| Background | Clean white | `#FFFFFF` |
|
||||
| Accent 1 | Spot orange | `#FF6B35` |
|
||||
| Accent 2 | Spot blue (optional) | `#3182CE` |
|
||||
| Text labels | Dark gray | `#4A4A4A` |
|
||||
| Panel border | Medium gray | `#666666` |
|
||||
|
||||
## Visual Elements
|
||||
|
||||
- Text labels with accent-color backgrounds or underlines for key terms
|
||||
- Simple icons: arrows, circles, checkmarks, crosses
|
||||
- Concept highlight boxes with spot color
|
||||
- Minimal speech bubbles (simple oval or rectangle, thin black outline)
|
||||
- No sound effects, no motion lines, no screen tones
|
||||
|
||||
## Quality Markers
|
||||
|
||||
- ✓ Clean, purposeful line work with no unnecessary detail
|
||||
- ✓ 90%+ black-and-white with strategic spot color
|
||||
- ✓ Simplified characters readable at small sizes
|
||||
- ✓ Text labels integrated naturally into panels
|
||||
- ✓ Strong negative space usage
|
||||
- ✓ Every element serves the narrative point
|
||||
|
||||
## Compatibility
|
||||
|
||||
| Tone | Fit | Notes |
|
||||
|------|-----|-------|
|
||||
| neutral | ✓✓ | Ideal for business/educational content |
|
||||
| warm | ✓ | Works for gentle stories, slight warmth in accent |
|
||||
| energetic | ✓ | Works for punchy, high-energy content |
|
||||
| dramatic | ✗ | Style too stripped down for dramatic intensity |
|
||||
| vintage | ✗ | Minimalist aesthetic conflicts with aged/textured look |
|
||||
| romantic | ✗ | No capacity for decorative/soft elements |
|
||||
| action | ✗ | No dynamic line capability for speed/impact |
|
||||
|
||||
## Best For
|
||||
|
||||
Business allegory, management fables, short concept illustration, four-panel comic strips, quick-insight education, social media content
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract text from documents using marker-pdf. High-quality OCR + layout analysis.
|
||||
|
||||
Requires ~3-5GB disk (PyTorch + models downloaded on first use).
|
||||
Supports: PDF, DOCX, PPTX, XLSX, HTML, EPUB, images.
|
||||
|
||||
Usage:
|
||||
python extract_marker.py document.pdf
|
||||
python extract_marker.py document.pdf --output_dir ./output
|
||||
python extract_marker.py presentation.pptx
|
||||
python extract_marker.py spreadsheet.xlsx
|
||||
python extract_marker.py scanned_doc.pdf # OCR works here
|
||||
python extract_marker.py document.pdf --json # Structured output
|
||||
python extract_marker.py document.pdf --use_llm # LLM-boosted accuracy
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
|
||||
def convert(path, output_dir=None, output_format="markdown", use_llm=False):
|
||||
from marker.converters.pdf import PdfConverter
|
||||
from marker.models import create_model_dict
|
||||
from marker.config.parser import ConfigParser
|
||||
|
||||
config_dict = {}
|
||||
if use_llm:
|
||||
config_dict["use_llm"] = True
|
||||
|
||||
config_parser = ConfigParser(config_dict)
|
||||
models = create_model_dict()
|
||||
converter = PdfConverter(config=config_parser.generate_config_dict(), artifact_dict=models)
|
||||
rendered = converter(path)
|
||||
|
||||
if output_format == "json":
|
||||
import json
|
||||
print(json.dumps({
|
||||
"markdown": rendered.markdown,
|
||||
"metadata": rendered.metadata if hasattr(rendered, "metadata") else {},
|
||||
}, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
print(rendered.markdown)
|
||||
|
||||
# Save images if output_dir specified
|
||||
if output_dir and hasattr(rendered, "images") and rendered.images:
|
||||
from pathlib import Path
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
for name, img_data in rendered.images.items():
|
||||
img_path = os.path.join(output_dir, name)
|
||||
with open(img_path, "wb") as f:
|
||||
f.write(img_data)
|
||||
print(f"\nSaved {len(rendered.images)} image(s) to {output_dir}/", file=sys.stderr)
|
||||
|
||||
|
||||
def check_requirements():
|
||||
"""Check disk space before installing."""
|
||||
import shutil
|
||||
free_gb = shutil.disk_usage("/").free / (1024**3)
|
||||
if free_gb < 5:
|
||||
print(f"⚠️ Only {free_gb:.1f}GB free. marker-pdf needs ~5GB for PyTorch + models.")
|
||||
print("Use pymupdf instead (scripts/extract_pymupdf.py) or free up disk space.")
|
||||
sys.exit(1)
|
||||
print(f"✓ {free_gb:.1f}GB free — sufficient for marker-pdf")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
if not args or args[0] in {"-h", "--help"}:
|
||||
print(__doc__)
|
||||
sys.exit(0)
|
||||
|
||||
if args[0] == "--check":
|
||||
check_requirements()
|
||||
sys.exit(0)
|
||||
|
||||
path = args[0]
|
||||
output_dir = None
|
||||
output_format = "markdown"
|
||||
use_llm = False
|
||||
|
||||
if "--output_dir" in args:
|
||||
idx = args.index("--output_dir")
|
||||
output_dir = args[idx + 1]
|
||||
if "--json" in args:
|
||||
output_format = "json"
|
||||
if "--use_llm" in args:
|
||||
use_llm = True
|
||||
|
||||
convert(path, output_dir=output_dir, output_format=output_format, use_llm=use_llm)
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract text from documents using pymupdf. Lightweight (~25MB), no models.
|
||||
|
||||
Usage:
|
||||
python extract_pymupdf.py document.pdf
|
||||
python extract_pymupdf.py document.pdf --markdown
|
||||
python extract_pymupdf.py document.pdf --pages 0-4
|
||||
python extract_pymupdf.py document.pdf --images output_dir/
|
||||
python extract_pymupdf.py document.pdf --tables
|
||||
python extract_pymupdf.py document.pdf --metadata
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
|
||||
def extract_text(path, pages=None):
|
||||
import pymupdf
|
||||
doc = pymupdf.open(path)
|
||||
page_range = range(len(doc)) if pages is None else pages
|
||||
for i in page_range:
|
||||
if i < len(doc):
|
||||
print(f"\n--- Page {i+1}/{len(doc)} ---\n")
|
||||
print(doc[i].get_text())
|
||||
|
||||
def extract_markdown(path, pages=None):
|
||||
import pymupdf4llm
|
||||
md = pymupdf4llm.to_markdown(path, pages=pages)
|
||||
print(md)
|
||||
|
||||
def extract_tables(path):
|
||||
import pymupdf
|
||||
doc = pymupdf.open(path)
|
||||
for i, page in enumerate(doc):
|
||||
tables = page.find_tables()
|
||||
for j, table in enumerate(tables.tables):
|
||||
print(f"\n--- Page {i+1}, Table {j+1} ---\n")
|
||||
df = table.to_pandas()
|
||||
print(df.to_markdown(index=False))
|
||||
|
||||
def extract_images(path, output_dir):
|
||||
import pymupdf
|
||||
from pathlib import Path
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
doc = pymupdf.open(path)
|
||||
count = 0
|
||||
for i, page in enumerate(doc):
|
||||
for img_idx, img in enumerate(page.get_images(full=True)):
|
||||
xref = img[0]
|
||||
pix = pymupdf.Pixmap(doc, xref)
|
||||
if pix.n >= 5:
|
||||
pix = pymupdf.Pixmap(pymupdf.csRGB, pix)
|
||||
out_path = f"{output_dir}/page{i+1}_img{img_idx+1}.png"
|
||||
pix.save(out_path)
|
||||
count += 1
|
||||
print(f"Extracted {count} images to {output_dir}/")
|
||||
|
||||
def show_metadata(path):
|
||||
import pymupdf
|
||||
doc = pymupdf.open(path)
|
||||
print(json.dumps({
|
||||
"pages": len(doc),
|
||||
"title": doc.metadata.get("title", ""),
|
||||
"author": doc.metadata.get("author", ""),
|
||||
"subject": doc.metadata.get("subject", ""),
|
||||
"creator": doc.metadata.get("creator", ""),
|
||||
"producer": doc.metadata.get("producer", ""),
|
||||
"format": doc.metadata.get("format", ""),
|
||||
}, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
if not args or args[0] in {"-h", "--help"}:
|
||||
print(__doc__)
|
||||
sys.exit(0)
|
||||
|
||||
path = args[0]
|
||||
pages = None
|
||||
|
||||
if "--pages" in args:
|
||||
idx = args.index("--pages")
|
||||
p = args[idx + 1]
|
||||
if "-" in p:
|
||||
start, end = p.split("-")
|
||||
pages = list(range(int(start), int(end) + 1))
|
||||
else:
|
||||
pages = [int(p)]
|
||||
|
||||
if "--metadata" in args:
|
||||
show_metadata(path)
|
||||
elif "--tables" in args:
|
||||
extract_tables(path)
|
||||
elif "--images" in args:
|
||||
idx = args.index("--images")
|
||||
output_dir = args[idx + 1] if idx + 1 < len(args) else "./images"
|
||||
extract_images(path, output_dir)
|
||||
elif "--markdown" in args:
|
||||
extract_markdown(path, pages=pages)
|
||||
else:
|
||||
extract_text(path, pages=pages)
|
||||
@@ -0,0 +1,65 @@
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
|
||||
|
||||
@dataclass
|
||||
class RectAndField:
|
||||
rect: list[float]
|
||||
rect_type: str
|
||||
field: dict
|
||||
|
||||
|
||||
def get_bounding_box_messages(fields_json_stream) -> list[str]:
|
||||
messages = []
|
||||
fields = json.load(fields_json_stream)
|
||||
messages.append(f"Read {len(fields['form_fields'])} fields")
|
||||
|
||||
def rects_intersect(r1, r2):
|
||||
disjoint_horizontal = r1[0] >= r2[2] or r1[2] <= r2[0]
|
||||
disjoint_vertical = r1[1] >= r2[3] or r1[3] <= r2[1]
|
||||
return not (disjoint_horizontal or disjoint_vertical)
|
||||
|
||||
rects_and_fields = []
|
||||
for f in fields["form_fields"]:
|
||||
rects_and_fields.append(RectAndField(f["label_bounding_box"], "label", f))
|
||||
rects_and_fields.append(RectAndField(f["entry_bounding_box"], "entry", f))
|
||||
|
||||
has_error = False
|
||||
for i, ri in enumerate(rects_and_fields):
|
||||
for j in range(i + 1, len(rects_and_fields)):
|
||||
rj = rects_and_fields[j]
|
||||
if ri.field["page_number"] == rj.field["page_number"] and rects_intersect(ri.rect, rj.rect):
|
||||
has_error = True
|
||||
if ri.field is rj.field:
|
||||
messages.append(f"FAILURE: intersection between label and entry bounding boxes for `{ri.field['description']}` ({ri.rect}, {rj.rect})")
|
||||
else:
|
||||
messages.append(f"FAILURE: intersection between {ri.rect_type} bounding box for `{ri.field['description']}` ({ri.rect}) and {rj.rect_type} bounding box for `{rj.field['description']}` ({rj.rect})")
|
||||
if len(messages) >= 20:
|
||||
messages.append("Aborting further checks; fix bounding boxes and try again")
|
||||
return messages
|
||||
if ri.rect_type == "entry":
|
||||
if "entry_text" in ri.field:
|
||||
font_size = ri.field["entry_text"].get("font_size", 14)
|
||||
entry_height = ri.rect[3] - ri.rect[1]
|
||||
if entry_height < font_size:
|
||||
has_error = True
|
||||
messages.append(f"FAILURE: entry bounding box height ({entry_height}) for `{ri.field['description']}` is too short for the text content (font size: {font_size}). Increase the box height or decrease the font size.")
|
||||
if len(messages) >= 20:
|
||||
messages.append("Aborting further checks; fix bounding boxes and try again")
|
||||
return messages
|
||||
|
||||
if not has_error:
|
||||
messages.append("SUCCESS: All bounding boxes are valid")
|
||||
return messages
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: check_bounding_boxes.py [fields.json]")
|
||||
sys.exit(1)
|
||||
with open(sys.argv[1]) as f:
|
||||
messages = get_bounding_box_messages(f)
|
||||
for msg in messages:
|
||||
print(msg)
|
||||
@@ -0,0 +1,11 @@
|
||||
import sys
|
||||
from pypdf import PdfReader
|
||||
|
||||
|
||||
|
||||
|
||||
reader = PdfReader(sys.argv[1])
|
||||
if (reader.get_fields()):
|
||||
print("This PDF has fillable form fields")
|
||||
else:
|
||||
print("This PDF does not have fillable form fields; you will need to visually determine where to enter data")
|
||||
@@ -0,0 +1,33 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pdf2image import convert_from_path
|
||||
|
||||
|
||||
|
||||
|
||||
def convert(pdf_path, output_dir, max_dim=1000):
|
||||
images = convert_from_path(pdf_path, dpi=200)
|
||||
|
||||
for i, image in enumerate(images):
|
||||
width, height = image.size
|
||||
if width > max_dim or height > max_dim:
|
||||
scale_factor = min(max_dim / width, max_dim / height)
|
||||
new_width = int(width * scale_factor)
|
||||
new_height = int(height * scale_factor)
|
||||
image = image.resize((new_width, new_height))
|
||||
|
||||
image_path = os.path.join(output_dir, f"page_{i+1}.png")
|
||||
image.save(image_path)
|
||||
print(f"Saved page {i+1} as {image_path} (size: {image.size})")
|
||||
|
||||
print(f"Converted {len(images)} pages to PNG images")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: convert_pdf_to_images.py [input pdf] [output directory]")
|
||||
sys.exit(1)
|
||||
pdf_path = sys.argv[1]
|
||||
output_directory = sys.argv[2]
|
||||
convert(pdf_path, output_directory)
|
||||
@@ -0,0 +1,37 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
|
||||
|
||||
def create_validation_image(page_number, fields_json_path, input_path, output_path):
|
||||
with open(fields_json_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
img = Image.open(input_path)
|
||||
draw = ImageDraw.Draw(img)
|
||||
num_boxes = 0
|
||||
|
||||
for field in data["form_fields"]:
|
||||
if field["page_number"] == page_number:
|
||||
entry_box = field['entry_bounding_box']
|
||||
label_box = field['label_bounding_box']
|
||||
draw.rectangle(entry_box, outline='red', width=2)
|
||||
draw.rectangle(label_box, outline='blue', width=2)
|
||||
num_boxes += 2
|
||||
|
||||
img.save(output_path)
|
||||
print(f"Created validation image at {output_path} with {num_boxes} bounding boxes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 5:
|
||||
print("Usage: create_validation_image.py [page number] [fields.json file] [input image path] [output image path]")
|
||||
sys.exit(1)
|
||||
page_number = int(sys.argv[1])
|
||||
fields_json_path = sys.argv[2]
|
||||
input_image_path = sys.argv[3]
|
||||
output_image_path = sys.argv[4]
|
||||
create_validation_image(page_number, fields_json_path, input_image_path, output_image_path)
|
||||
@@ -0,0 +1,122 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from pypdf import PdfReader
|
||||
|
||||
|
||||
|
||||
|
||||
def get_full_annotation_field_id(annotation):
|
||||
components = []
|
||||
while annotation:
|
||||
field_name = annotation.get('/T')
|
||||
if field_name:
|
||||
components.append(field_name)
|
||||
annotation = annotation.get('/Parent')
|
||||
return ".".join(reversed(components)) if components else None
|
||||
|
||||
|
||||
def make_field_dict(field, field_id):
|
||||
field_dict = {"field_id": field_id}
|
||||
ft = field.get('/FT')
|
||||
if ft == "/Tx":
|
||||
field_dict["type"] = "text"
|
||||
elif ft == "/Btn":
|
||||
field_dict["type"] = "checkbox"
|
||||
states = field.get("/_States_", [])
|
||||
if len(states) == 2:
|
||||
if "/Off" in states:
|
||||
field_dict["checked_value"] = states[0] if states[0] != "/Off" else states[1]
|
||||
field_dict["unchecked_value"] = "/Off"
|
||||
else:
|
||||
print(f"Unexpected state values for checkbox `${field_id}`. Its checked and unchecked values may not be correct; if you're trying to check it, visually verify the results.")
|
||||
field_dict["checked_value"] = states[0]
|
||||
field_dict["unchecked_value"] = states[1]
|
||||
elif ft == "/Ch":
|
||||
field_dict["type"] = "choice"
|
||||
states = field.get("/_States_", [])
|
||||
field_dict["choice_options"] = [{
|
||||
"value": state[0],
|
||||
"text": state[1],
|
||||
} for state in states]
|
||||
else:
|
||||
field_dict["type"] = f"unknown ({ft})"
|
||||
return field_dict
|
||||
|
||||
|
||||
def get_field_info(reader: PdfReader):
|
||||
fields = reader.get_fields()
|
||||
|
||||
field_info_by_id = {}
|
||||
possible_radio_names = set()
|
||||
|
||||
for field_id, field in fields.items():
|
||||
if field.get("/Kids"):
|
||||
if field.get("/FT") == "/Btn":
|
||||
possible_radio_names.add(field_id)
|
||||
continue
|
||||
field_info_by_id[field_id] = make_field_dict(field, field_id)
|
||||
|
||||
|
||||
radio_fields_by_id = {}
|
||||
|
||||
for page_index, page in enumerate(reader.pages):
|
||||
annotations = page.get('/Annots', [])
|
||||
for ann in annotations:
|
||||
field_id = get_full_annotation_field_id(ann)
|
||||
if field_id in field_info_by_id:
|
||||
field_info_by_id[field_id]["page"] = page_index + 1
|
||||
field_info_by_id[field_id]["rect"] = ann.get('/Rect')
|
||||
elif field_id in possible_radio_names:
|
||||
try:
|
||||
on_values = [v for v in ann["/AP"]["/N"] if v != "/Off"]
|
||||
except KeyError:
|
||||
continue
|
||||
if len(on_values) == 1:
|
||||
rect = ann.get("/Rect")
|
||||
if field_id not in radio_fields_by_id:
|
||||
radio_fields_by_id[field_id] = {
|
||||
"field_id": field_id,
|
||||
"type": "radio_group",
|
||||
"page": page_index + 1,
|
||||
"radio_options": [],
|
||||
}
|
||||
radio_fields_by_id[field_id]["radio_options"].append({
|
||||
"value": on_values[0],
|
||||
"rect": rect,
|
||||
})
|
||||
|
||||
fields_with_location = []
|
||||
for field_info in field_info_by_id.values():
|
||||
if "page" in field_info:
|
||||
fields_with_location.append(field_info)
|
||||
else:
|
||||
print(f"Unable to determine location for field id: {field_info.get('field_id')}, ignoring")
|
||||
|
||||
def sort_key(f):
|
||||
if "radio_options" in f:
|
||||
rect = f["radio_options"][0]["rect"] or [0, 0, 0, 0]
|
||||
else:
|
||||
rect = f.get("rect") or [0, 0, 0, 0]
|
||||
adjusted_position = [-rect[1], rect[0]]
|
||||
return [f.get("page"), adjusted_position]
|
||||
|
||||
sorted_fields = fields_with_location + list(radio_fields_by_id.values())
|
||||
sorted_fields.sort(key=sort_key)
|
||||
|
||||
return sorted_fields
|
||||
|
||||
|
||||
def write_field_info(pdf_path: str, json_output_path: str):
|
||||
reader = PdfReader(pdf_path)
|
||||
field_info = get_field_info(reader)
|
||||
with open(json_output_path, "w") as f:
|
||||
json.dump(field_info, f, indent=2)
|
||||
print(f"Wrote {len(field_info)} fields to {json_output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: extract_form_field_info.py [input pdf] [output json]")
|
||||
sys.exit(1)
|
||||
write_field_info(sys.argv[1], sys.argv[2])
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
Extract form structure from a non-fillable PDF.
|
||||
|
||||
This script analyzes the PDF to find:
|
||||
- Text labels with their exact coordinates
|
||||
- Horizontal lines (row boundaries)
|
||||
- Checkboxes (small rectangles)
|
||||
|
||||
Output: A JSON file with the form structure that can be used to generate
|
||||
accurate field coordinates for filling.
|
||||
|
||||
Usage: python extract_form_structure.py <input.pdf> <output.json>
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import pdfplumber
|
||||
|
||||
|
||||
def extract_form_structure(pdf_path):
|
||||
structure = {
|
||||
"pages": [],
|
||||
"labels": [],
|
||||
"lines": [],
|
||||
"checkboxes": [],
|
||||
"row_boundaries": []
|
||||
}
|
||||
|
||||
with pdfplumber.open(pdf_path) as pdf:
|
||||
for page_num, page in enumerate(pdf.pages, 1):
|
||||
structure["pages"].append({
|
||||
"page_number": page_num,
|
||||
"width": float(page.width),
|
||||
"height": float(page.height)
|
||||
})
|
||||
|
||||
words = page.extract_words()
|
||||
for word in words:
|
||||
structure["labels"].append({
|
||||
"page": page_num,
|
||||
"text": word["text"],
|
||||
"x0": round(float(word["x0"]), 1),
|
||||
"top": round(float(word["top"]), 1),
|
||||
"x1": round(float(word["x1"]), 1),
|
||||
"bottom": round(float(word["bottom"]), 1)
|
||||
})
|
||||
|
||||
for line in page.lines:
|
||||
if abs(float(line["x1"]) - float(line["x0"])) > page.width * 0.5:
|
||||
structure["lines"].append({
|
||||
"page": page_num,
|
||||
"y": round(float(line["top"]), 1),
|
||||
"x0": round(float(line["x0"]), 1),
|
||||
"x1": round(float(line["x1"]), 1)
|
||||
})
|
||||
|
||||
for rect in page.rects:
|
||||
width = float(rect["x1"]) - float(rect["x0"])
|
||||
height = float(rect["bottom"]) - float(rect["top"])
|
||||
if 5 <= width <= 15 and 5 <= height <= 15 and abs(width - height) < 2:
|
||||
structure["checkboxes"].append({
|
||||
"page": page_num,
|
||||
"x0": round(float(rect["x0"]), 1),
|
||||
"top": round(float(rect["top"]), 1),
|
||||
"x1": round(float(rect["x1"]), 1),
|
||||
"bottom": round(float(rect["bottom"]), 1),
|
||||
"center_x": round((float(rect["x0"]) + float(rect["x1"])) / 2, 1),
|
||||
"center_y": round((float(rect["top"]) + float(rect["bottom"])) / 2, 1)
|
||||
})
|
||||
|
||||
lines_by_page = {}
|
||||
for line in structure["lines"]:
|
||||
page = line["page"]
|
||||
if page not in lines_by_page:
|
||||
lines_by_page[page] = []
|
||||
lines_by_page[page].append(line["y"])
|
||||
|
||||
for page, y_coords in lines_by_page.items():
|
||||
y_coords = sorted(set(y_coords))
|
||||
for i in range(len(y_coords) - 1):
|
||||
structure["row_boundaries"].append({
|
||||
"page": page,
|
||||
"row_top": y_coords[i],
|
||||
"row_bottom": y_coords[i + 1],
|
||||
"row_height": round(y_coords[i + 1] - y_coords[i], 1)
|
||||
})
|
||||
|
||||
return structure
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: extract_form_structure.py <input.pdf> <output.json>")
|
||||
sys.exit(1)
|
||||
|
||||
pdf_path = sys.argv[1]
|
||||
output_path = sys.argv[2]
|
||||
|
||||
print(f"Extracting structure from {pdf_path}...")
|
||||
structure = extract_form_structure(pdf_path)
|
||||
|
||||
with open(output_path, "w") as f:
|
||||
json.dump(structure, f, indent=2)
|
||||
|
||||
print(f"Found:")
|
||||
print(f" - {len(structure['pages'])} pages")
|
||||
print(f" - {len(structure['labels'])} text labels")
|
||||
print(f" - {len(structure['lines'])} horizontal lines")
|
||||
print(f" - {len(structure['checkboxes'])} checkboxes")
|
||||
print(f" - {len(structure['row_boundaries'])} row boundaries")
|
||||
print(f"Saved to {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,98 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from pypdf import PdfReader, PdfWriter
|
||||
|
||||
from extract_form_field_info import get_field_info
|
||||
|
||||
|
||||
|
||||
|
||||
def fill_pdf_fields(input_pdf_path: str, fields_json_path: str, output_pdf_path: str):
|
||||
with open(fields_json_path) as f:
|
||||
fields = json.load(f)
|
||||
fields_by_page = {}
|
||||
for field in fields:
|
||||
if "value" in field:
|
||||
field_id = field["field_id"]
|
||||
page = field["page"]
|
||||
if page not in fields_by_page:
|
||||
fields_by_page[page] = {}
|
||||
fields_by_page[page][field_id] = field["value"]
|
||||
|
||||
reader = PdfReader(input_pdf_path)
|
||||
|
||||
has_error = False
|
||||
field_info = get_field_info(reader)
|
||||
fields_by_ids = {f["field_id"]: f for f in field_info}
|
||||
for field in fields:
|
||||
existing_field = fields_by_ids.get(field["field_id"])
|
||||
if not existing_field:
|
||||
has_error = True
|
||||
print(f"ERROR: `{field['field_id']}` is not a valid field ID")
|
||||
elif field["page"] != existing_field["page"]:
|
||||
has_error = True
|
||||
print(f"ERROR: Incorrect page number for `{field['field_id']}` (got {field['page']}, expected {existing_field['page']})")
|
||||
else:
|
||||
if "value" in field:
|
||||
err = validation_error_for_field_value(existing_field, field["value"])
|
||||
if err:
|
||||
print(err)
|
||||
has_error = True
|
||||
if has_error:
|
||||
sys.exit(1)
|
||||
|
||||
writer = PdfWriter(clone_from=reader)
|
||||
for page, field_values in fields_by_page.items():
|
||||
writer.update_page_form_field_values(writer.pages[page - 1], field_values, auto_regenerate=False)
|
||||
|
||||
writer.set_need_appearances_writer(True)
|
||||
|
||||
with open(output_pdf_path, "wb") as f:
|
||||
writer.write(f)
|
||||
|
||||
|
||||
def validation_error_for_field_value(field_info, field_value):
|
||||
field_type = field_info["type"]
|
||||
field_id = field_info["field_id"]
|
||||
if field_type == "checkbox":
|
||||
checked_val = field_info["checked_value"]
|
||||
unchecked_val = field_info["unchecked_value"]
|
||||
if field_value != checked_val and field_value != unchecked_val:
|
||||
return f'ERROR: Invalid value "{field_value}" for checkbox field "{field_id}". The checked value is "{checked_val}" and the unchecked value is "{unchecked_val}"'
|
||||
elif field_type == "radio_group":
|
||||
option_values = [opt["value"] for opt in field_info["radio_options"]]
|
||||
if field_value not in option_values:
|
||||
return f'ERROR: Invalid value "{field_value}" for radio group field "{field_id}". Valid values are: {option_values}'
|
||||
elif field_type == "choice":
|
||||
choice_values = [opt["value"] for opt in field_info["choice_options"]]
|
||||
if field_value not in choice_values:
|
||||
return f'ERROR: Invalid value "{field_value}" for choice field "{field_id}". Valid values are: {choice_values}'
|
||||
return None
|
||||
|
||||
|
||||
def monkeypatch_pydpf_method():
|
||||
from pypdf.generic import DictionaryObject
|
||||
from pypdf.constants import FieldDictionaryAttributes
|
||||
|
||||
original_get_inherited = DictionaryObject.get_inherited
|
||||
|
||||
def patched_get_inherited(self, key: str, default = None):
|
||||
result = original_get_inherited(self, key, default)
|
||||
if key == FieldDictionaryAttributes.Opt:
|
||||
if isinstance(result, list) and all(isinstance(v, list) and len(v) == 2 for v in result):
|
||||
result = [r[0] for r in result]
|
||||
return result
|
||||
|
||||
DictionaryObject.get_inherited = patched_get_inherited
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: fill_fillable_fields.py [input pdf] [field_values.json] [output pdf]")
|
||||
sys.exit(1)
|
||||
monkeypatch_pydpf_method()
|
||||
input_pdf = sys.argv[1]
|
||||
fields_json = sys.argv[2]
|
||||
output_pdf = sys.argv[3]
|
||||
fill_pdf_fields(input_pdf, fields_json, output_pdf)
|
||||
@@ -0,0 +1,107 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from pypdf import PdfReader, PdfWriter
|
||||
from pypdf.annotations import FreeText
|
||||
|
||||
|
||||
|
||||
|
||||
def transform_from_image_coords(bbox, image_width, image_height, pdf_width, pdf_height):
|
||||
x_scale = pdf_width / image_width
|
||||
y_scale = pdf_height / image_height
|
||||
|
||||
left = bbox[0] * x_scale
|
||||
right = bbox[2] * x_scale
|
||||
|
||||
top = pdf_height - (bbox[1] * y_scale)
|
||||
bottom = pdf_height - (bbox[3] * y_scale)
|
||||
|
||||
return left, bottom, right, top
|
||||
|
||||
|
||||
def transform_from_pdf_coords(bbox, pdf_height):
|
||||
left = bbox[0]
|
||||
right = bbox[2]
|
||||
|
||||
pypdf_top = pdf_height - bbox[1]
|
||||
pypdf_bottom = pdf_height - bbox[3]
|
||||
|
||||
return left, pypdf_bottom, right, pypdf_top
|
||||
|
||||
|
||||
def fill_pdf_form(input_pdf_path, fields_json_path, output_pdf_path):
|
||||
|
||||
with open(fields_json_path, "r") as f:
|
||||
fields_data = json.load(f)
|
||||
|
||||
reader = PdfReader(input_pdf_path)
|
||||
writer = PdfWriter()
|
||||
|
||||
writer.append(reader)
|
||||
|
||||
pdf_dimensions = {}
|
||||
for i, page in enumerate(reader.pages):
|
||||
mediabox = page.mediabox
|
||||
pdf_dimensions[i + 1] = [mediabox.width, mediabox.height]
|
||||
|
||||
annotations = []
|
||||
for field in fields_data["form_fields"]:
|
||||
page_num = field["page_number"]
|
||||
|
||||
page_info = next(p for p in fields_data["pages"] if p["page_number"] == page_num)
|
||||
pdf_width, pdf_height = pdf_dimensions[page_num]
|
||||
|
||||
if "pdf_width" in page_info:
|
||||
transformed_entry_box = transform_from_pdf_coords(
|
||||
field["entry_bounding_box"],
|
||||
float(pdf_height)
|
||||
)
|
||||
else:
|
||||
image_width = page_info["image_width"]
|
||||
image_height = page_info["image_height"]
|
||||
transformed_entry_box = transform_from_image_coords(
|
||||
field["entry_bounding_box"],
|
||||
image_width, image_height,
|
||||
float(pdf_width), float(pdf_height)
|
||||
)
|
||||
|
||||
if "entry_text" not in field or "text" not in field["entry_text"]:
|
||||
continue
|
||||
entry_text = field["entry_text"]
|
||||
text = entry_text["text"]
|
||||
if not text:
|
||||
continue
|
||||
|
||||
font_name = entry_text.get("font", "Arial")
|
||||
font_size = str(entry_text.get("font_size", 14)) + "pt"
|
||||
font_color = entry_text.get("font_color", "000000")
|
||||
|
||||
annotation = FreeText(
|
||||
text=text,
|
||||
rect=transformed_entry_box,
|
||||
font=font_name,
|
||||
font_size=font_size,
|
||||
font_color=font_color,
|
||||
border_color=None,
|
||||
background_color=None,
|
||||
)
|
||||
annotations.append(annotation)
|
||||
writer.add_annotation(page_number=page_num - 1, annotation=annotation)
|
||||
|
||||
with open(output_pdf_path, "wb") as output:
|
||||
writer.write(output)
|
||||
|
||||
print(f"Successfully filled PDF form and saved to {output_pdf_path}")
|
||||
print(f"Added {len(annotations)} text annotations")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: fill_pdf_form_with_annotations.py [input pdf] [fields.json] [output pdf]")
|
||||
sys.exit(1)
|
||||
input_pdf = sys.argv[1]
|
||||
fields_json = sys.argv[2]
|
||||
output_pdf = sys.argv[3]
|
||||
|
||||
fill_pdf_form(input_pdf, fields_json, output_pdf)
|
||||
@@ -0,0 +1,40 @@
|
||||
# OBLITERATUS Analysis Study Config
|
||||
# Usage: obliteratus run this-file.yaml --preset jailbreak
|
||||
#
|
||||
# Run analysis modules to understand refusal geometry BEFORE abliterating.
|
||||
# Useful for research or when you want to understand what you're removing.
|
||||
|
||||
# Model to analyze
|
||||
model:
|
||||
name: "meta-llama/Llama-3.1-8B-Instruct"
|
||||
dtype: "bfloat16"
|
||||
quantization: "4bit" # Saves VRAM for analysis
|
||||
device: "auto"
|
||||
|
||||
# Study configuration
|
||||
study:
|
||||
# Available presets: quick, full, attention, jailbreak, guardrail, knowledge
|
||||
preset: "jailbreak"
|
||||
|
||||
# Or specify individual strategies:
|
||||
# strategies:
|
||||
# - layer_removal
|
||||
# - head_pruning
|
||||
# - ffn_ablation
|
||||
# - embedding_ablation
|
||||
|
||||
# Analysis modules to run (subset of the 27 available)
|
||||
analysis:
|
||||
- alignment_imprint # Detect DPO/RLHF/CAI/SFT training method
|
||||
- concept_geometry # Map refusal cone geometry
|
||||
- logit_lens # Find which layer decides to refuse
|
||||
- anti_ouroboros # Detect self-repair tendency
|
||||
- cross_layer # Cross-layer alignment clustering
|
||||
- causal_tracing # Causal necessity of components
|
||||
- residual_stream # Attention vs MLP contribution
|
||||
|
||||
# Output
|
||||
output:
|
||||
directory: "./analysis-results"
|
||||
save_plots: true # Generate matplotlib visualizations
|
||||
save_report: true # Generate markdown report
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Add a new slide to an unpacked PPTX directory.
|
||||
|
||||
Usage: python add_slide.py <unpacked_dir> <source>
|
||||
|
||||
The source can be:
|
||||
- A slide file (e.g., slide2.xml) - duplicates the slide
|
||||
- A layout file (e.g., slideLayout2.xml) - creates from layout
|
||||
|
||||
Examples:
|
||||
python add_slide.py unpacked/ slide2.xml
|
||||
# Duplicates slide2, creates slide5.xml
|
||||
|
||||
python add_slide.py unpacked/ slideLayout2.xml
|
||||
# Creates slide5.xml from slideLayout2.xml
|
||||
|
||||
To see available layouts: ls unpacked/ppt/slideLayouts/
|
||||
|
||||
Prints the <p:sldId> element to add to presentation.xml.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_next_slide_number(slides_dir: Path) -> int:
|
||||
existing = [int(m.group(1)) for f in slides_dir.glob("slide*.xml")
|
||||
if (m := re.match(r"slide(\d+)\.xml", f.name))]
|
||||
return max(existing) + 1 if existing else 1
|
||||
|
||||
|
||||
def create_slide_from_layout(unpacked_dir: Path, layout_file: str) -> None:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
rels_dir = slides_dir / "_rels"
|
||||
layouts_dir = unpacked_dir / "ppt" / "slideLayouts"
|
||||
|
||||
layout_path = layouts_dir / layout_file
|
||||
if not layout_path.exists():
|
||||
print(f"Error: {layout_path} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
next_num = get_next_slide_number(slides_dir)
|
||||
dest = f"slide{next_num}.xml"
|
||||
dest_slide = slides_dir / dest
|
||||
dest_rels = rels_dir / f"{dest}.rels"
|
||||
|
||||
slide_xml = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<p:sld xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main">
|
||||
<p:cSld>
|
||||
<p:spTree>
|
||||
<p:nvGrpSpPr>
|
||||
<p:cNvPr id="1" name=""/>
|
||||
<p:cNvGrpSpPr/>
|
||||
<p:nvPr/>
|
||||
</p:nvGrpSpPr>
|
||||
<p:grpSpPr>
|
||||
<a:xfrm>
|
||||
<a:off x="0" y="0"/>
|
||||
<a:ext cx="0" cy="0"/>
|
||||
<a:chOff x="0" y="0"/>
|
||||
<a:chExt cx="0" cy="0"/>
|
||||
</a:xfrm>
|
||||
</p:grpSpPr>
|
||||
</p:spTree>
|
||||
</p:cSld>
|
||||
<p:clrMapOvr>
|
||||
<a:masterClrMapping/>
|
||||
</p:clrMapOvr>
|
||||
</p:sld>'''
|
||||
dest_slide.write_text(slide_xml, encoding="utf-8")
|
||||
|
||||
rels_dir.mkdir(exist_ok=True)
|
||||
rels_xml = f'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" Target="../slideLayouts/{layout_file}"/>
|
||||
</Relationships>'''
|
||||
dest_rels.write_text(rels_xml, encoding="utf-8")
|
||||
|
||||
_add_to_content_types(unpacked_dir, dest)
|
||||
|
||||
rid = _add_to_presentation_rels(unpacked_dir, dest)
|
||||
|
||||
next_slide_id = _get_next_slide_id(unpacked_dir)
|
||||
|
||||
print(f"Created {dest} from {layout_file}")
|
||||
print(f'Add to presentation.xml <p:sldIdLst>: <p:sldId id="{next_slide_id}" r:id="{rid}"/>')
|
||||
|
||||
|
||||
def duplicate_slide(unpacked_dir: Path, source: str) -> None:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
rels_dir = slides_dir / "_rels"
|
||||
|
||||
source_slide = slides_dir / source
|
||||
|
||||
if not source_slide.exists():
|
||||
print(f"Error: {source_slide} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
next_num = get_next_slide_number(slides_dir)
|
||||
dest = f"slide{next_num}.xml"
|
||||
dest_slide = slides_dir / dest
|
||||
|
||||
source_rels = rels_dir / f"{source}.rels"
|
||||
dest_rels = rels_dir / f"{dest}.rels"
|
||||
|
||||
shutil.copy2(source_slide, dest_slide)
|
||||
|
||||
if source_rels.exists():
|
||||
shutil.copy2(source_rels, dest_rels)
|
||||
|
||||
rels_content = dest_rels.read_text(encoding="utf-8")
|
||||
rels_content = re.sub(
|
||||
r'\s*<Relationship[^>]*Type="[^"]*notesSlide"[^>]*/>\s*',
|
||||
"\n",
|
||||
rels_content,
|
||||
)
|
||||
dest_rels.write_text(rels_content, encoding="utf-8")
|
||||
|
||||
_add_to_content_types(unpacked_dir, dest)
|
||||
|
||||
rid = _add_to_presentation_rels(unpacked_dir, dest)
|
||||
|
||||
next_slide_id = _get_next_slide_id(unpacked_dir)
|
||||
|
||||
print(f"Created {dest} from {source}")
|
||||
print(f'Add to presentation.xml <p:sldIdLst>: <p:sldId id="{next_slide_id}" r:id="{rid}"/>')
|
||||
|
||||
|
||||
def _add_to_content_types(unpacked_dir: Path, dest: str) -> None:
|
||||
content_types_path = unpacked_dir / "[Content_Types].xml"
|
||||
content_types = content_types_path.read_text(encoding="utf-8")
|
||||
|
||||
new_override = f'<Override PartName="/ppt/slides/{dest}" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'
|
||||
|
||||
if f"/ppt/slides/{dest}" not in content_types:
|
||||
content_types = content_types.replace("</Types>", f" {new_override}\n</Types>")
|
||||
content_types_path.write_text(content_types, encoding="utf-8")
|
||||
|
||||
|
||||
def _add_to_presentation_rels(unpacked_dir: Path, dest: str) -> str:
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
pres_rels = pres_rels_path.read_text(encoding="utf-8")
|
||||
|
||||
rids = [int(m) for m in re.findall(r'Id="rId(\d+)"', pres_rels)]
|
||||
next_rid = max(rids) + 1 if rids else 1
|
||||
rid = f"rId{next_rid}"
|
||||
|
||||
new_rel = f'<Relationship Id="{rid}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" Target="slides/{dest}"/>'
|
||||
|
||||
if f"slides/{dest}" not in pres_rels:
|
||||
pres_rels = pres_rels.replace("</Relationships>", f" {new_rel}\n</Relationships>")
|
||||
pres_rels_path.write_text(pres_rels, encoding="utf-8")
|
||||
|
||||
return rid
|
||||
|
||||
|
||||
def _get_next_slide_id(unpacked_dir: Path) -> int:
|
||||
pres_path = unpacked_dir / "ppt" / "presentation.xml"
|
||||
pres_content = pres_path.read_text(encoding="utf-8")
|
||||
slide_ids = [int(m) for m in re.findall(r'<p:sldId[^>]*id="(\d+)"', pres_content)]
|
||||
return max(slide_ids) + 1 if slide_ids else 256
|
||||
|
||||
|
||||
def parse_source(source: str) -> tuple[str, str | None]:
|
||||
if source.startswith("slideLayout") and source.endswith(".xml"):
|
||||
return ("layout", source)
|
||||
|
||||
return ("slide", None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python add_slide.py <unpacked_dir> <source>", file=sys.stderr)
|
||||
print("", file=sys.stderr)
|
||||
print("Source can be:", file=sys.stderr)
|
||||
print(" slide2.xml - duplicate an existing slide", file=sys.stderr)
|
||||
print(" slideLayout2.xml - create from a layout template", file=sys.stderr)
|
||||
print("", file=sys.stderr)
|
||||
print("To see available layouts: ls <unpacked_dir>/ppt/slideLayouts/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
unpacked_dir = Path(sys.argv[1])
|
||||
source = sys.argv[2]
|
||||
|
||||
if not unpacked_dir.exists():
|
||||
print(f"Error: {unpacked_dir} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
source_type, layout_file = parse_source(source)
|
||||
|
||||
if source_type == "layout" and layout_file is not None:
|
||||
create_slide_from_layout(unpacked_dir, layout_file)
|
||||
else:
|
||||
duplicate_slide(unpacked_dir, source)
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Remove unreferenced files from an unpacked PPTX directory.
|
||||
|
||||
Usage: python clean.py <unpacked_dir>
|
||||
|
||||
Example:
|
||||
python clean.py unpacked/
|
||||
|
||||
This script removes:
|
||||
- Orphaned slides (not in sldIdLst) and their relationships
|
||||
- [trash] directory (unreferenced files)
|
||||
- Orphaned .rels files for deleted resources
|
||||
- Unreferenced media, embeddings, charts, diagrams, drawings, ink files
|
||||
- Unreferenced theme files
|
||||
- Unreferenced notes slides
|
||||
- Content-Type overrides for deleted files
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def get_slides_in_sldidlst(unpacked_dir: Path) -> set[str]:
|
||||
pres_path = unpacked_dir / "ppt" / "presentation.xml"
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
|
||||
if not pres_path.exists() or not pres_rels_path.exists():
|
||||
return set()
|
||||
|
||||
rels_dom = defusedxml.minidom.parse(str(pres_rels_path))
|
||||
rid_to_slide = {}
|
||||
for rel in rels_dom.getElementsByTagName("Relationship"):
|
||||
rid = rel.getAttribute("Id")
|
||||
target = rel.getAttribute("Target")
|
||||
rel_type = rel.getAttribute("Type")
|
||||
if "slide" in rel_type and target.startswith("slides/"):
|
||||
rid_to_slide[rid] = target.replace("slides/", "")
|
||||
|
||||
pres_content = pres_path.read_text(encoding="utf-8")
|
||||
referenced_rids = set(re.findall(r'<p:sldId[^>]*r:id="([^"]+)"', pres_content))
|
||||
|
||||
return {rid_to_slide[rid] for rid in referenced_rids if rid in rid_to_slide}
|
||||
|
||||
|
||||
def remove_orphaned_slides(unpacked_dir: Path) -> list[str]:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
slides_rels_dir = slides_dir / "_rels"
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
|
||||
if not slides_dir.exists():
|
||||
return []
|
||||
|
||||
referenced_slides = get_slides_in_sldidlst(unpacked_dir)
|
||||
removed = []
|
||||
|
||||
for slide_file in slides_dir.glob("slide*.xml"):
|
||||
if slide_file.name not in referenced_slides:
|
||||
rel_path = slide_file.relative_to(unpacked_dir)
|
||||
slide_file.unlink()
|
||||
removed.append(str(rel_path))
|
||||
|
||||
rels_file = slides_rels_dir / f"{slide_file.name}.rels"
|
||||
if rels_file.exists():
|
||||
rels_file.unlink()
|
||||
removed.append(str(rels_file.relative_to(unpacked_dir)))
|
||||
|
||||
if removed and pres_rels_path.exists():
|
||||
rels_dom = defusedxml.minidom.parse(str(pres_rels_path))
|
||||
changed = False
|
||||
|
||||
for rel in list(rels_dom.getElementsByTagName("Relationship")):
|
||||
target = rel.getAttribute("Target")
|
||||
if target.startswith("slides/"):
|
||||
slide_name = target.replace("slides/", "")
|
||||
if slide_name not in referenced_slides:
|
||||
if rel.parentNode:
|
||||
rel.parentNode.removeChild(rel)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(pres_rels_path, "wb") as f:
|
||||
f.write(rels_dom.toxml(encoding="utf-8"))
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def remove_trash_directory(unpacked_dir: Path) -> list[str]:
|
||||
trash_dir = unpacked_dir / "[trash]"
|
||||
removed = []
|
||||
|
||||
if trash_dir.exists() and trash_dir.is_dir():
|
||||
for file_path in trash_dir.iterdir():
|
||||
if file_path.is_file():
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
removed.append(str(rel_path))
|
||||
file_path.unlink()
|
||||
trash_dir.rmdir()
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def get_slide_referenced_files(unpacked_dir: Path) -> set:
|
||||
referenced = set()
|
||||
slides_rels_dir = unpacked_dir / "ppt" / "slides" / "_rels"
|
||||
|
||||
if not slides_rels_dir.exists():
|
||||
return referenced
|
||||
|
||||
for rels_file in slides_rels_dir.glob("*.rels"):
|
||||
dom = defusedxml.minidom.parse(str(rels_file))
|
||||
for rel in dom.getElementsByTagName("Relationship"):
|
||||
target = rel.getAttribute("Target")
|
||||
if not target:
|
||||
continue
|
||||
target_path = (rels_file.parent.parent / target).resolve()
|
||||
try:
|
||||
referenced.add(target_path.relative_to(unpacked_dir.resolve()))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def remove_orphaned_rels_files(unpacked_dir: Path) -> list[str]:
|
||||
resource_dirs = ["charts", "diagrams", "drawings"]
|
||||
removed = []
|
||||
slide_referenced = get_slide_referenced_files(unpacked_dir)
|
||||
|
||||
for dir_name in resource_dirs:
|
||||
rels_dir = unpacked_dir / "ppt" / dir_name / "_rels"
|
||||
if not rels_dir.exists():
|
||||
continue
|
||||
|
||||
for rels_file in rels_dir.glob("*.rels"):
|
||||
resource_file = rels_dir.parent / rels_file.name.replace(".rels", "")
|
||||
try:
|
||||
resource_rel_path = resource_file.resolve().relative_to(unpacked_dir.resolve())
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if not resource_file.exists() or resource_rel_path not in slide_referenced:
|
||||
rels_file.unlink()
|
||||
rel_path = rels_file.relative_to(unpacked_dir)
|
||||
removed.append(str(rel_path))
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def get_referenced_files(unpacked_dir: Path) -> set:
|
||||
referenced = set()
|
||||
|
||||
for rels_file in unpacked_dir.rglob("*.rels"):
|
||||
dom = defusedxml.minidom.parse(str(rels_file))
|
||||
for rel in dom.getElementsByTagName("Relationship"):
|
||||
target = rel.getAttribute("Target")
|
||||
if not target:
|
||||
continue
|
||||
target_path = (rels_file.parent.parent / target).resolve()
|
||||
try:
|
||||
referenced.add(target_path.relative_to(unpacked_dir.resolve()))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def remove_orphaned_files(unpacked_dir: Path, referenced: set) -> list[str]:
|
||||
resource_dirs = ["media", "embeddings", "charts", "diagrams", "tags", "drawings", "ink"]
|
||||
removed = []
|
||||
|
||||
for dir_name in resource_dirs:
|
||||
dir_path = unpacked_dir / "ppt" / dir_name
|
||||
if not dir_path.exists():
|
||||
continue
|
||||
|
||||
for file_path in dir_path.glob("*"):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
if rel_path not in referenced:
|
||||
file_path.unlink()
|
||||
removed.append(str(rel_path))
|
||||
|
||||
theme_dir = unpacked_dir / "ppt" / "theme"
|
||||
if theme_dir.exists():
|
||||
for file_path in theme_dir.glob("theme*.xml"):
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
if rel_path not in referenced:
|
||||
file_path.unlink()
|
||||
removed.append(str(rel_path))
|
||||
theme_rels = theme_dir / "_rels" / f"{file_path.name}.rels"
|
||||
if theme_rels.exists():
|
||||
theme_rels.unlink()
|
||||
removed.append(str(theme_rels.relative_to(unpacked_dir)))
|
||||
|
||||
notes_dir = unpacked_dir / "ppt" / "notesSlides"
|
||||
if notes_dir.exists():
|
||||
for file_path in notes_dir.glob("*.xml"):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
if rel_path not in referenced:
|
||||
file_path.unlink()
|
||||
removed.append(str(rel_path))
|
||||
|
||||
notes_rels_dir = notes_dir / "_rels"
|
||||
if notes_rels_dir.exists():
|
||||
for file_path in notes_rels_dir.glob("*.rels"):
|
||||
notes_file = notes_dir / file_path.name.replace(".rels", "")
|
||||
if not notes_file.exists():
|
||||
file_path.unlink()
|
||||
removed.append(str(file_path.relative_to(unpacked_dir)))
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def update_content_types(unpacked_dir: Path, removed_files: list[str]) -> None:
|
||||
ct_path = unpacked_dir / "[Content_Types].xml"
|
||||
if not ct_path.exists():
|
||||
return
|
||||
|
||||
dom = defusedxml.minidom.parse(str(ct_path))
|
||||
changed = False
|
||||
|
||||
for override in list(dom.getElementsByTagName("Override")):
|
||||
part_name = override.getAttribute("PartName").lstrip("/")
|
||||
if part_name in removed_files:
|
||||
if override.parentNode:
|
||||
override.parentNode.removeChild(override)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(ct_path, "wb") as f:
|
||||
f.write(dom.toxml(encoding="utf-8"))
|
||||
|
||||
|
||||
def clean_unused_files(unpacked_dir: Path) -> list[str]:
|
||||
all_removed = []
|
||||
|
||||
slides_removed = remove_orphaned_slides(unpacked_dir)
|
||||
all_removed.extend(slides_removed)
|
||||
|
||||
trash_removed = remove_trash_directory(unpacked_dir)
|
||||
all_removed.extend(trash_removed)
|
||||
|
||||
while True:
|
||||
removed_rels = remove_orphaned_rels_files(unpacked_dir)
|
||||
referenced = get_referenced_files(unpacked_dir)
|
||||
removed_files = remove_orphaned_files(unpacked_dir, referenced)
|
||||
|
||||
total_removed = removed_rels + removed_files
|
||||
if not total_removed:
|
||||
break
|
||||
|
||||
all_removed.extend(total_removed)
|
||||
|
||||
if all_removed:
|
||||
update_content_types(unpacked_dir, all_removed)
|
||||
|
||||
return all_removed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python clean.py <unpacked_dir>", file=sys.stderr)
|
||||
print("Example: python clean.py unpacked/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
unpacked_dir = Path(sys.argv[1])
|
||||
|
||||
if not unpacked_dir.exists():
|
||||
print(f"Error: {unpacked_dir} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
removed = clean_unused_files(unpacked_dir)
|
||||
|
||||
if removed:
|
||||
print(f"Removed {len(removed)} unreferenced files:")
|
||||
for f in removed:
|
||||
print(f" {f}")
|
||||
else:
|
||||
print("No unreferenced files found")
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Merge adjacent runs with identical formatting in DOCX.
|
||||
|
||||
Merges adjacent <w:r> elements that have identical <w:rPr> properties.
|
||||
Works on runs in paragraphs and inside tracked changes (<w:ins>, <w:del>).
|
||||
|
||||
Also:
|
||||
- Removes rsid attributes from runs (revision metadata that doesn't affect rendering)
|
||||
- Removes proofErr elements (spell/grammar markers that block merging)
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
|
||||
|
||||
def merge_runs(input_dir: str) -> tuple[int, str]:
|
||||
doc_xml = Path(input_dir) / "word" / "document.xml"
|
||||
|
||||
if not doc_xml.exists():
|
||||
return 0, f"Error: {doc_xml} not found"
|
||||
|
||||
try:
|
||||
dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8"))
|
||||
root = dom.documentElement
|
||||
|
||||
_remove_elements(root, "proofErr")
|
||||
_strip_run_rsid_attrs(root)
|
||||
|
||||
containers = {run.parentNode for run in _find_elements(root, "r")}
|
||||
|
||||
merge_count = 0
|
||||
for container in containers:
|
||||
merge_count += _merge_runs_in(container)
|
||||
|
||||
doc_xml.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
return merge_count, f"Merged {merge_count} runs"
|
||||
|
||||
except Exception as e:
|
||||
return 0, f"Error: {e}"
|
||||
|
||||
|
||||
|
||||
|
||||
def _find_elements(root, tag: str) -> list:
|
||||
results = []
|
||||
|
||||
def traverse(node):
|
||||
if node.nodeType == node.ELEMENT_NODE:
|
||||
name = node.localName or node.tagName
|
||||
if name == tag or name.endswith(f":{tag}"):
|
||||
results.append(node)
|
||||
for child in node.childNodes:
|
||||
traverse(child)
|
||||
|
||||
traverse(root)
|
||||
return results
|
||||
|
||||
|
||||
def _get_child(parent, tag: str):
|
||||
for child in parent.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name == tag or name.endswith(f":{tag}"):
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def _get_children(parent, tag: str) -> list:
|
||||
results = []
|
||||
for child in parent.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name == tag or name.endswith(f":{tag}"):
|
||||
results.append(child)
|
||||
return results
|
||||
|
||||
|
||||
def _is_adjacent(elem1, elem2) -> bool:
|
||||
node = elem1.nextSibling
|
||||
while node:
|
||||
if node == elem2:
|
||||
return True
|
||||
if node.nodeType == node.ELEMENT_NODE:
|
||||
return False
|
||||
if node.nodeType == node.TEXT_NODE and node.data.strip():
|
||||
return False
|
||||
node = node.nextSibling
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def _remove_elements(root, tag: str):
|
||||
for elem in _find_elements(root, tag):
|
||||
if elem.parentNode:
|
||||
elem.parentNode.removeChild(elem)
|
||||
|
||||
|
||||
def _strip_run_rsid_attrs(root):
|
||||
for run in _find_elements(root, "r"):
|
||||
for attr in list(run.attributes.values()):
|
||||
if "rsid" in attr.name.lower():
|
||||
run.removeAttribute(attr.name)
|
||||
|
||||
|
||||
|
||||
|
||||
def _merge_runs_in(container) -> int:
|
||||
merge_count = 0
|
||||
run = _first_child_run(container)
|
||||
|
||||
while run:
|
||||
while True:
|
||||
next_elem = _next_element_sibling(run)
|
||||
if next_elem and _is_run(next_elem) and _can_merge(run, next_elem):
|
||||
_merge_run_content(run, next_elem)
|
||||
container.removeChild(next_elem)
|
||||
merge_count += 1
|
||||
else:
|
||||
break
|
||||
|
||||
_consolidate_text(run)
|
||||
run = _next_sibling_run(run)
|
||||
|
||||
return merge_count
|
||||
|
||||
|
||||
def _first_child_run(container):
|
||||
for child in container.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE and _is_run(child):
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def _next_element_sibling(node):
|
||||
sibling = node.nextSibling
|
||||
while sibling:
|
||||
if sibling.nodeType == sibling.ELEMENT_NODE:
|
||||
return sibling
|
||||
sibling = sibling.nextSibling
|
||||
return None
|
||||
|
||||
|
||||
def _next_sibling_run(node):
|
||||
sibling = node.nextSibling
|
||||
while sibling:
|
||||
if sibling.nodeType == sibling.ELEMENT_NODE:
|
||||
if _is_run(sibling):
|
||||
return sibling
|
||||
sibling = sibling.nextSibling
|
||||
return None
|
||||
|
||||
|
||||
def _is_run(node) -> bool:
|
||||
name = node.localName or node.tagName
|
||||
return name == "r" or name.endswith(":r")
|
||||
|
||||
|
||||
def _can_merge(run1, run2) -> bool:
|
||||
rpr1 = _get_child(run1, "rPr")
|
||||
rpr2 = _get_child(run2, "rPr")
|
||||
|
||||
if (rpr1 is None) != (rpr2 is None):
|
||||
return False
|
||||
if rpr1 is None:
|
||||
return True
|
||||
return rpr1.toxml() == rpr2.toxml()
|
||||
|
||||
|
||||
def _merge_run_content(target, source):
|
||||
for child in list(source.childNodes):
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name != "rPr" and not name.endswith(":rPr"):
|
||||
target.appendChild(child)
|
||||
|
||||
|
||||
def _consolidate_text(run):
|
||||
t_elements = _get_children(run, "t")
|
||||
|
||||
for i in range(len(t_elements) - 1, 0, -1):
|
||||
curr, prev = t_elements[i], t_elements[i - 1]
|
||||
|
||||
if _is_adjacent(prev, curr):
|
||||
prev_text = prev.firstChild.data if prev.firstChild else ""
|
||||
curr_text = curr.firstChild.data if curr.firstChild else ""
|
||||
merged = prev_text + curr_text
|
||||
|
||||
if prev.firstChild:
|
||||
prev.firstChild.data = merged
|
||||
else:
|
||||
prev.appendChild(run.ownerDocument.createTextNode(merged))
|
||||
|
||||
if merged.startswith(" ") or merged.endswith(" "):
|
||||
prev.setAttribute("xml:space", "preserve")
|
||||
elif prev.hasAttribute("xml:space"):
|
||||
prev.removeAttribute("xml:space")
|
||||
|
||||
run.removeChild(curr)
|
||||
@@ -0,0 +1,167 @@
|
||||
---
|
||||
name: jupyter-live-kernel
|
||||
description: "Iterative Python via live Jupyter kernel (hamelnb)."
|
||||
version: 1.0.0
|
||||
author: Hermes Agent
|
||||
license: MIT
|
||||
platforms: [linux, macos, windows]
|
||||
metadata:
|
||||
hermes:
|
||||
tags: [jupyter, notebook, repl, data-science, exploration, iterative]
|
||||
category: data-science
|
||||
---
|
||||
|
||||
# Jupyter Live Kernel (hamelnb)
|
||||
|
||||
Gives you a **stateful Python REPL** via a live Jupyter kernel. Variables persist
|
||||
across executions. Use this instead of `execute_code` when you need to build up
|
||||
state incrementally, explore APIs, inspect DataFrames, or iterate on complex code.
|
||||
|
||||
## When to Use This vs Other Tools
|
||||
|
||||
| Tool | Use When |
|
||||
|------|----------|
|
||||
| **This skill** | Iterative exploration, state across steps, data science, ML, "let me try this and check" |
|
||||
| `execute_code` | One-shot scripts needing hermes tool access (web_search, file ops). Stateless. |
|
||||
| `terminal` | Shell commands, builds, installs, git, process management |
|
||||
|
||||
**Rule of thumb:** If you'd want a Jupyter notebook for the task, use this skill.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **uv** must be installed (check: `which uv`)
|
||||
2. **JupyterLab** must be installed: `uv tool install jupyterlab`
|
||||
3. A Jupyter server must be running (see Setup below)
|
||||
|
||||
## Setup
|
||||
|
||||
The hamelnb script location:
|
||||
```
|
||||
SCRIPT="$HOME/.agent-skills/hamelnb/skills/jupyter-live-kernel/scripts/jupyter_live_kernel.py"
|
||||
```
|
||||
|
||||
If not cloned yet:
|
||||
```
|
||||
git clone https://github.com/hamelsmu/hamelnb.git ~/.agent-skills/hamelnb
|
||||
```
|
||||
|
||||
### Starting JupyterLab
|
||||
|
||||
Check if a server is already running:
|
||||
```
|
||||
uv run "$SCRIPT" servers
|
||||
```
|
||||
|
||||
If no servers found, start one:
|
||||
```
|
||||
jupyter-lab --no-browser --port=8888 --notebook-dir=$HOME/notebooks \
|
||||
--IdentityProvider.token='' --ServerApp.password='' > /tmp/jupyter.log 2>&1 &
|
||||
sleep 3
|
||||
```
|
||||
|
||||
Note: Token/password disabled for local agent access. The server runs headless.
|
||||
|
||||
### Creating a Notebook for REPL Use
|
||||
|
||||
If you just need a REPL (no existing notebook), create a minimal notebook file:
|
||||
```
|
||||
mkdir -p ~/notebooks
|
||||
```
|
||||
Write a minimal .ipynb JSON file with one empty code cell, then start a kernel
|
||||
session via the Jupyter REST API:
|
||||
```
|
||||
curl -s -X POST http://127.0.0.1:8888/api/sessions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"path":"scratch.ipynb","type":"notebook","name":"scratch.ipynb","kernel":{"name":"python3"}}'
|
||||
```
|
||||
|
||||
## Core Workflow
|
||||
|
||||
All commands return structured JSON. Always use `--compact` to save tokens.
|
||||
|
||||
### 1. Discover servers and notebooks
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" servers --compact
|
||||
uv run "$SCRIPT" notebooks --compact
|
||||
```
|
||||
|
||||
### 2. Execute code (primary operation)
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" execute --path <notebook.ipynb> --code '<python code>' --compact
|
||||
```
|
||||
|
||||
State persists across execute calls. Variables, imports, objects all survive.
|
||||
|
||||
Multi-line code works with $'...' quoting:
|
||||
```
|
||||
uv run "$SCRIPT" execute --path scratch.ipynb --code $'import os\nfiles = os.listdir(".")\nprint(f"Found {len(files)} files")' --compact
|
||||
```
|
||||
|
||||
### 3. Inspect live variables
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" variables --path <notebook.ipynb> list --compact
|
||||
uv run "$SCRIPT" variables --path <notebook.ipynb> preview --name <varname> --compact
|
||||
```
|
||||
|
||||
### 4. Edit notebook cells
|
||||
|
||||
```
|
||||
# View current cells
|
||||
uv run "$SCRIPT" contents --path <notebook.ipynb> --compact
|
||||
|
||||
# Insert a new cell
|
||||
uv run "$SCRIPT" edit --path <notebook.ipynb> insert \
|
||||
--at-index <N> --cell-type code --source '<code>' --compact
|
||||
|
||||
# Replace cell source (use cell-id from contents output)
|
||||
uv run "$SCRIPT" edit --path <notebook.ipynb> replace-source \
|
||||
--cell-id <id> --source '<new code>' --compact
|
||||
|
||||
# Delete a cell
|
||||
uv run "$SCRIPT" edit --path <notebook.ipynb> delete --cell-id <id> --compact
|
||||
```
|
||||
|
||||
### 5. Verification (restart + run all)
|
||||
|
||||
Only use when the user asks for a clean verification or you need to confirm
|
||||
the notebook runs top-to-bottom:
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" restart-run-all --path <notebook.ipynb> --save-outputs --compact
|
||||
```
|
||||
|
||||
## Practical Tips from Experience
|
||||
|
||||
1. **First execution after server start may timeout** — the kernel needs a moment
|
||||
to initialize. If you get a timeout, just retry.
|
||||
|
||||
2. **The kernel Python is JupyterLab's Python** — packages must be installed in
|
||||
that environment. If you need additional packages, install them into the
|
||||
JupyterLab tool environment first.
|
||||
|
||||
3. **--compact flag saves significant tokens** — always use it. JSON output can
|
||||
be very verbose without it.
|
||||
|
||||
4. **For pure REPL use**, create a scratch.ipynb and don't bother with cell editing.
|
||||
Just use `execute` repeatedly.
|
||||
|
||||
5. **Argument order matters** — subcommand flags like `--path` go BEFORE the
|
||||
sub-subcommand. E.g.: `variables --path nb.ipynb list` not `variables list --path nb.ipynb`.
|
||||
|
||||
6. **If a session doesn't exist yet**, you need to start one via the REST API
|
||||
(see Setup section). The tool can't execute without a live kernel session.
|
||||
|
||||
7. **Errors are returned as JSON** with traceback — read the `ename` and `evalue`
|
||||
fields to understand what went wrong.
|
||||
|
||||
8. **Occasional websocket timeouts** — some operations may timeout on first try,
|
||||
especially after a kernel restart. Retry once before escalating.
|
||||
|
||||
## Timeout Defaults
|
||||
|
||||
The script has a 30-second default timeout per execution. For long-running
|
||||
operations, pass `--timeout 120`. Use generous timeouts (60+) for initial
|
||||
setup or heavy computation.
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
description: Diagram creation skills for generating visual diagrams, flowcharts, architecture diagrams, and illustrations using tools like Excalidraw.
|
||||
---
|
||||
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
Helper for running LibreOffice (soffice) in environments where AF_UNIX
|
||||
sockets may be blocked (e.g., sandboxed VMs). Detects the restriction
|
||||
at runtime and applies an LD_PRELOAD shim if needed.
|
||||
|
||||
Usage:
|
||||
from office.soffice import run_soffice
|
||||
|
||||
result = run_soffice(["--headless", "--convert-to", "pdf", "input.docx"])
|
||||
|
||||
Call soffice through run_soffice, not through subprocess with get_soffice_env():
|
||||
the env dict carries the shim but names no user profile, and a non-root sandbox
|
||||
cannot bootstrap the default one -- soffice aborts with "User installation could
|
||||
not be completed" and converts nothing. get_soffice_env() stays public for the
|
||||
callers that build their own argv (they must pass -env:UserInstallation too).
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_soffice_env() -> dict:
|
||||
env = os.environ.copy()
|
||||
env["SAL_USE_VCLPLUGIN"] = "svp"
|
||||
|
||||
if _needs_shim():
|
||||
shim = _ensure_shim()
|
||||
env["LD_PRELOAD"] = str(shim)
|
||||
|
||||
return env
|
||||
|
||||
|
||||
def run_soffice(args: Iterable[str], **kwargs) -> subprocess.CompletedProcess:
|
||||
args = list(args)
|
||||
with contextlib.ExitStack() as stack:
|
||||
if not any(str(a).startswith("-env:UserInstallation") for a in args):
|
||||
profile = stack.enter_context(
|
||||
tempfile.TemporaryDirectory(prefix="lo_profile_", ignore_cleanup_errors=True)
|
||||
)
|
||||
args = [f"-env:UserInstallation={Path(profile).as_uri()}"] + args
|
||||
return subprocess.run(["soffice"] + args, env=get_soffice_env(), **kwargs)
|
||||
|
||||
|
||||
|
||||
_SHIM_SO = Path(tempfile.gettempdir()) / "lo_socket_shim.so"
|
||||
|
||||
|
||||
def _needs_shim() -> bool:
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.close()
|
||||
return False
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
|
||||
def _ensure_shim() -> Path:
|
||||
if _SHIM_SO.exists():
|
||||
return _SHIM_SO
|
||||
|
||||
src = Path(tempfile.gettempdir()) / "lo_socket_shim.c"
|
||||
src.write_text(_SHIM_SOURCE)
|
||||
subprocess.run(
|
||||
["gcc", "-shared", "-fPIC", "-o", str(_SHIM_SO), str(src), "-ldl"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
src.unlink()
|
||||
return _SHIM_SO
|
||||
|
||||
|
||||
|
||||
_SHIM_SOURCE = r"""
|
||||
#define _GNU_SOURCE
|
||||
#include <dlfcn.h>
|
||||
#include <errno.h>
|
||||
#include <signal.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int (*real_socket)(int, int, int);
|
||||
static int (*real_socketpair)(int, int, int, int[2]);
|
||||
static int (*real_listen)(int, int);
|
||||
static int (*real_accept)(int, struct sockaddr *, socklen_t *);
|
||||
static int (*real_close)(int);
|
||||
static int (*real_read)(int, void *, size_t);
|
||||
|
||||
/* Per-FD bookkeeping (FDs >= 1024 are passed through unshimmed). */
|
||||
static int is_shimmed[1024];
|
||||
static int peer_of[1024];
|
||||
static int wake_r[1024]; /* accept() blocks reading this */
|
||||
static int wake_w[1024]; /* close() writes to this */
|
||||
static int listener_fd = -1; /* FD that received listen() */
|
||||
|
||||
__attribute__((constructor))
|
||||
static void init(void) {
|
||||
real_socket = dlsym(RTLD_NEXT, "socket");
|
||||
real_socketpair = dlsym(RTLD_NEXT, "socketpair");
|
||||
real_listen = dlsym(RTLD_NEXT, "listen");
|
||||
real_accept = dlsym(RTLD_NEXT, "accept");
|
||||
real_close = dlsym(RTLD_NEXT, "close");
|
||||
real_read = dlsym(RTLD_NEXT, "read");
|
||||
for (int i = 0; i < 1024; i++) {
|
||||
peer_of[i] = -1;
|
||||
wake_r[i] = -1;
|
||||
wake_w[i] = -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- socket ---------------------------------------------------------- */
|
||||
int socket(int domain, int type, int protocol) {
|
||||
if (domain == AF_UNIX) {
|
||||
int fd = real_socket(domain, type, protocol);
|
||||
if (fd >= 0) return fd;
|
||||
/* socket(AF_UNIX) blocked – fall back to socketpair(). */
|
||||
int sv[2];
|
||||
if (real_socketpair(domain, type, protocol, sv) == 0) {
|
||||
if (sv[0] >= 0 && sv[0] < 1024) {
|
||||
is_shimmed[sv[0]] = 1;
|
||||
peer_of[sv[0]] = sv[1];
|
||||
int wp[2];
|
||||
if (pipe(wp) == 0) {
|
||||
wake_r[sv[0]] = wp[0];
|
||||
wake_w[sv[0]] = wp[1];
|
||||
}
|
||||
}
|
||||
return sv[0];
|
||||
}
|
||||
errno = EPERM;
|
||||
return -1;
|
||||
}
|
||||
return real_socket(domain, type, protocol);
|
||||
}
|
||||
|
||||
/* ---- listen ---------------------------------------------------------- */
|
||||
int listen(int sockfd, int backlog) {
|
||||
if (sockfd >= 0 && sockfd < 1024 && is_shimmed[sockfd]) {
|
||||
listener_fd = sockfd;
|
||||
return 0;
|
||||
}
|
||||
return real_listen(sockfd, backlog);
|
||||
}
|
||||
|
||||
/* ---- accept ---------------------------------------------------------- */
|
||||
int accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) {
|
||||
if (sockfd >= 0 && sockfd < 1024 && is_shimmed[sockfd]) {
|
||||
/* Block until close() writes to the wake pipe. */
|
||||
if (wake_r[sockfd] >= 0) {
|
||||
char buf;
|
||||
real_read(wake_r[sockfd], &buf, 1);
|
||||
}
|
||||
errno = ECONNABORTED;
|
||||
return -1;
|
||||
}
|
||||
return real_accept(sockfd, addr, addrlen);
|
||||
}
|
||||
|
||||
/* ---- close ----------------------------------------------------------- */
|
||||
int close(int fd) {
|
||||
if (fd >= 0 && fd < 1024 && is_shimmed[fd]) {
|
||||
int was_listener = (fd == listener_fd);
|
||||
is_shimmed[fd] = 0;
|
||||
|
||||
if (wake_w[fd] >= 0) { /* unblock accept() */
|
||||
char c = 0;
|
||||
write(wake_w[fd], &c, 1);
|
||||
real_close(wake_w[fd]);
|
||||
wake_w[fd] = -1;
|
||||
}
|
||||
if (wake_r[fd] >= 0) { real_close(wake_r[fd]); wake_r[fd] = -1; }
|
||||
if (peer_of[fd] >= 0) { real_close(peer_of[fd]); peer_of[fd] = -1; }
|
||||
|
||||
if (was_listener)
|
||||
_exit(0); /* conversion done – exit */
|
||||
}
|
||||
return real_close(fd);
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
result = run_soffice(sys.argv[1:])
|
||||
sys.exit(result.returncode)
|
||||
Executable
+308
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
Excel Formula Recalculation Script
|
||||
Recalculates all formulas in an Excel file using LibreOffice
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from office.soffice import get_soffice_env, run_soffice
|
||||
|
||||
from openpyxl import load_workbook
|
||||
|
||||
MACRO_FILENAME = "Module1.xba"
|
||||
SOFFICE_MISSING = "soffice not found on PATH; LibreOffice is required to recalculate"
|
||||
|
||||
MAX_LOCATIONS = 100
|
||||
|
||||
EXTERNAL_REF_RE = re.compile(r"""(?<![\w"\[])'?\[\d+\][^!"\[\]]*'?!""")
|
||||
|
||||
RECALCULATE_MACRO = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE script:module PUBLIC "-//OpenOffice.org//DTD OfficeDocument 1.0//EN" "module.dtd">
|
||||
<script:module xmlns:script="http://openoffice.org/2000/script" script:name="Module1" script:language="StarBasic">
|
||||
Sub RecalculateAndSave()
|
||||
ThisComponent.calculateAll()
|
||||
ThisComponent.store()
|
||||
ThisComponent.close(True)
|
||||
End Sub
|
||||
</script:module>"""
|
||||
|
||||
|
||||
def has_gtimeout():
|
||||
try:
|
||||
subprocess.run(
|
||||
["gtimeout", "--version"], capture_output=True, timeout=1, check=False
|
||||
)
|
||||
return True
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
|
||||
|
||||
def _stamp(path):
|
||||
st = os.stat(path)
|
||||
return st.st_mtime_ns, st.st_size
|
||||
|
||||
|
||||
def setup_libreoffice_macro(profile_dir: Path, timeout=30):
|
||||
url = profile_dir.as_uri()
|
||||
try:
|
||||
run_soffice(
|
||||
["--headless", "--terminate_after_init", f"-env:UserInstallation={url}"],
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return None, SOFFICE_MISSING
|
||||
except subprocess.TimeoutExpired:
|
||||
return None, "LibreOffice timed out creating its profile; formulas were NOT recalculated"
|
||||
|
||||
macro_dir = profile_dir / "user" / "basic" / "Standard"
|
||||
if not macro_dir.exists():
|
||||
return None, "LibreOffice did not create a usable profile; formulas were NOT recalculated"
|
||||
|
||||
try:
|
||||
(macro_dir / MACRO_FILENAME).write_text(RECALCULATE_MACRO)
|
||||
except OSError as e:
|
||||
return None, f"Could not install the recalculation macro: {e}"
|
||||
|
||||
return url, None
|
||||
|
||||
|
||||
def external_links_at_risk(filename):
|
||||
try:
|
||||
with zipfile.ZipFile(filename) as archive:
|
||||
names = archive.namelist()
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
return []
|
||||
if not any(n.startswith("xl/externalLinks/") for n in names):
|
||||
return []
|
||||
|
||||
with contextlib.ExitStack() as stack:
|
||||
formulas = load_workbook(filename, data_only=False)
|
||||
stack.callback(formulas.close)
|
||||
values = load_workbook(filename, data_only=True)
|
||||
stack.callback(values.close)
|
||||
|
||||
external_names = [
|
||||
name
|
||||
for name, dn in formulas.defined_names.items()
|
||||
if isinstance(getattr(dn, "value", None), str) and EXTERNAL_REF_RE.search(dn.value)
|
||||
]
|
||||
name_re = (
|
||||
re.compile(r"\b(" + "|".join(re.escape(n) for n in external_names) + r")\b")
|
||||
if external_names
|
||||
else None
|
||||
)
|
||||
|
||||
at_risk = []
|
||||
for sheet in formulas.sheetnames:
|
||||
ws = formulas[sheet]
|
||||
if not hasattr(ws, "iter_rows"):
|
||||
continue
|
||||
cached = values[sheet]
|
||||
for row in ws.iter_rows():
|
||||
for cell in row:
|
||||
v = cell.value
|
||||
if not (isinstance(v, str) and v.startswith("=")):
|
||||
continue
|
||||
reaches_out = EXTERNAL_REF_RE.search(v) or (name_re and name_re.search(v))
|
||||
if reaches_out and cached[cell.coordinate].value is None:
|
||||
at_risk.append(f"{sheet}!{cell.coordinate}")
|
||||
return at_risk
|
||||
|
||||
|
||||
def recalc(filename, timeout=30, force=False):
|
||||
if not Path(filename).exists():
|
||||
return {"error": f"File {filename} does not exist"}
|
||||
|
||||
abs_path = str(Path(filename).absolute())
|
||||
|
||||
if not os.access(abs_path, os.W_OK):
|
||||
return {"error": f"{filename} is not writable; recalculation rewrites the file in place"}
|
||||
|
||||
try:
|
||||
get_soffice_env()
|
||||
except Exception as e:
|
||||
return {"error": f"Could not prepare the LibreOffice environment: {e}"}
|
||||
|
||||
if not force:
|
||||
try:
|
||||
at_risk = external_links_at_risk(filename)
|
||||
except Exception as e:
|
||||
return {"error": f"Could not inspect {filename} for external links: {e}"}
|
||||
if at_risk:
|
||||
shown = at_risk[:MAX_LOCATIONS]
|
||||
return {
|
||||
"error": (
|
||||
"Refusing to recalculate: this workbook links to another workbook, and "
|
||||
f"{len(at_risk)} linked cell(s) have lost their cached value (openpyxl strips "
|
||||
"these on save). Recalculating would resolve them to #NAME? and delete the "
|
||||
"external links for good. Copy those cells' values from the original file "
|
||||
"before saving, or pass --force to accept the loss. Charts and conditional "
|
||||
"formats can hold external references too, so this list may not be exhaustive."
|
||||
),
|
||||
"external_link_cells": shown,
|
||||
"external_link_cells_truncated": max(0, len(at_risk) - len(shown)),
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="recalc-lo-profile-", ignore_cleanup_errors=True
|
||||
) as profile_dir:
|
||||
return _recalc_with_profile(filename, abs_path, timeout, Path(profile_dir))
|
||||
|
||||
|
||||
def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
|
||||
started = time.monotonic()
|
||||
profile_url, err = setup_libreoffice_macro(profile_dir, timeout=timeout)
|
||||
if err:
|
||||
return {"error": err}
|
||||
|
||||
timeout = max(5, int(timeout - (time.monotonic() - started)))
|
||||
|
||||
before = _stamp(abs_path)
|
||||
|
||||
cmd = [
|
||||
"soffice",
|
||||
"--headless",
|
||||
"--norestore",
|
||||
f"-env:UserInstallation={profile_url}",
|
||||
"vnd.sun.star.script:Standard.Module1.RecalculateAndSave?language=Basic&location=application",
|
||||
abs_path,
|
||||
]
|
||||
|
||||
if platform.system() == "Linux" and shutil.which("timeout"):
|
||||
cmd = ["timeout", str(timeout)] + cmd
|
||||
elif platform.system() == "Darwin" and has_gtimeout():
|
||||
cmd = ["gtimeout", str(timeout)] + cmd
|
||||
|
||||
timed_out = f"LibreOffice timed out after {timeout}s; formulas were NOT recalculated. Re-run with a longer timeout."
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, env=get_soffice_env(), timeout=timeout + 15
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"error": timed_out}
|
||||
except FileNotFoundError:
|
||||
return {"error": SOFFICE_MISSING}
|
||||
|
||||
if result.returncode == 124:
|
||||
return {"error": timed_out}
|
||||
|
||||
if result.returncode != 0:
|
||||
detail = (result.stderr or "").strip() or f"soffice exited {result.returncode}"
|
||||
return {"error": f"LibreOffice failed to recalculate: {detail}"}
|
||||
|
||||
if _stamp(abs_path) == before:
|
||||
return {
|
||||
"error": (
|
||||
"LibreOffice exited cleanly but never rewrote the file, so nothing was "
|
||||
"recalculated. Check that no other LibreOffice instance is running, then retry."
|
||||
)
|
||||
}
|
||||
|
||||
try:
|
||||
wb = load_workbook(filename, data_only=True)
|
||||
|
||||
excel_errors = [
|
||||
"#VALUE!",
|
||||
"#DIV/0!",
|
||||
"#REF!",
|
||||
"#NAME?",
|
||||
"#NULL!",
|
||||
"#NUM!",
|
||||
"#N/A",
|
||||
]
|
||||
error_details = {err: [] for err in excel_errors}
|
||||
total_errors = 0
|
||||
|
||||
for sheet_name in wb.sheetnames:
|
||||
ws = wb[sheet_name]
|
||||
if not hasattr(ws, "iter_rows"):
|
||||
continue
|
||||
for row in ws.iter_rows():
|
||||
for cell in row:
|
||||
if cell.value is not None and isinstance(cell.value, str):
|
||||
for err in excel_errors:
|
||||
if err in cell.value:
|
||||
location = f"{sheet_name}!{cell.coordinate}"
|
||||
error_details[err].append(location)
|
||||
total_errors += 1
|
||||
break
|
||||
|
||||
result = {
|
||||
"status": "success" if total_errors == 0 else "errors_found",
|
||||
"total_errors": total_errors,
|
||||
"error_summary": {},
|
||||
}
|
||||
|
||||
for err_type, locations in error_details.items():
|
||||
if locations:
|
||||
entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]}
|
||||
if len(locations) > MAX_LOCATIONS:
|
||||
entry["locations_truncated"] = len(locations) - MAX_LOCATIONS
|
||||
result["error_summary"][err_type] = entry
|
||||
|
||||
wb.close()
|
||||
|
||||
wb_formulas = load_workbook(filename, data_only=False)
|
||||
formula_count = 0
|
||||
for sheet_name in wb_formulas.sheetnames:
|
||||
ws = wb_formulas[sheet_name]
|
||||
if not hasattr(ws, "iter_rows"):
|
||||
continue
|
||||
for row in ws.iter_rows():
|
||||
for cell in row:
|
||||
if (
|
||||
cell.value
|
||||
and isinstance(cell.value, str)
|
||||
and cell.value.startswith("=")
|
||||
):
|
||||
formula_count += 1
|
||||
wb_formulas.close()
|
||||
|
||||
result["total_formulas"] = formula_count
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if a != "--force"]
|
||||
force = "--force" in sys.argv[1:]
|
||||
|
||||
if not args:
|
||||
print("Usage: python recalc.py <excel_file> [timeout_seconds] [--force]")
|
||||
print("\nRecalculates all formulas in an Excel file using LibreOffice")
|
||||
print("\nReturns JSON with error details:")
|
||||
print(" - status: 'success' or 'errors_found'")
|
||||
print(" - total_errors: Total number of Excel errors found")
|
||||
print(" - total_formulas: Number of formulas in the file")
|
||||
print(" - error_summary: Breakdown by error type with locations")
|
||||
print(" - #VALUE!, #DIV/0!, #REF!, #NAME?, #NULL!, #NUM!, #N/A")
|
||||
print("\nOn any failure the JSON has an 'error' key and no 'status'.")
|
||||
print("--force recalculates even when it would destroy external links.")
|
||||
sys.exit(1)
|
||||
|
||||
filename = args[0]
|
||||
timeout = int(args[1]) if len(args) > 1 else 30
|
||||
|
||||
result = recalc(filename, timeout, force=force)
|
||||
print(json.dumps(result, indent=2))
|
||||
sys.exit(1 if "error" in result else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user