Initial commit: 25 scripts Python sauvegardés
Date: 2026-08-28 11:43:34 EDT
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
# OBLITERATUS Analysis Study Config
|
||||
# Usage: obliteratus run this-file.yaml --preset jailbreak
|
||||
#
|
||||
# Run analysis modules to understand refusal geometry BEFORE abliterating.
|
||||
# Useful for research or when you want to understand what you're removing.
|
||||
|
||||
# Model to analyze
|
||||
model:
|
||||
name: "meta-llama/Llama-3.1-8B-Instruct"
|
||||
dtype: "bfloat16"
|
||||
quantization: "4bit" # Saves VRAM for analysis
|
||||
device: "auto"
|
||||
|
||||
# Study configuration
|
||||
study:
|
||||
# Available presets: quick, full, attention, jailbreak, guardrail, knowledge
|
||||
preset: "jailbreak"
|
||||
|
||||
# Or specify individual strategies:
|
||||
# strategies:
|
||||
# - layer_removal
|
||||
# - head_pruning
|
||||
# - ffn_ablation
|
||||
# - embedding_ablation
|
||||
|
||||
# Analysis modules to run (subset of the 27 available)
|
||||
analysis:
|
||||
- alignment_imprint # Detect DPO/RLHF/CAI/SFT training method
|
||||
- concept_geometry # Map refusal cone geometry
|
||||
- logit_lens # Find which layer decides to refuse
|
||||
- anti_ouroboros # Detect self-repair tendency
|
||||
- cross_layer # Cross-layer alignment clustering
|
||||
- causal_tracing # Causal necessity of components
|
||||
- residual_stream # Attention vs MLP contribution
|
||||
|
||||
# Output
|
||||
output:
|
||||
directory: "./analysis-results"
|
||||
save_plots: true # Generate matplotlib visualizations
|
||||
save_report: true # Generate markdown report
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Add a new slide to an unpacked PPTX directory.
|
||||
|
||||
Usage: python add_slide.py <unpacked_dir> <source>
|
||||
|
||||
The source can be:
|
||||
- A slide file (e.g., slide2.xml) - duplicates the slide
|
||||
- A layout file (e.g., slideLayout2.xml) - creates from layout
|
||||
|
||||
Examples:
|
||||
python add_slide.py unpacked/ slide2.xml
|
||||
# Duplicates slide2, creates slide5.xml
|
||||
|
||||
python add_slide.py unpacked/ slideLayout2.xml
|
||||
# Creates slide5.xml from slideLayout2.xml
|
||||
|
||||
To see available layouts: ls unpacked/ppt/slideLayouts/
|
||||
|
||||
Prints the <p:sldId> element to add to presentation.xml.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_next_slide_number(slides_dir: Path) -> int:
|
||||
existing = [int(m.group(1)) for f in slides_dir.glob("slide*.xml")
|
||||
if (m := re.match(r"slide(\d+)\.xml", f.name))]
|
||||
return max(existing) + 1 if existing else 1
|
||||
|
||||
|
||||
def create_slide_from_layout(unpacked_dir: Path, layout_file: str) -> None:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
rels_dir = slides_dir / "_rels"
|
||||
layouts_dir = unpacked_dir / "ppt" / "slideLayouts"
|
||||
|
||||
layout_path = layouts_dir / layout_file
|
||||
if not layout_path.exists():
|
||||
print(f"Error: {layout_path} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
next_num = get_next_slide_number(slides_dir)
|
||||
dest = f"slide{next_num}.xml"
|
||||
dest_slide = slides_dir / dest
|
||||
dest_rels = rels_dir / f"{dest}.rels"
|
||||
|
||||
slide_xml = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<p:sld xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main">
|
||||
<p:cSld>
|
||||
<p:spTree>
|
||||
<p:nvGrpSpPr>
|
||||
<p:cNvPr id="1" name=""/>
|
||||
<p:cNvGrpSpPr/>
|
||||
<p:nvPr/>
|
||||
</p:nvGrpSpPr>
|
||||
<p:grpSpPr>
|
||||
<a:xfrm>
|
||||
<a:off x="0" y="0"/>
|
||||
<a:ext cx="0" cy="0"/>
|
||||
<a:chOff x="0" y="0"/>
|
||||
<a:chExt cx="0" cy="0"/>
|
||||
</a:xfrm>
|
||||
</p:grpSpPr>
|
||||
</p:spTree>
|
||||
</p:cSld>
|
||||
<p:clrMapOvr>
|
||||
<a:masterClrMapping/>
|
||||
</p:clrMapOvr>
|
||||
</p:sld>'''
|
||||
dest_slide.write_text(slide_xml, encoding="utf-8")
|
||||
|
||||
rels_dir.mkdir(exist_ok=True)
|
||||
rels_xml = f'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" Target="../slideLayouts/{layout_file}"/>
|
||||
</Relationships>'''
|
||||
dest_rels.write_text(rels_xml, encoding="utf-8")
|
||||
|
||||
_add_to_content_types(unpacked_dir, dest)
|
||||
|
||||
rid = _add_to_presentation_rels(unpacked_dir, dest)
|
||||
|
||||
next_slide_id = _get_next_slide_id(unpacked_dir)
|
||||
|
||||
print(f"Created {dest} from {layout_file}")
|
||||
print(f'Add to presentation.xml <p:sldIdLst>: <p:sldId id="{next_slide_id}" r:id="{rid}"/>')
|
||||
|
||||
|
||||
def duplicate_slide(unpacked_dir: Path, source: str) -> None:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
rels_dir = slides_dir / "_rels"
|
||||
|
||||
source_slide = slides_dir / source
|
||||
|
||||
if not source_slide.exists():
|
||||
print(f"Error: {source_slide} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
next_num = get_next_slide_number(slides_dir)
|
||||
dest = f"slide{next_num}.xml"
|
||||
dest_slide = slides_dir / dest
|
||||
|
||||
source_rels = rels_dir / f"{source}.rels"
|
||||
dest_rels = rels_dir / f"{dest}.rels"
|
||||
|
||||
shutil.copy2(source_slide, dest_slide)
|
||||
|
||||
if source_rels.exists():
|
||||
shutil.copy2(source_rels, dest_rels)
|
||||
|
||||
rels_content = dest_rels.read_text(encoding="utf-8")
|
||||
rels_content = re.sub(
|
||||
r'\s*<Relationship[^>]*Type="[^"]*notesSlide"[^>]*/>\s*',
|
||||
"\n",
|
||||
rels_content,
|
||||
)
|
||||
dest_rels.write_text(rels_content, encoding="utf-8")
|
||||
|
||||
_add_to_content_types(unpacked_dir, dest)
|
||||
|
||||
rid = _add_to_presentation_rels(unpacked_dir, dest)
|
||||
|
||||
next_slide_id = _get_next_slide_id(unpacked_dir)
|
||||
|
||||
print(f"Created {dest} from {source}")
|
||||
print(f'Add to presentation.xml <p:sldIdLst>: <p:sldId id="{next_slide_id}" r:id="{rid}"/>')
|
||||
|
||||
|
||||
def _add_to_content_types(unpacked_dir: Path, dest: str) -> None:
|
||||
content_types_path = unpacked_dir / "[Content_Types].xml"
|
||||
content_types = content_types_path.read_text(encoding="utf-8")
|
||||
|
||||
new_override = f'<Override PartName="/ppt/slides/{dest}" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'
|
||||
|
||||
if f"/ppt/slides/{dest}" not in content_types:
|
||||
content_types = content_types.replace("</Types>", f" {new_override}\n</Types>")
|
||||
content_types_path.write_text(content_types, encoding="utf-8")
|
||||
|
||||
|
||||
def _add_to_presentation_rels(unpacked_dir: Path, dest: str) -> str:
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
pres_rels = pres_rels_path.read_text(encoding="utf-8")
|
||||
|
||||
rids = [int(m) for m in re.findall(r'Id="rId(\d+)"', pres_rels)]
|
||||
next_rid = max(rids) + 1 if rids else 1
|
||||
rid = f"rId{next_rid}"
|
||||
|
||||
new_rel = f'<Relationship Id="{rid}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" Target="slides/{dest}"/>'
|
||||
|
||||
if f"slides/{dest}" not in pres_rels:
|
||||
pres_rels = pres_rels.replace("</Relationships>", f" {new_rel}\n</Relationships>")
|
||||
pres_rels_path.write_text(pres_rels, encoding="utf-8")
|
||||
|
||||
return rid
|
||||
|
||||
|
||||
def _get_next_slide_id(unpacked_dir: Path) -> int:
|
||||
pres_path = unpacked_dir / "ppt" / "presentation.xml"
|
||||
pres_content = pres_path.read_text(encoding="utf-8")
|
||||
slide_ids = [int(m) for m in re.findall(r'<p:sldId[^>]*id="(\d+)"', pres_content)]
|
||||
return max(slide_ids) + 1 if slide_ids else 256
|
||||
|
||||
|
||||
def parse_source(source: str) -> tuple[str, str | None]:
|
||||
if source.startswith("slideLayout") and source.endswith(".xml"):
|
||||
return ("layout", source)
|
||||
|
||||
return ("slide", None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python add_slide.py <unpacked_dir> <source>", file=sys.stderr)
|
||||
print("", file=sys.stderr)
|
||||
print("Source can be:", file=sys.stderr)
|
||||
print(" slide2.xml - duplicate an existing slide", file=sys.stderr)
|
||||
print(" slideLayout2.xml - create from a layout template", file=sys.stderr)
|
||||
print("", file=sys.stderr)
|
||||
print("To see available layouts: ls <unpacked_dir>/ppt/slideLayouts/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
unpacked_dir = Path(sys.argv[1])
|
||||
source = sys.argv[2]
|
||||
|
||||
if not unpacked_dir.exists():
|
||||
print(f"Error: {unpacked_dir} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
source_type, layout_file = parse_source(source)
|
||||
|
||||
if source_type == "layout" and layout_file is not None:
|
||||
create_slide_from_layout(unpacked_dir, layout_file)
|
||||
else:
|
||||
duplicate_slide(unpacked_dir, source)
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Remove unreferenced files from an unpacked PPTX directory.
|
||||
|
||||
Usage: python clean.py <unpacked_dir>
|
||||
|
||||
Example:
|
||||
python clean.py unpacked/
|
||||
|
||||
This script removes:
|
||||
- Orphaned slides (not in sldIdLst) and their relationships
|
||||
- [trash] directory (unreferenced files)
|
||||
- Orphaned .rels files for deleted resources
|
||||
- Unreferenced media, embeddings, charts, diagrams, drawings, ink files
|
||||
- Unreferenced theme files
|
||||
- Unreferenced notes slides
|
||||
- Content-Type overrides for deleted files
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def get_slides_in_sldidlst(unpacked_dir: Path) -> set[str]:
|
||||
pres_path = unpacked_dir / "ppt" / "presentation.xml"
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
|
||||
if not pres_path.exists() or not pres_rels_path.exists():
|
||||
return set()
|
||||
|
||||
rels_dom = defusedxml.minidom.parse(str(pres_rels_path))
|
||||
rid_to_slide = {}
|
||||
for rel in rels_dom.getElementsByTagName("Relationship"):
|
||||
rid = rel.getAttribute("Id")
|
||||
target = rel.getAttribute("Target")
|
||||
rel_type = rel.getAttribute("Type")
|
||||
if "slide" in rel_type and target.startswith("slides/"):
|
||||
rid_to_slide[rid] = target.replace("slides/", "")
|
||||
|
||||
pres_content = pres_path.read_text(encoding="utf-8")
|
||||
referenced_rids = set(re.findall(r'<p:sldId[^>]*r:id="([^"]+)"', pres_content))
|
||||
|
||||
return {rid_to_slide[rid] for rid in referenced_rids if rid in rid_to_slide}
|
||||
|
||||
|
||||
def remove_orphaned_slides(unpacked_dir: Path) -> list[str]:
|
||||
slides_dir = unpacked_dir / "ppt" / "slides"
|
||||
slides_rels_dir = slides_dir / "_rels"
|
||||
pres_rels_path = unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"
|
||||
|
||||
if not slides_dir.exists():
|
||||
return []
|
||||
|
||||
referenced_slides = get_slides_in_sldidlst(unpacked_dir)
|
||||
removed = []
|
||||
|
||||
for slide_file in slides_dir.glob("slide*.xml"):
|
||||
if slide_file.name not in referenced_slides:
|
||||
rel_path = slide_file.relative_to(unpacked_dir)
|
||||
slide_file.unlink()
|
||||
removed.append(str(rel_path))
|
||||
|
||||
rels_file = slides_rels_dir / f"{slide_file.name}.rels"
|
||||
if rels_file.exists():
|
||||
rels_file.unlink()
|
||||
removed.append(str(rels_file.relative_to(unpacked_dir)))
|
||||
|
||||
if removed and pres_rels_path.exists():
|
||||
rels_dom = defusedxml.minidom.parse(str(pres_rels_path))
|
||||
changed = False
|
||||
|
||||
for rel in list(rels_dom.getElementsByTagName("Relationship")):
|
||||
target = rel.getAttribute("Target")
|
||||
if target.startswith("slides/"):
|
||||
slide_name = target.replace("slides/", "")
|
||||
if slide_name not in referenced_slides:
|
||||
if rel.parentNode:
|
||||
rel.parentNode.removeChild(rel)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(pres_rels_path, "wb") as f:
|
||||
f.write(rels_dom.toxml(encoding="utf-8"))
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def remove_trash_directory(unpacked_dir: Path) -> list[str]:
|
||||
trash_dir = unpacked_dir / "[trash]"
|
||||
removed = []
|
||||
|
||||
if trash_dir.exists() and trash_dir.is_dir():
|
||||
for file_path in trash_dir.iterdir():
|
||||
if file_path.is_file():
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
removed.append(str(rel_path))
|
||||
file_path.unlink()
|
||||
trash_dir.rmdir()
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def get_slide_referenced_files(unpacked_dir: Path) -> set:
|
||||
referenced = set()
|
||||
slides_rels_dir = unpacked_dir / "ppt" / "slides" / "_rels"
|
||||
|
||||
if not slides_rels_dir.exists():
|
||||
return referenced
|
||||
|
||||
for rels_file in slides_rels_dir.glob("*.rels"):
|
||||
dom = defusedxml.minidom.parse(str(rels_file))
|
||||
for rel in dom.getElementsByTagName("Relationship"):
|
||||
target = rel.getAttribute("Target")
|
||||
if not target:
|
||||
continue
|
||||
target_path = (rels_file.parent.parent / target).resolve()
|
||||
try:
|
||||
referenced.add(target_path.relative_to(unpacked_dir.resolve()))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def remove_orphaned_rels_files(unpacked_dir: Path) -> list[str]:
|
||||
resource_dirs = ["charts", "diagrams", "drawings"]
|
||||
removed = []
|
||||
slide_referenced = get_slide_referenced_files(unpacked_dir)
|
||||
|
||||
for dir_name in resource_dirs:
|
||||
rels_dir = unpacked_dir / "ppt" / dir_name / "_rels"
|
||||
if not rels_dir.exists():
|
||||
continue
|
||||
|
||||
for rels_file in rels_dir.glob("*.rels"):
|
||||
resource_file = rels_dir.parent / rels_file.name.replace(".rels", "")
|
||||
try:
|
||||
resource_rel_path = resource_file.resolve().relative_to(unpacked_dir.resolve())
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if not resource_file.exists() or resource_rel_path not in slide_referenced:
|
||||
rels_file.unlink()
|
||||
rel_path = rels_file.relative_to(unpacked_dir)
|
||||
removed.append(str(rel_path))
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def get_referenced_files(unpacked_dir: Path) -> set:
|
||||
referenced = set()
|
||||
|
||||
for rels_file in unpacked_dir.rglob("*.rels"):
|
||||
dom = defusedxml.minidom.parse(str(rels_file))
|
||||
for rel in dom.getElementsByTagName("Relationship"):
|
||||
target = rel.getAttribute("Target")
|
||||
if not target:
|
||||
continue
|
||||
target_path = (rels_file.parent.parent / target).resolve()
|
||||
try:
|
||||
referenced.add(target_path.relative_to(unpacked_dir.resolve()))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def remove_orphaned_files(unpacked_dir: Path, referenced: set) -> list[str]:
|
||||
resource_dirs = ["media", "embeddings", "charts", "diagrams", "tags", "drawings", "ink"]
|
||||
removed = []
|
||||
|
||||
for dir_name in resource_dirs:
|
||||
dir_path = unpacked_dir / "ppt" / dir_name
|
||||
if not dir_path.exists():
|
||||
continue
|
||||
|
||||
for file_path in dir_path.glob("*"):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
if rel_path not in referenced:
|
||||
file_path.unlink()
|
||||
removed.append(str(rel_path))
|
||||
|
||||
theme_dir = unpacked_dir / "ppt" / "theme"
|
||||
if theme_dir.exists():
|
||||
for file_path in theme_dir.glob("theme*.xml"):
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
if rel_path not in referenced:
|
||||
file_path.unlink()
|
||||
removed.append(str(rel_path))
|
||||
theme_rels = theme_dir / "_rels" / f"{file_path.name}.rels"
|
||||
if theme_rels.exists():
|
||||
theme_rels.unlink()
|
||||
removed.append(str(theme_rels.relative_to(unpacked_dir)))
|
||||
|
||||
notes_dir = unpacked_dir / "ppt" / "notesSlides"
|
||||
if notes_dir.exists():
|
||||
for file_path in notes_dir.glob("*.xml"):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
rel_path = file_path.relative_to(unpacked_dir)
|
||||
if rel_path not in referenced:
|
||||
file_path.unlink()
|
||||
removed.append(str(rel_path))
|
||||
|
||||
notes_rels_dir = notes_dir / "_rels"
|
||||
if notes_rels_dir.exists():
|
||||
for file_path in notes_rels_dir.glob("*.rels"):
|
||||
notes_file = notes_dir / file_path.name.replace(".rels", "")
|
||||
if not notes_file.exists():
|
||||
file_path.unlink()
|
||||
removed.append(str(file_path.relative_to(unpacked_dir)))
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def update_content_types(unpacked_dir: Path, removed_files: list[str]) -> None:
|
||||
ct_path = unpacked_dir / "[Content_Types].xml"
|
||||
if not ct_path.exists():
|
||||
return
|
||||
|
||||
dom = defusedxml.minidom.parse(str(ct_path))
|
||||
changed = False
|
||||
|
||||
for override in list(dom.getElementsByTagName("Override")):
|
||||
part_name = override.getAttribute("PartName").lstrip("/")
|
||||
if part_name in removed_files:
|
||||
if override.parentNode:
|
||||
override.parentNode.removeChild(override)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
with open(ct_path, "wb") as f:
|
||||
f.write(dom.toxml(encoding="utf-8"))
|
||||
|
||||
|
||||
def clean_unused_files(unpacked_dir: Path) -> list[str]:
|
||||
all_removed = []
|
||||
|
||||
slides_removed = remove_orphaned_slides(unpacked_dir)
|
||||
all_removed.extend(slides_removed)
|
||||
|
||||
trash_removed = remove_trash_directory(unpacked_dir)
|
||||
all_removed.extend(trash_removed)
|
||||
|
||||
while True:
|
||||
removed_rels = remove_orphaned_rels_files(unpacked_dir)
|
||||
referenced = get_referenced_files(unpacked_dir)
|
||||
removed_files = remove_orphaned_files(unpacked_dir, referenced)
|
||||
|
||||
total_removed = removed_rels + removed_files
|
||||
if not total_removed:
|
||||
break
|
||||
|
||||
all_removed.extend(total_removed)
|
||||
|
||||
if all_removed:
|
||||
update_content_types(unpacked_dir, all_removed)
|
||||
|
||||
return all_removed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python clean.py <unpacked_dir>", file=sys.stderr)
|
||||
print("Example: python clean.py unpacked/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
unpacked_dir = Path(sys.argv[1])
|
||||
|
||||
if not unpacked_dir.exists():
|
||||
print(f"Error: {unpacked_dir} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
removed = clean_unused_files(unpacked_dir)
|
||||
|
||||
if removed:
|
||||
print(f"Removed {len(removed)} unreferenced files:")
|
||||
for f in removed:
|
||||
print(f" {f}")
|
||||
else:
|
||||
print("No unreferenced files found")
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Merge adjacent runs with identical formatting in DOCX.
|
||||
|
||||
Merges adjacent <w:r> elements that have identical <w:rPr> properties.
|
||||
Works on runs in paragraphs and inside tracked changes (<w:ins>, <w:del>).
|
||||
|
||||
Also:
|
||||
- Removes rsid attributes from runs (revision metadata that doesn't affect rendering)
|
||||
- Removes proofErr elements (spell/grammar markers that block merging)
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import defusedxml.minidom
|
||||
|
||||
|
||||
def merge_runs(input_dir: str) -> tuple[int, str]:
|
||||
doc_xml = Path(input_dir) / "word" / "document.xml"
|
||||
|
||||
if not doc_xml.exists():
|
||||
return 0, f"Error: {doc_xml} not found"
|
||||
|
||||
try:
|
||||
dom = defusedxml.minidom.parseString(doc_xml.read_text(encoding="utf-8"))
|
||||
root = dom.documentElement
|
||||
|
||||
_remove_elements(root, "proofErr")
|
||||
_strip_run_rsid_attrs(root)
|
||||
|
||||
containers = {run.parentNode for run in _find_elements(root, "r")}
|
||||
|
||||
merge_count = 0
|
||||
for container in containers:
|
||||
merge_count += _merge_runs_in(container)
|
||||
|
||||
doc_xml.write_bytes(dom.toxml(encoding="UTF-8"))
|
||||
return merge_count, f"Merged {merge_count} runs"
|
||||
|
||||
except Exception as e:
|
||||
return 0, f"Error: {e}"
|
||||
|
||||
|
||||
|
||||
|
||||
def _find_elements(root, tag: str) -> list:
|
||||
results = []
|
||||
|
||||
def traverse(node):
|
||||
if node.nodeType == node.ELEMENT_NODE:
|
||||
name = node.localName or node.tagName
|
||||
if name == tag or name.endswith(f":{tag}"):
|
||||
results.append(node)
|
||||
for child in node.childNodes:
|
||||
traverse(child)
|
||||
|
||||
traverse(root)
|
||||
return results
|
||||
|
||||
|
||||
def _get_child(parent, tag: str):
|
||||
for child in parent.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name == tag or name.endswith(f":{tag}"):
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def _get_children(parent, tag: str) -> list:
|
||||
results = []
|
||||
for child in parent.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name == tag or name.endswith(f":{tag}"):
|
||||
results.append(child)
|
||||
return results
|
||||
|
||||
|
||||
def _is_adjacent(elem1, elem2) -> bool:
|
||||
node = elem1.nextSibling
|
||||
while node:
|
||||
if node == elem2:
|
||||
return True
|
||||
if node.nodeType == node.ELEMENT_NODE:
|
||||
return False
|
||||
if node.nodeType == node.TEXT_NODE and node.data.strip():
|
||||
return False
|
||||
node = node.nextSibling
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def _remove_elements(root, tag: str):
|
||||
for elem in _find_elements(root, tag):
|
||||
if elem.parentNode:
|
||||
elem.parentNode.removeChild(elem)
|
||||
|
||||
|
||||
def _strip_run_rsid_attrs(root):
|
||||
for run in _find_elements(root, "r"):
|
||||
for attr in list(run.attributes.values()):
|
||||
if "rsid" in attr.name.lower():
|
||||
run.removeAttribute(attr.name)
|
||||
|
||||
|
||||
|
||||
|
||||
def _merge_runs_in(container) -> int:
|
||||
merge_count = 0
|
||||
run = _first_child_run(container)
|
||||
|
||||
while run:
|
||||
while True:
|
||||
next_elem = _next_element_sibling(run)
|
||||
if next_elem and _is_run(next_elem) and _can_merge(run, next_elem):
|
||||
_merge_run_content(run, next_elem)
|
||||
container.removeChild(next_elem)
|
||||
merge_count += 1
|
||||
else:
|
||||
break
|
||||
|
||||
_consolidate_text(run)
|
||||
run = _next_sibling_run(run)
|
||||
|
||||
return merge_count
|
||||
|
||||
|
||||
def _first_child_run(container):
|
||||
for child in container.childNodes:
|
||||
if child.nodeType == child.ELEMENT_NODE and _is_run(child):
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def _next_element_sibling(node):
|
||||
sibling = node.nextSibling
|
||||
while sibling:
|
||||
if sibling.nodeType == sibling.ELEMENT_NODE:
|
||||
return sibling
|
||||
sibling = sibling.nextSibling
|
||||
return None
|
||||
|
||||
|
||||
def _next_sibling_run(node):
|
||||
sibling = node.nextSibling
|
||||
while sibling:
|
||||
if sibling.nodeType == sibling.ELEMENT_NODE:
|
||||
if _is_run(sibling):
|
||||
return sibling
|
||||
sibling = sibling.nextSibling
|
||||
return None
|
||||
|
||||
|
||||
def _is_run(node) -> bool:
|
||||
name = node.localName or node.tagName
|
||||
return name == "r" or name.endswith(":r")
|
||||
|
||||
|
||||
def _can_merge(run1, run2) -> bool:
|
||||
rpr1 = _get_child(run1, "rPr")
|
||||
rpr2 = _get_child(run2, "rPr")
|
||||
|
||||
if (rpr1 is None) != (rpr2 is None):
|
||||
return False
|
||||
if rpr1 is None:
|
||||
return True
|
||||
return rpr1.toxml() == rpr2.toxml()
|
||||
|
||||
|
||||
def _merge_run_content(target, source):
|
||||
for child in list(source.childNodes):
|
||||
if child.nodeType == child.ELEMENT_NODE:
|
||||
name = child.localName or child.tagName
|
||||
if name != "rPr" and not name.endswith(":rPr"):
|
||||
target.appendChild(child)
|
||||
|
||||
|
||||
def _consolidate_text(run):
|
||||
t_elements = _get_children(run, "t")
|
||||
|
||||
for i in range(len(t_elements) - 1, 0, -1):
|
||||
curr, prev = t_elements[i], t_elements[i - 1]
|
||||
|
||||
if _is_adjacent(prev, curr):
|
||||
prev_text = prev.firstChild.data if prev.firstChild else ""
|
||||
curr_text = curr.firstChild.data if curr.firstChild else ""
|
||||
merged = prev_text + curr_text
|
||||
|
||||
if prev.firstChild:
|
||||
prev.firstChild.data = merged
|
||||
else:
|
||||
prev.appendChild(run.ownerDocument.createTextNode(merged))
|
||||
|
||||
if merged.startswith(" ") or merged.endswith(" "):
|
||||
prev.setAttribute("xml:space", "preserve")
|
||||
elif prev.hasAttribute("xml:space"):
|
||||
prev.removeAttribute("xml:space")
|
||||
|
||||
run.removeChild(curr)
|
||||
@@ -0,0 +1,167 @@
|
||||
---
|
||||
name: jupyter-live-kernel
|
||||
description: "Iterative Python via live Jupyter kernel (hamelnb)."
|
||||
version: 1.0.0
|
||||
author: Hermes Agent
|
||||
license: MIT
|
||||
platforms: [linux, macos, windows]
|
||||
metadata:
|
||||
hermes:
|
||||
tags: [jupyter, notebook, repl, data-science, exploration, iterative]
|
||||
category: data-science
|
||||
---
|
||||
|
||||
# Jupyter Live Kernel (hamelnb)
|
||||
|
||||
Gives you a **stateful Python REPL** via a live Jupyter kernel. Variables persist
|
||||
across executions. Use this instead of `execute_code` when you need to build up
|
||||
state incrementally, explore APIs, inspect DataFrames, or iterate on complex code.
|
||||
|
||||
## When to Use This vs Other Tools
|
||||
|
||||
| Tool | Use When |
|
||||
|------|----------|
|
||||
| **This skill** | Iterative exploration, state across steps, data science, ML, "let me try this and check" |
|
||||
| `execute_code` | One-shot scripts needing hermes tool access (web_search, file ops). Stateless. |
|
||||
| `terminal` | Shell commands, builds, installs, git, process management |
|
||||
|
||||
**Rule of thumb:** If you'd want a Jupyter notebook for the task, use this skill.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **uv** must be installed (check: `which uv`)
|
||||
2. **JupyterLab** must be installed: `uv tool install jupyterlab`
|
||||
3. A Jupyter server must be running (see Setup below)
|
||||
|
||||
## Setup
|
||||
|
||||
The hamelnb script location:
|
||||
```
|
||||
SCRIPT="$HOME/.agent-skills/hamelnb/skills/jupyter-live-kernel/scripts/jupyter_live_kernel.py"
|
||||
```
|
||||
|
||||
If not cloned yet:
|
||||
```
|
||||
git clone https://github.com/hamelsmu/hamelnb.git ~/.agent-skills/hamelnb
|
||||
```
|
||||
|
||||
### Starting JupyterLab
|
||||
|
||||
Check if a server is already running:
|
||||
```
|
||||
uv run "$SCRIPT" servers
|
||||
```
|
||||
|
||||
If no servers found, start one:
|
||||
```
|
||||
jupyter-lab --no-browser --port=8888 --notebook-dir=$HOME/notebooks \
|
||||
--IdentityProvider.token='' --ServerApp.password='' > /tmp/jupyter.log 2>&1 &
|
||||
sleep 3
|
||||
```
|
||||
|
||||
Note: Token/password disabled for local agent access. The server runs headless.
|
||||
|
||||
### Creating a Notebook for REPL Use
|
||||
|
||||
If you just need a REPL (no existing notebook), create a minimal notebook file:
|
||||
```
|
||||
mkdir -p ~/notebooks
|
||||
```
|
||||
Write a minimal .ipynb JSON file with one empty code cell, then start a kernel
|
||||
session via the Jupyter REST API:
|
||||
```
|
||||
curl -s -X POST http://127.0.0.1:8888/api/sessions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"path":"scratch.ipynb","type":"notebook","name":"scratch.ipynb","kernel":{"name":"python3"}}'
|
||||
```
|
||||
|
||||
## Core Workflow
|
||||
|
||||
All commands return structured JSON. Always use `--compact` to save tokens.
|
||||
|
||||
### 1. Discover servers and notebooks
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" servers --compact
|
||||
uv run "$SCRIPT" notebooks --compact
|
||||
```
|
||||
|
||||
### 2. Execute code (primary operation)
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" execute --path <notebook.ipynb> --code '<python code>' --compact
|
||||
```
|
||||
|
||||
State persists across execute calls. Variables, imports, objects all survive.
|
||||
|
||||
Multi-line code works with $'...' quoting:
|
||||
```
|
||||
uv run "$SCRIPT" execute --path scratch.ipynb --code $'import os\nfiles = os.listdir(".")\nprint(f"Found {len(files)} files")' --compact
|
||||
```
|
||||
|
||||
### 3. Inspect live variables
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" variables --path <notebook.ipynb> list --compact
|
||||
uv run "$SCRIPT" variables --path <notebook.ipynb> preview --name <varname> --compact
|
||||
```
|
||||
|
||||
### 4. Edit notebook cells
|
||||
|
||||
```
|
||||
# View current cells
|
||||
uv run "$SCRIPT" contents --path <notebook.ipynb> --compact
|
||||
|
||||
# Insert a new cell
|
||||
uv run "$SCRIPT" edit --path <notebook.ipynb> insert \
|
||||
--at-index <N> --cell-type code --source '<code>' --compact
|
||||
|
||||
# Replace cell source (use cell-id from contents output)
|
||||
uv run "$SCRIPT" edit --path <notebook.ipynb> replace-source \
|
||||
--cell-id <id> --source '<new code>' --compact
|
||||
|
||||
# Delete a cell
|
||||
uv run "$SCRIPT" edit --path <notebook.ipynb> delete --cell-id <id> --compact
|
||||
```
|
||||
|
||||
### 5. Verification (restart + run all)
|
||||
|
||||
Only use when the user asks for a clean verification or you need to confirm
|
||||
the notebook runs top-to-bottom:
|
||||
|
||||
```
|
||||
uv run "$SCRIPT" restart-run-all --path <notebook.ipynb> --save-outputs --compact
|
||||
```
|
||||
|
||||
## Practical Tips from Experience
|
||||
|
||||
1. **First execution after server start may timeout** — the kernel needs a moment
|
||||
to initialize. If you get a timeout, just retry.
|
||||
|
||||
2. **The kernel Python is JupyterLab's Python** — packages must be installed in
|
||||
that environment. If you need additional packages, install them into the
|
||||
JupyterLab tool environment first.
|
||||
|
||||
3. **--compact flag saves significant tokens** — always use it. JSON output can
|
||||
be very verbose without it.
|
||||
|
||||
4. **For pure REPL use**, create a scratch.ipynb and don't bother with cell editing.
|
||||
Just use `execute` repeatedly.
|
||||
|
||||
5. **Argument order matters** — subcommand flags like `--path` go BEFORE the
|
||||
sub-subcommand. E.g.: `variables --path nb.ipynb list` not `variables list --path nb.ipynb`.
|
||||
|
||||
6. **If a session doesn't exist yet**, you need to start one via the REST API
|
||||
(see Setup section). The tool can't execute without a live kernel session.
|
||||
|
||||
7. **Errors are returned as JSON** with traceback — read the `ename` and `evalue`
|
||||
fields to understand what went wrong.
|
||||
|
||||
8. **Occasional websocket timeouts** — some operations may timeout on first try,
|
||||
especially after a kernel restart. Retry once before escalating.
|
||||
|
||||
## Timeout Defaults
|
||||
|
||||
The script has a 30-second default timeout per execution. For long-running
|
||||
operations, pass `--timeout 120`. Use generous timeouts (60+) for initial
|
||||
setup or heavy computation.
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
description: Diagram creation skills for generating visual diagrams, flowcharts, architecture diagrams, and illustrations using tools like Excalidraw.
|
||||
---
|
||||
Reference in New Issue
Block a user