feat: phase 2 — am lab (benchmark N agents) + plugins d'événements (contrat JSON)

- #62: am lab --agents a,b --task <f> — séquentiel ou --parallel, capture
  durée/exit/sortie/coût estimé, rapport comparatif tableau + --json stable,
  tâches versionnables dans <state>/lab/ (--list), rejouables à l'identique
- #75: plugins/ sous state_dir, déclenchés sur on_install/on_start/on_stop/
  on_update (App::emit), contrat JSON stdin/stdout, timeout configurable,
  échec non bloquant, garde anti-récursion AM_PLUGINS_RUNNING, am plugins
  + am plugins --test <name> (CI), exemples notify + ci-webhook
- intégration REPL (parse, complétion, bannière, is_am_command), help, tips,
  man pages (am-lab.1, am-plugins.1), README, ROADMAP (26/27), v0.5.4
- 334 tests verts (309 + 25)

closes #62
closes #75
This commit is contained in:
2026-08-18 14:17:30 -04:00
parent aa0d8e43eb
commit 4c7866a25a
29 changed files with 2123 additions and 8 deletions
+101
View File
@@ -0,0 +1,101 @@
//! lab: benchmark the same task on several agents (issue #62).
use super::*;
use crate::cli::LabArgs;
use crate::lab::{self, LabReport};
use anyhow::Result;
pub fn run(app: &App, args: &LabArgs) -> Result<i32> {
if args.list {
let tasks = lab::list_tasks(app);
if app.json() {
let names: Vec<String> = tasks.iter().map(|p| p.display().to_string()).collect();
crate::output::print_json(&names);
return Ok(0);
}
if tasks.is_empty() {
app.log.info(&format!(
"no tasks in {} — write a task.yaml (name, args, timeout) or run 'lab --agents a,b --task hello'",
lab::dir(app).display()
));
return Ok(0);
}
let mut table = crate::output::Table::new(vec!["TASK"]);
for p in tasks {
table.row(vec![p.display().to_string()]);
}
print!("{}", table.render());
return Ok(0);
}
let path = match args.task.as_deref() {
Some(p) => lab::resolve_task(app, p),
None => anyhow::bail!("--task <file> is required"),
};
if !path.exists() {
anyhow::bail!(
"task not found: {} — write it or use 'lab --list' to see the tasks of {}",
path.display(),
lab::dir(app).display()
);
}
let task = lab::load_task(&path)?;
let agents = lab::split_agents(args.agents.as_deref().unwrap_or(""));
if agents.is_empty() {
anyhow::bail!(
"--agents expects at least one agent name, got '{}'",
args.agents.as_deref().unwrap_or("")
);
}
app.log.info(&format!(
"lab '{}' — {} agent(s), {}",
task.name,
agents.len(),
if args.parallel { "parallèle" } else { "séquentiel" }
));
let report = lab::run_benchmark(app, &agents, &task, args.parallel, args.timeout)?;
if app.json() {
crate::output::print_json(&report);
} else {
print_report(app, &report);
}
// Exit code: 0 when every run succeeded, 1 otherwise (CI-friendly).
Ok(if report.runs.iter().all(|r| r.status == "ok") {
0
} else {
1
})
}
fn print_report(app: &App, report: &LabReport) {
let mut table = crate::output::Table::new(vec![
"AGENT", "STATUS", "EXIT", "DURATION", "COST", "OUTPUT",
]);
for r in &report.runs {
table.row(vec![
r.agent.clone(),
r.status.clone(),
r.exit_code
.map(|c| c.to_string())
.unwrap_or_else(|| "-".into()),
format!("{:.2}s", r.duration_ms as f64 / 1000.0),
r.cost_usd
.map(|c| format!("${c:.4}"))
.unwrap_or_else(|| "-".into()),
crate::output::truncate(r.output.lines().next().unwrap_or(""), 40),
]);
}
print!("{}", table.render());
let s = &report.summary;
let mut line = format!(
"{} run(s) — {} ok, {} failed, {} timeout",
s.runs, s.ok, s.failed, s.timeout
);
if let Some(f) = &s.fastest {
line.push_str(&format!(" — fastest: {f}"));
}
if let Some(c) = &s.cheapest {
line.push_str(&format!(" — cheapest: {c}"));
}
app.log.info(&line);
}
+4
View File
@@ -15,6 +15,7 @@ pub mod history_cmd;
pub mod info_cmd;
pub mod init_cmd;
pub mod install_cmd;
pub mod lab_cmd;
pub mod list_cmd;
pub mod log_cmd;
pub mod logs_cmd;
@@ -38,6 +39,7 @@ pub mod status_cmd;
pub mod suggest_cmd;
pub mod sync_cmd;
pub mod playbook_cmd;
pub mod plugins_cmd;
pub mod theme_cmd;
pub mod timeline_cmd;
pub mod tip_cmd;
@@ -195,6 +197,8 @@ pub fn execute_command(app: &App, cmd: &Command) -> Result<i32> {
save.as_deref(),
),
Command::Playbook { path, var } => playbook_cmd::run(app, path, var),
Command::Lab(args) => lab_cmd::run(app, args),
Command::Plugins(args) => plugins_cmd::run(app, args),
Command::Info { agent } => info_cmd::run(app, agent),
Command::Init { force, template } => {
init_cmd::run(app, *force, template.as_deref())
+102
View File
@@ -0,0 +1,102 @@
//! plugins: list the event plugins and test one against a synthetic event
//! (issue #75). 'am plugins --test <name>' is the CI hook: it exercises the
//! JSON contract end to end and exits non-zero on failure.
use super::*;
use crate::cli::PluginsArgs;
use crate::plugins::{self, PluginStatus};
use anyhow::Result;
pub fn run(app: &App, args: &PluginsArgs) -> Result<i32> {
let dir = plugins::dir(app);
let all = plugins::discover(app, &dir);
if let Some(name) = &args.test {
let Some(plugin) = all.iter().find(|p| &p.name == name) else {
anyhow::bail!(
"unknown plugin '{name}' — 'am plugins' lists the plugins of {}",
dir.display()
);
};
// Synthetic event (a start, agent 'test') to exercise the contract.
let event = crate::events::Event::now(crate::events::EventKind::Start)
.with_agent("test")
.with_args(vec!["--probe".to_string()]);
let r = plugins::run_plugin(app, plugin, "on_start", &event);
if app.json() {
crate::output::print_json(&serde_json::json!({
"name": plugin.name,
"status": r.status.as_str(),
"exit_code": r.exit_code,
"duration_ms": r.duration_ms,
"response": r.response,
"stdout": r.stdout,
"stderr": r.stderr,
"error": r.error,
}));
} else {
app.log.info(&format!(
"plugin '{}' — {} in {}ms",
plugin.name,
r.status.as_str(),
r.duration_ms
));
if let Some(code) = r.exit_code {
app.log.info(&format!("exit code: {code}"));
}
if let Some(resp) = &r.response {
if let Some(msg) = &resp.message {
app.log.info(&format!("message: {msg}"));
}
if let Some(err) = &resp.error {
app.log.warn(&format!("error: {err}"));
}
}
if !r.stdout.trim().is_empty() {
app.log.info(&format!("stdout: {}", r.stdout.trim()));
}
if !r.stderr.trim().is_empty() {
app.log.warn(&format!("stderr: {}", r.stderr.trim()));
}
if let Some(e) = &r.error {
app.log.warn(&format!("{e}"));
}
}
return Ok(if r.status == PluginStatus::Ok { 0 } else { 1 });
}
if app.json() {
let items: Vec<serde_json::Value> = all
.iter()
.map(|p| {
serde_json::json!({
"name": p.name,
"events": p.events,
"timeout_secs": p.timeout.as_secs(),
"script": p.file.display().to_string(),
})
})
.collect();
crate::output::print_json(&items);
return Ok(0);
}
if all.is_empty() {
app.log.info(&format!(
"no plugins in {} — copy one from examples/plugins/ (notify, ci-webhook)",
dir.display()
));
return Ok(0);
}
let mut table = crate::output::Table::new(vec!["PLUGIN", "EVENTS", "TIMEOUT", "SCRIPT"]);
for p in &all {
table.row(vec![
p.name.clone(),
p.events.join(", "),
format!("{}s", p.timeout.as_secs()),
p.file.display().to_string(),
]);
}
print!("{}", table.render());
Ok(0)
}
+12
View File
@@ -260,6 +260,18 @@ pub static SECTIONS: &[TipSection] = &[
options: &[("--save <f>", "fichier playbook"), ("playbook <f>", "rejoue pas à pas avec confirmation")],
example: "history 12..25 --save deploy.yaml && playbook deploy.yaml",
},
TipEntry {
usage: "lab --agents claude-code,aider --task hello.yaml",
about: "benchmark : même tâche sur plusieurs agents (durée, exit, coût)",
options: &[("--parallel", "exécution concurrente"), ("--json", "rapport comparatif stable (scripts/CI)")],
example: "lab --agents claude-code,aider --task bench/hello.yaml --json",
},
TipEntry {
usage: "plugins --test notify",
about: "scripts d'extension sur les événements (contrat JSON stdin/stdout, timeout)",
options: &[("--test <name>", "exécute un plugin sur un événement de test (CI)")],
example: "plugins && plugins --test notify",
},
TipEntry {
usage: "timeline",
about: "une vue chronologique de toute l'activité",