899 lines
29 KiB
Rust
899 lines
29 KiB
Rust
//! Browser end-to-end tests against the local ChatGPT fixture page.
|
|
//!
|
|
//! They need a real Chromium-based browser and are therefore ignored by default:
|
|
//!
|
|
//! cargo test --test e2e_browser -- --ignored --test-threads=1
|
|
//!
|
|
//! The fixture is a static HTML file that imitates the real page (editor, send
|
|
//! button, streaming indicator, login form, captcha, error banner), so the whole
|
|
//! capture pipeline can be exercised without a network or a login.
|
|
|
|
use std::path::PathBuf;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use llm_bridge::browser::{external_launch, resolve_executable, Session};
|
|
use llm_bridge::config::{GlobalConfig, ScreenshotMode};
|
|
use llm_bridge::error::GatewayError;
|
|
use llm_bridge::paths::Layout;
|
|
use llm_bridge::providers::config::ProviderConfig;
|
|
use llm_bridge::providers::{
|
|
Attachment, AttachmentKind, LlmProvider, ProviderRegistry, TurnOutcome, TurnRequest,
|
|
WebUiProvider,
|
|
};
|
|
use llm_bridge::selftest;
|
|
use tokio::time::timeout;
|
|
|
|
/// Install a tracing subscriber once so the tests are diagnosable.
|
|
fn init_logs() {
|
|
static ONCE: std::sync::Once = std::sync::Once::new();
|
|
ONCE.call_once(|| {
|
|
let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string());
|
|
let _ = tracing_subscriber::fmt()
|
|
.with_env_filter(filter)
|
|
.with_test_writer()
|
|
.try_init();
|
|
});
|
|
}
|
|
|
|
/// Absolute file:// URL of the fixture, optionally with a query string.
|
|
fn fixture_url(query: &str) -> String {
|
|
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
|
.join("tests")
|
|
.join("fixtures")
|
|
.join("chatgpt_mock.html");
|
|
let path = path.to_string_lossy().replace('\\', "/");
|
|
let url = format!("file:///{path}");
|
|
if query.is_empty() {
|
|
url
|
|
} else {
|
|
format!("{url}?{query}")
|
|
}
|
|
}
|
|
|
|
/// Provider configuration pointing at the fixture.
|
|
fn provider_toml(base: &str) -> String {
|
|
format!(
|
|
r##"
|
|
[provider]
|
|
name = "fixture"
|
|
display_name = "ChatGPT fixture"
|
|
web_url = "{base}"
|
|
new_conversation_url = "{base}"
|
|
conversation_url_pattern = "#c=([0-9a-zA-Z-]{{8,}})"
|
|
default_models = ["fixture-1"]
|
|
enabled = true
|
|
|
|
[selectors]
|
|
input_field = "div#prompt-textarea"
|
|
send_button = "button[data-testid='send-button']"
|
|
response_container = "div[data-message-author-role='assistant']"
|
|
streaming_indicator = "button[data-testid='stop-button']"
|
|
file_input = "input[type='file']"
|
|
new_chat_button = "a[data-testid='create-new-chat-button']"
|
|
login_page_indicator = "form[action*='login']"
|
|
captcha_indicator = "div#challenge-form"
|
|
error_banner = "[data-testid='error-message'], div[role='alert']"
|
|
attachment_indicator = "div[data-testid='attachment-chip']"
|
|
copy_button = "button[data-testid='copy-button']"
|
|
|
|
[login]
|
|
url_patterns = ["login=1"]
|
|
|
|
[rate_limit]
|
|
text_patterns = ["too many requests"]
|
|
|
|
[timeouts]
|
|
page_load = 20
|
|
response_generation = 15
|
|
response_timeout = 30
|
|
|
|
[input]
|
|
poll_interval_ms = 100
|
|
quiet_ms = 500
|
|
|
|
[options]
|
|
supports_images = true
|
|
supports_streaming = true
|
|
supports_new_chat = true
|
|
"##
|
|
)
|
|
}
|
|
|
|
struct Harness {
|
|
registry: Arc<ProviderRegistry>,
|
|
provider: Arc<WebUiProvider>,
|
|
_tmp: tempfile::TempDir,
|
|
}
|
|
|
|
/// Boot a registry with a single fixture provider, or None when no browser exists.
|
|
async fn harness(toml: &str) -> Option<Harness> {
|
|
harness_with(toml, |_| {}).await
|
|
}
|
|
|
|
/// Same, with a chance to tweak the global configuration.
|
|
async fn harness_with(toml: &str, configure: impl FnOnce(&mut GlobalConfig)) -> Option<Harness> {
|
|
init_logs();
|
|
let chrome = resolve_executable(None, std::env::var("CHROME").ok())?;
|
|
let tmp = tempfile::tempdir().expect("temp dir");
|
|
let layout = Layout::rooted_at(tmp.path().join("state"));
|
|
layout.ensure().expect("layout");
|
|
// Only the fixture provider may be registered: the state directory is
|
|
// materialised with the providers shipped in the build.
|
|
if let Ok(entries) = std::fs::read_dir(layout.providers_dir()) {
|
|
for entry in entries.flatten() {
|
|
std::fs::remove_file(entry.path()).ok();
|
|
}
|
|
}
|
|
std::fs::write(layout.provider_config("fixture"), toml).expect("provider config");
|
|
|
|
let mut config = GlobalConfig::default();
|
|
config.browser.headless = true;
|
|
config.browser.executable = chrome.to_string_lossy().to_string();
|
|
config.capture.poll_interval_ms = 100;
|
|
config.capture.quiet_ms = 500;
|
|
config.capture.response_timeout_s = 30;
|
|
config.debug.screenshots = ScreenshotMode::OnError;
|
|
config.debug.dir = tmp.path().join("debug").to_string_lossy().to_string();
|
|
configure(&mut config);
|
|
|
|
let registry = Arc::new(ProviderRegistry::new(layout, config));
|
|
registry.reload().await.expect("registry reload");
|
|
let provider = registry
|
|
.get("fixture")
|
|
.expect("fixture provider registered");
|
|
Some(Harness {
|
|
registry,
|
|
provider,
|
|
_tmp: tmp,
|
|
})
|
|
}
|
|
|
|
/// Run one turn and return the deltas plus the final result.
|
|
async fn run_turn(
|
|
provider: &Arc<WebUiProvider>,
|
|
session: &Session,
|
|
prompt: &str,
|
|
reuse_url: Option<String>,
|
|
attachments: Vec<Attachment>,
|
|
) -> (Vec<String>, Result<TurnOutcome, GatewayError>) {
|
|
let request = TurnRequest {
|
|
prompt: prompt.to_string(),
|
|
attachments,
|
|
replay_history: reuse_url.is_none(),
|
|
reuse_url,
|
|
probe: false,
|
|
trace_id: "e2e".to_string(),
|
|
wait_for_tab: None,
|
|
};
|
|
|
|
let mut stream = match provider.run_turn(session.clone(), request).await {
|
|
Ok(stream) => stream,
|
|
Err(err) => return (Vec::new(), Err(err)),
|
|
};
|
|
|
|
let mut deltas = Vec::new();
|
|
loop {
|
|
match stream.recv().await {
|
|
Some(llm_bridge::providers::TurnEvent::Delta(delta)) => deltas.push(delta),
|
|
Some(llm_bridge::providers::TurnEvent::Completed(outcome)) => {
|
|
return (deltas, Ok(outcome))
|
|
}
|
|
Some(llm_bridge::providers::TurnEvent::Failed(err)) => return (deltas, Err(err)),
|
|
None => {
|
|
return (
|
|
deltas,
|
|
Err(GatewayError::internal(
|
|
"the turn ended without a terminal event",
|
|
)),
|
|
)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Assert that the error carries the expected machine readable code.
|
|
fn expect_code(result: Result<TurnOutcome, GatewayError>, code: &str) {
|
|
match result {
|
|
Ok(outcome) => panic!("expected {code}, got an answer: {}", outcome.text),
|
|
Err(err) => assert_eq!(err.code(), code, "unexpected error: {err}"),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn fixture_full_pipeline() {
|
|
let Some(harness) = harness(&provider_toml(&fixture_url("conv=1"))).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
assert_eq!(
|
|
harness.registry.all().len(),
|
|
1,
|
|
"only the fixture provider is configured"
|
|
);
|
|
let session = harness.provider.session(None).await.expect("session");
|
|
|
|
let (deltas, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"hello fixture",
|
|
None,
|
|
Vec::new(),
|
|
),
|
|
)
|
|
.await
|
|
.expect("the first turn timed out");
|
|
|
|
let outcome = result.expect("the first turn failed");
|
|
assert!(
|
|
outcome.text.starts_with("Echo: hello fixture"),
|
|
"{}",
|
|
outcome.text
|
|
);
|
|
assert!(outcome.text.contains("fixture page"), "{}", outcome.text);
|
|
assert!(
|
|
deltas.len() >= 2,
|
|
"expected several streamed deltas, got {deltas:?}"
|
|
);
|
|
assert_eq!(deltas.concat().trim(), outcome.text.trim());
|
|
let web_id = outcome
|
|
.web_id
|
|
.clone()
|
|
.expect("the conversation id must be extracted");
|
|
assert!(
|
|
web_id.len() >= 8,
|
|
"suspicious conversation id {web_id:?} from {:?}",
|
|
outcome.web_url
|
|
);
|
|
|
|
// Continuing the thread reopens the same web conversation.
|
|
let reuse = outcome.web_url.clone().expect("web url");
|
|
let (_, second) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"second question",
|
|
Some(reuse),
|
|
Vec::new(),
|
|
),
|
|
)
|
|
.await
|
|
.expect("the second turn timed out");
|
|
let second = second.expect("the second turn failed");
|
|
assert!(
|
|
second.text.starts_with("Echo: second question"),
|
|
"{}",
|
|
second.text
|
|
);
|
|
assert_eq!(
|
|
second.web_id, outcome.web_id,
|
|
"the web conversation changed"
|
|
);
|
|
|
|
// The provider passes its own validation. The validation opens its own
|
|
// tab, so the one this test holds is released first.
|
|
drop(session);
|
|
let cfg = harness.provider.cfg();
|
|
let report = selftest::validate_provider(harness.provider.as_ref(), &cfg, None)
|
|
.await
|
|
.expect("validation");
|
|
assert!(report.ok, "selftest failed: {report:?}");
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn fixture_selftest_works_on_a_brand_new_session() {
|
|
let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let session = harness.provider.session(None).await.expect("session");
|
|
|
|
// This is what the CLI does: a session that never navigated anywhere yet.
|
|
let start_page = session
|
|
.page()
|
|
.url()
|
|
.await
|
|
.unwrap_or_default()
|
|
.unwrap_or_default();
|
|
assert!(
|
|
start_page.is_empty() || start_page == "about:blank",
|
|
"a new session should start on about:blank, got {start_page:?}"
|
|
);
|
|
|
|
// The selftest acquires its own tab: release this one first, otherwise a
|
|
// single-tab provider (max_tabs = 1) answers "busy" instead.
|
|
drop(session);
|
|
let cfg = harness.provider.cfg();
|
|
let report = selftest::validate_provider(harness.provider.as_ref(), &cfg, None)
|
|
.await
|
|
.expect("validation on a fresh session");
|
|
assert!(report.ok, "selftest on a fresh session failed: {report:?}");
|
|
assert_eq!(report.status, "ok");
|
|
// The check must be quick: it opens the page instead of waiting for a
|
|
// selector to appear on about:blank.
|
|
assert!(
|
|
report.duration_ms < 20_000,
|
|
"selftest took {} ms",
|
|
report.duration_ms
|
|
);
|
|
let probe_step = report
|
|
.steps
|
|
.iter()
|
|
.find(|step| step.step == "page_probe")
|
|
.expect("page_probe step");
|
|
assert!(probe_step.ok, "page_probe failed: {probe_step:?}");
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn fixture_reports_login_captcha_error_and_rate_limit() {
|
|
let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let session = harness.provider.session(None).await.expect("session");
|
|
|
|
let cases = [
|
|
("login=1", "requires_login"),
|
|
("captcha=1", "captcha_required"),
|
|
("error=1", "upstream_error"),
|
|
("ratelimit=1", "upstream_rate_limit"),
|
|
];
|
|
|
|
for (query, code) in cases {
|
|
let url = fixture_url(query);
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(60),
|
|
run_turn(&harness.provider, &session, "hello", Some(url), Vec::new()),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|_| panic!("the {query} case timed out"));
|
|
expect_code(result, code);
|
|
}
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn fixture_timeouts_and_missing_selectors() {
|
|
// A page that never produces text trips the first-token budget.
|
|
let impatient = provider_toml(&fixture_url(""))
|
|
.replace("response_generation = 15", "response_generation = 3");
|
|
let Some(harness) = harness(&impatient).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let session = harness.provider.session(None).await.expect("session");
|
|
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(60),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"hello",
|
|
Some(fixture_url("empty=1")),
|
|
Vec::new(),
|
|
),
|
|
)
|
|
.await
|
|
.expect("the empty case timed out");
|
|
expect_code(result, "upstream_timeout");
|
|
|
|
// A page whose editor appears late is waited for, not failed.
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"late editor",
|
|
Some(fixture_url("loading=1")),
|
|
Vec::new(),
|
|
),
|
|
)
|
|
.await
|
|
.expect("the loading case timed out");
|
|
let outcome = result.expect("the late editor should still be driven");
|
|
assert!(
|
|
outcome.text.starts_with("Echo: late editor"),
|
|
"{}",
|
|
outcome.text
|
|
);
|
|
|
|
// A stale selector is reported as such.
|
|
let broken = provider_toml(&fixture_url(""))
|
|
.replace(
|
|
"input_field = \"div#prompt-textarea\"",
|
|
"input_field = \"div#this-does-not-exist\"",
|
|
)
|
|
.replace("page_load = 20", "page_load = 4");
|
|
harness
|
|
.provider
|
|
.replace_config(ProviderConfig::parse(&broken, "fixture.toml").expect("broken config"))
|
|
.await;
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(60),
|
|
run_turn(&harness.provider, &session, "hello", None, Vec::new()),
|
|
)
|
|
.await
|
|
.expect("the broken selector case timed out");
|
|
expect_code(result, "selector_missing");
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn two_tabs_of_one_browser_answer_concurrently() {
|
|
let toml = provider_toml(&fixture_url("conv=1"));
|
|
let Some(harness) = harness_with(&toml, |config| {
|
|
config.browser.max_tabs = 2;
|
|
})
|
|
.await
|
|
else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let first = harness.provider.session(None).await.expect("first tab");
|
|
let second = harness.provider.session(None).await.expect("second tab");
|
|
let url = fixture_url("conv=1");
|
|
|
|
let outcome = timeout(Duration::from_secs(30), async {
|
|
// Navigate both tabs at once...
|
|
let (a, b) = tokio::join!(
|
|
llm_bridge::browser::dom::goto(first.page(), &url, Duration::from_secs(10)),
|
|
llm_bridge::browser::dom::goto(second.page(), &url, Duration::from_secs(10)),
|
|
);
|
|
a.expect("the first tab did not navigate");
|
|
b.expect("the second tab did not navigate");
|
|
|
|
// ... then hammer both, the way the capture loop does.
|
|
for _ in 0..20 {
|
|
let (a, b) = tokio::join!(
|
|
first.page().evaluate_expression("document.title"),
|
|
second.page().evaluate_expression("document.title"),
|
|
);
|
|
a.expect("the first tab stopped answering");
|
|
b.expect("the second tab stopped answering");
|
|
}
|
|
})
|
|
.await;
|
|
|
|
assert!(
|
|
outcome.is_ok(),
|
|
"two tabs of one browser stopped answering: {outcome:?}"
|
|
);
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn two_tabs_serve_two_turns_in_parallel() {
|
|
let toml = provider_toml(&fixture_url("conv=1&delay=5000"));
|
|
let Some(harness) = harness_with(&toml, |config| {
|
|
config.browser.max_tabs = 2;
|
|
})
|
|
.await
|
|
else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
|
|
// Two tabs of the same account can be leased at the same time.
|
|
let first = harness.provider.session(None).await.expect("first tab");
|
|
let second = harness.provider.session(None).await.expect("second tab");
|
|
assert_ne!(first.tab_slot(), second.tab_slot());
|
|
let stats = harness.provider.tab_stats();
|
|
assert_eq!(stats[0].1.max_tabs, 2);
|
|
assert_eq!(stats[0].1.busy, 2);
|
|
|
|
let started = std::time::Instant::now();
|
|
let (a, b) = tokio::join!(
|
|
run_turn(&harness.provider, &first, "first", None, Vec::new()),
|
|
run_turn(&harness.provider, &second, "second", None, Vec::new()),
|
|
);
|
|
let elapsed = started.elapsed();
|
|
|
|
let a = a.1.expect("the first turn failed");
|
|
let b = b.1.expect("the second turn failed");
|
|
assert!(a.text.contains("first"), "{}", a.text);
|
|
assert!(b.text.contains("second"), "{}", b.text);
|
|
// Each generation takes five seconds: serving them one after the other
|
|
// takes more than ten, two tabs take about six.
|
|
assert!(
|
|
elapsed < Duration::from_secs(9),
|
|
"two tabs did not run in parallel: {elapsed:?}"
|
|
);
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn a_busy_single_tab_account_refuses_a_second_request() {
|
|
let toml = provider_toml(&fixture_url("conv=1"));
|
|
let Some(harness) = harness_with(&toml, |config| {
|
|
config.browser.max_tabs = 1;
|
|
config.browser.busy_wait_s = 1;
|
|
})
|
|
.await
|
|
else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
|
|
let held = harness.provider.session(None).await.expect("the only tab");
|
|
let started = std::time::Instant::now();
|
|
let err = harness
|
|
.provider
|
|
.session(None)
|
|
.await
|
|
.expect_err("the second request must not get a tab");
|
|
assert_eq!(err.code(), "provider_busy");
|
|
assert_eq!(err.http_status(), 429);
|
|
assert!(
|
|
started.elapsed() >= Duration::from_secs(1),
|
|
"the request did not wait for the configured budget"
|
|
);
|
|
|
|
drop(held);
|
|
assert!(harness.provider.session(None).await.is_ok());
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn each_account_has_its_own_browser_and_tab_pool() {
|
|
let mut toml = provider_toml(&fixture_url("conv=1&delay=3000"));
|
|
toml.push_str("\n[[accounts]]\nid = \"un\"\ndefault = true\n\n[[accounts]]\nid = \"deux\"\n");
|
|
let Some(harness) = harness(&toml).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
|
|
let accounts = harness.provider.accounts();
|
|
assert_eq!(accounts.len(), 2);
|
|
assert_ne!(accounts[0].profile, accounts[1].profile);
|
|
|
|
// max_tabs is 1 for both accounts, yet one tab per account is available at
|
|
// the same time: the pools are per account, not per provider.
|
|
let one = harness
|
|
.provider
|
|
.session(Some("un"))
|
|
.await
|
|
.expect("account un");
|
|
let two = harness
|
|
.provider
|
|
.session(Some("deux"))
|
|
.await
|
|
.expect("account deux");
|
|
assert_eq!(one.account(), Some("un"));
|
|
assert_eq!(two.account(), Some("deux"));
|
|
|
|
let started = std::time::Instant::now();
|
|
let (a, b) = tokio::join!(
|
|
run_turn(&harness.provider, &one, "un", None, Vec::new()),
|
|
run_turn(&harness.provider, &two, "deux", None, Vec::new()),
|
|
);
|
|
let elapsed = started.elapsed();
|
|
assert!(a.1.is_ok(), "account un failed: {:?}", a.1.err());
|
|
assert!(b.1.is_ok(), "account deux failed: {:?}", b.1.err());
|
|
assert!(
|
|
elapsed < Duration::from_secs(6),
|
|
"the two accounts did not answer at the same time: {elapsed:?}"
|
|
);
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn fixture_returns_markdown_and_never_the_toolbar_text() {
|
|
let toml = provider_toml(&fixture_url("conv=1&rich=1"));
|
|
let Some(harness) = harness(&toml).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let session = harness.provider.session(None).await.expect("session");
|
|
|
|
// Auto mode: the provider copy button first, the HTML conversion otherwise.
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(&harness.provider, &session, "rich answer", None, Vec::new()),
|
|
)
|
|
.await
|
|
.expect("the rich turn timed out");
|
|
let outcome = result.expect("the rich turn failed");
|
|
|
|
assert!(
|
|
outcome.fidelity == "clipboard" || outcome.fidelity == "dom",
|
|
"unexpected fidelity {:?}: {}",
|
|
outcome.fidelity,
|
|
outcome.text
|
|
);
|
|
eprintln!("rich answer captured through the {} path", outcome.fidelity);
|
|
|
|
// Structure the plain text capture used to lose.
|
|
assert!(outcome.text.contains("## Answer"), "{}", outcome.text);
|
|
assert!(outcome.text.contains("- first point"), "{}", outcome.text);
|
|
assert!(outcome.text.contains("```rust"), "{}", outcome.text);
|
|
assert!(
|
|
outcome.text.contains("| Name | Value |"),
|
|
"{}",
|
|
outcome.text
|
|
);
|
|
assert!(outcome.text.contains("| alpha | 1 |"), "{}", outcome.text);
|
|
// The toolbar sits next to the answer, so its label is not answer text.
|
|
assert!(!outcome.text.contains("Copy"), "{}", outcome.text);
|
|
|
|
// Forcing the DOM conversion must give the same structure.
|
|
let forced = provider_toml(&fixture_url("conv=1&rich=1")).replace(
|
|
"poll_interval_ms = 100",
|
|
"poll_interval_ms = 100\nmarkdown = \"dom\"",
|
|
);
|
|
harness
|
|
.provider
|
|
.replace_config(ProviderConfig::parse(&forced, "fixture.toml").expect("forced config"))
|
|
.await;
|
|
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(&harness.provider, &session, "dom answer", None, Vec::new()),
|
|
)
|
|
.await
|
|
.expect("the forced DOM turn timed out");
|
|
let forced_outcome = result.expect("the forced DOM turn failed");
|
|
assert_eq!(forced_outcome.fidelity, "dom");
|
|
assert!(
|
|
forced_outcome.text.contains("## Answer"),
|
|
"{}",
|
|
forced_outcome.text
|
|
);
|
|
assert!(
|
|
forced_outcome.text.contains("```rust"),
|
|
"{}",
|
|
forced_outcome.text
|
|
);
|
|
assert!(
|
|
forced_outcome.text.contains("| alpha | 1 |"),
|
|
"{}",
|
|
forced_outcome.text
|
|
);
|
|
assert!(
|
|
!forced_outcome.text.contains("Copy"),
|
|
"{}",
|
|
forced_outcome.text
|
|
);
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn fixture_handles_rewrites_and_attachments() {
|
|
let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let session = harness.provider.session(None).await.expect("session");
|
|
|
|
// A mid-stream rewrite must not corrupt the final answer.
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"rewrite please",
|
|
Some(fixture_url("rewrite=1")),
|
|
Vec::new(),
|
|
),
|
|
)
|
|
.await
|
|
.expect("the rewrite case timed out");
|
|
let outcome = result.expect("the rewrite case failed");
|
|
assert!(
|
|
outcome.text.starts_with("Echo: rewrite please"),
|
|
"{}",
|
|
outcome.text
|
|
);
|
|
assert!(!outcome.text.contains("Draft answer"), "{}", outcome.text);
|
|
|
|
// Attachments are pushed into the file input and acknowledged by the page.
|
|
let dir = std::env::temp_dir().join("llm-gateway-e2e");
|
|
std::fs::create_dir_all(&dir).expect("temp dir");
|
|
let image = dir.join("pixel.png");
|
|
std::fs::write(&image, b"\x89PNG\r\n\x1a\n").expect("write png");
|
|
|
|
let (_, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"with an image",
|
|
None,
|
|
vec![Attachment {
|
|
path: image,
|
|
mime: "image/png".to_string(),
|
|
kind: AttachmentKind::Image,
|
|
}],
|
|
),
|
|
)
|
|
.await
|
|
.expect("the attachment case timed out");
|
|
let outcome = result.expect("the attachment case failed");
|
|
assert!(outcome.text.contains("pixel.png"), "{}", outcome.text);
|
|
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
}
|
|
|
|
/// Wait until something accepts TCP connections on a port.
|
|
async fn wait_for_port(port: u16, budget: Duration) -> bool {
|
|
let deadline = std::time::Instant::now() + budget;
|
|
while std::time::Instant::now() < deadline {
|
|
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() {
|
|
return true;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
}
|
|
false
|
|
}
|
|
|
|
/// Wait until the browser is really up: its debugging endpoint answers and the
|
|
/// page we asked it to open is loaded.
|
|
async fn wait_for_ready_browser(port: u16, expected_url: &str, budget: Duration) -> bool {
|
|
let client = reqwest::Client::new();
|
|
let endpoint = format!("http://127.0.0.1:{port}/json/list");
|
|
let deadline = std::time::Instant::now() + budget;
|
|
while std::time::Instant::now() < deadline {
|
|
if let Ok(response) = client.get(&endpoint).send().await {
|
|
if let Ok(targets) = response.json::<Vec<serde_json::Value>>().await {
|
|
let ready = targets.iter().any(|target| {
|
|
target["type"] == "page"
|
|
&& target["url"]
|
|
.as_str()
|
|
.is_some_and(|url| url.contains(expected_url))
|
|
});
|
|
if ready {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
}
|
|
false
|
|
}
|
|
|
|
/// Kills and reaps a browser process the test started, on every path.
|
|
struct BrowserProcess {
|
|
child: std::process::Child,
|
|
}
|
|
|
|
impl BrowserProcess {
|
|
fn spawn(launch: &llm_bridge::browser::ExternalLaunch) -> Self {
|
|
Self {
|
|
child: launch.spawn().expect("spawning the browser"),
|
|
}
|
|
}
|
|
|
|
fn is_running(&mut self) -> bool {
|
|
self.child.try_wait().ok().flatten().is_none()
|
|
}
|
|
}
|
|
|
|
impl Drop for BrowserProcess {
|
|
fn drop(&mut self) {
|
|
let _ = self.child.kill();
|
|
let _ = self.child.wait();
|
|
}
|
|
}
|
|
|
|
/// Ask the OS for a free port.
|
|
fn free_port() -> u16 {
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind an ephemeral port");
|
|
listener.local_addr().expect("local address").port()
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore = "requires a real Chromium-based browser"]
|
|
async fn attach_mode_uses_a_browser_the_user_started() {
|
|
init_logs();
|
|
let Some(chrome) = resolve_executable(None, std::env::var("CHROME").ok()) else {
|
|
eprintln!("no browser found, skipping");
|
|
return;
|
|
};
|
|
let tmp = tempfile::tempdir().expect("temp dir");
|
|
let profile = tmp.path().join("attached-profile");
|
|
let port = free_port();
|
|
|
|
// The user starts the browser: no automation switch, no user agent spoofing.
|
|
let launch = external_launch(Some(&chrome), &profile, &fixture_url("conv=1"), Some(port));
|
|
assert!(!launch.command_line().contains("enable-automation"));
|
|
let mut browser = BrowserProcess::spawn(&launch);
|
|
if !wait_for_port(port, Duration::from_secs(30)).await {
|
|
panic!("the browser never opened its debugging port");
|
|
}
|
|
// Attaching to a browser that is still starting up races with Chrome
|
|
// creating its first tab: wait until the page is really there.
|
|
if !wait_for_ready_browser(port, "chatgpt_mock.html", Duration::from_secs(30)).await {
|
|
panic!("the browser never finished opening the fixture page");
|
|
}
|
|
|
|
let Some(harness) = harness_with(&provider_toml(&fixture_url("conv=1")), |config| {
|
|
config.browser.attach = true;
|
|
config.browser.debug_port = port;
|
|
})
|
|
.await
|
|
else {
|
|
return;
|
|
};
|
|
|
|
let session = harness
|
|
.provider
|
|
.session(None)
|
|
.await
|
|
.expect("attach to the running browser");
|
|
|
|
// The gateway reuses a tab the browser exposes to us, and otherwise opens
|
|
// one of its own in that same browser: either way the session, the cookies
|
|
// and the window belong to the user and are never closed by the gateway.
|
|
let url = session
|
|
.page()
|
|
.url()
|
|
.await
|
|
.unwrap_or_default()
|
|
.unwrap_or_default();
|
|
assert!(
|
|
!url.starts_with("devtools://"),
|
|
"unexpected page attached: {url:?}"
|
|
);
|
|
|
|
let (deltas, result) = timeout(
|
|
Duration::from_secs(90),
|
|
run_turn(
|
|
&harness.provider,
|
|
&session,
|
|
"attached hello",
|
|
None,
|
|
Vec::new(),
|
|
),
|
|
)
|
|
.await
|
|
.expect("the attached turn timed out");
|
|
let outcome = result.expect("the attached turn failed");
|
|
assert!(
|
|
outcome.text.starts_with("Echo: attached hello"),
|
|
"{}",
|
|
outcome.text
|
|
);
|
|
assert!(
|
|
deltas.len() >= 2,
|
|
"expected streamed deltas, got {deltas:?}"
|
|
);
|
|
|
|
// Shutting the gateway down must leave the user's browser alone.
|
|
harness.provider.shutdown().await.expect("shutdown");
|
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
assert!(
|
|
browser.is_running(),
|
|
"attach mode must not close the browser it connected to"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_fixture_configuration_is_valid() {
|
|
let cfg = ProviderConfig::parse(&provider_toml(&fixture_url("")), "fixture.toml").unwrap();
|
|
assert!(cfg.is_usable(), "{:?}", cfg.problems);
|
|
assert!(cfg.capabilities().supports_images);
|
|
assert_eq!(cfg.file.timeouts.response_generation, 15);
|
|
}
|