//! Browser end-to-end tests against the local ChatGPT fixture page. //! //! They need a real Chromium-based browser and are therefore ignored by default: //! //! cargo test --test e2e_browser -- --ignored --test-threads=1 //! //! The fixture is a static HTML file that imitates the real page (editor, send //! button, streaming indicator, login form, captcha, error banner), so the whole //! capture pipeline can be exercised without a network or a login. use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use llm_bridge::browser::{external_launch, resolve_executable, Session}; use llm_bridge::config::{GlobalConfig, ScreenshotMode}; use llm_bridge::error::GatewayError; use llm_bridge::paths::Layout; use llm_bridge::providers::config::ProviderConfig; use llm_bridge::providers::{ Attachment, AttachmentKind, LlmProvider, ProviderRegistry, TurnOutcome, TurnRequest, WebUiProvider, }; use llm_bridge::selftest; use tokio::time::timeout; /// Install a tracing subscriber once so the tests are diagnosable. fn init_logs() { static ONCE: std::sync::Once = std::sync::Once::new(); ONCE.call_once(|| { let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); let _ = tracing_subscriber::fmt() .with_env_filter(filter) .with_test_writer() .try_init(); }); } /// Absolute file:// URL of the fixture, optionally with a query string. fn fixture_url(query: &str) -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests") .join("fixtures") .join("chatgpt_mock.html"); let path = path.to_string_lossy().replace('\\', "/"); let url = format!("file:///{path}"); if query.is_empty() { url } else { format!("{url}?{query}") } } /// Provider configuration pointing at the fixture. fn provider_toml(base: &str) -> String { format!( r##" [provider] name = "fixture" display_name = "ChatGPT fixture" web_url = "{base}" new_conversation_url = "{base}" conversation_url_pattern = "#c=([0-9a-zA-Z-]{{8,}})" default_models = ["fixture-1"] enabled = true [selectors] input_field = "div#prompt-textarea" send_button = "button[data-testid='send-button']" response_container = "div[data-message-author-role='assistant']" streaming_indicator = "button[data-testid='stop-button']" file_input = "input[type='file']" new_chat_button = "a[data-testid='create-new-chat-button']" login_page_indicator = "form[action*='login']" captcha_indicator = "div#challenge-form" error_banner = "[data-testid='error-message'], div[role='alert']" attachment_indicator = "div[data-testid='attachment-chip']" copy_button = "button[data-testid='copy-button']" [login] url_patterns = ["login=1"] [rate_limit] text_patterns = ["too many requests"] [timeouts] page_load = 20 response_generation = 15 response_timeout = 30 [input] poll_interval_ms = 100 quiet_ms = 500 [options] supports_images = true supports_streaming = true supports_new_chat = true "## ) } struct Harness { registry: Arc, provider: Arc, _tmp: tempfile::TempDir, } /// Boot a registry with a single fixture provider, or None when no browser exists. async fn harness(toml: &str) -> Option { harness_with(toml, |_| {}).await } /// Same, with a chance to tweak the global configuration. async fn harness_with(toml: &str, configure: impl FnOnce(&mut GlobalConfig)) -> Option { init_logs(); let chrome = resolve_executable(None, std::env::var("CHROME").ok())?; let tmp = tempfile::tempdir().expect("temp dir"); let layout = Layout::rooted_at(tmp.path().join("state")); layout.ensure().expect("layout"); // Only the fixture provider may be registered: the state directory is // materialised with the providers shipped in the build. if let Ok(entries) = std::fs::read_dir(layout.providers_dir()) { for entry in entries.flatten() { std::fs::remove_file(entry.path()).ok(); } } std::fs::write(layout.provider_config("fixture"), toml).expect("provider config"); let mut config = GlobalConfig::default(); config.browser.headless = true; config.browser.executable = chrome.to_string_lossy().to_string(); config.capture.poll_interval_ms = 100; config.capture.quiet_ms = 500; config.capture.response_timeout_s = 30; config.debug.screenshots = ScreenshotMode::OnError; config.debug.dir = tmp.path().join("debug").to_string_lossy().to_string(); configure(&mut config); let registry = Arc::new(ProviderRegistry::new(layout, config)); registry.reload().await.expect("registry reload"); let provider = registry .get("fixture") .expect("fixture provider registered"); Some(Harness { registry, provider, _tmp: tmp, }) } /// Run one turn and return the deltas plus the final result. async fn run_turn( provider: &Arc, session: &Session, prompt: &str, reuse_url: Option, attachments: Vec, ) -> (Vec, Result) { let request = TurnRequest { prompt: prompt.to_string(), attachments, replay_history: reuse_url.is_none(), reuse_url, probe: false, trace_id: "e2e".to_string(), wait_for_tab: None, }; let mut stream = match provider.run_turn(session.clone(), request).await { Ok(stream) => stream, Err(err) => return (Vec::new(), Err(err)), }; let mut deltas = Vec::new(); loop { match stream.recv().await { Some(llm_bridge::providers::TurnEvent::Delta(delta)) => deltas.push(delta), Some(llm_bridge::providers::TurnEvent::Completed(outcome)) => { return (deltas, Ok(outcome)) } Some(llm_bridge::providers::TurnEvent::Failed(err)) => return (deltas, Err(err)), None => { return ( deltas, Err(GatewayError::internal( "the turn ended without a terminal event", )), ) } } } } /// Assert that the error carries the expected machine readable code. fn expect_code(result: Result, code: &str) { match result { Ok(outcome) => panic!("expected {code}, got an answer: {}", outcome.text), Err(err) => assert_eq!(err.code(), code, "unexpected error: {err}"), } } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn fixture_full_pipeline() { let Some(harness) = harness(&provider_toml(&fixture_url("conv=1"))).await else { eprintln!("no browser found, skipping"); return; }; assert_eq!( harness.registry.all().len(), 1, "only the fixture provider is configured" ); let session = harness.provider.session(None).await.expect("session"); let (deltas, result) = timeout( Duration::from_secs(90), run_turn( &harness.provider, &session, "hello fixture", None, Vec::new(), ), ) .await .expect("the first turn timed out"); let outcome = result.expect("the first turn failed"); assert!( outcome.text.starts_with("Echo: hello fixture"), "{}", outcome.text ); assert!(outcome.text.contains("fixture page"), "{}", outcome.text); assert!( deltas.len() >= 2, "expected several streamed deltas, got {deltas:?}" ); assert_eq!(deltas.concat().trim(), outcome.text.trim()); let web_id = outcome .web_id .clone() .expect("the conversation id must be extracted"); assert!( web_id.len() >= 8, "suspicious conversation id {web_id:?} from {:?}", outcome.web_url ); // Continuing the thread reopens the same web conversation. let reuse = outcome.web_url.clone().expect("web url"); let (_, second) = timeout( Duration::from_secs(90), run_turn( &harness.provider, &session, "second question", Some(reuse), Vec::new(), ), ) .await .expect("the second turn timed out"); let second = second.expect("the second turn failed"); assert!( second.text.starts_with("Echo: second question"), "{}", second.text ); assert_eq!( second.web_id, outcome.web_id, "the web conversation changed" ); // The provider passes its own validation. The validation opens its own // tab, so the one this test holds is released first. drop(session); let cfg = harness.provider.cfg(); let report = selftest::validate_provider(harness.provider.as_ref(), &cfg, None) .await .expect("validation"); assert!(report.ok, "selftest failed: {report:?}"); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn fixture_selftest_works_on_a_brand_new_session() { let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else { eprintln!("no browser found, skipping"); return; }; let session = harness.provider.session(None).await.expect("session"); // This is what the CLI does: a session that never navigated anywhere yet. let start_page = session .page() .url() .await .unwrap_or_default() .unwrap_or_default(); assert!( start_page.is_empty() || start_page == "about:blank", "a new session should start on about:blank, got {start_page:?}" ); // The selftest acquires its own tab: release this one first, otherwise a // single-tab provider (max_tabs = 1) answers "busy" instead. drop(session); let cfg = harness.provider.cfg(); let report = selftest::validate_provider(harness.provider.as_ref(), &cfg, None) .await .expect("validation on a fresh session"); assert!(report.ok, "selftest on a fresh session failed: {report:?}"); assert_eq!(report.status, "ok"); // The check must be quick: it opens the page instead of waiting for a // selector to appear on about:blank. assert!( report.duration_ms < 20_000, "selftest took {} ms", report.duration_ms ); let probe_step = report .steps .iter() .find(|step| step.step == "page_probe") .expect("page_probe step"); assert!(probe_step.ok, "page_probe failed: {probe_step:?}"); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn fixture_reports_login_captcha_error_and_rate_limit() { let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else { eprintln!("no browser found, skipping"); return; }; let session = harness.provider.session(None).await.expect("session"); let cases = [ ("login=1", "requires_login"), ("captcha=1", "captcha_required"), ("error=1", "upstream_error"), ("ratelimit=1", "upstream_rate_limit"), ]; for (query, code) in cases { let url = fixture_url(query); let (_, result) = timeout( Duration::from_secs(60), run_turn(&harness.provider, &session, "hello", Some(url), Vec::new()), ) .await .unwrap_or_else(|_| panic!("the {query} case timed out")); expect_code(result, code); } harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn fixture_timeouts_and_missing_selectors() { // A page that never produces text trips the first-token budget. let impatient = provider_toml(&fixture_url("")) .replace("response_generation = 15", "response_generation = 3"); let Some(harness) = harness(&impatient).await else { eprintln!("no browser found, skipping"); return; }; let session = harness.provider.session(None).await.expect("session"); let (_, result) = timeout( Duration::from_secs(60), run_turn( &harness.provider, &session, "hello", Some(fixture_url("empty=1")), Vec::new(), ), ) .await .expect("the empty case timed out"); expect_code(result, "upstream_timeout"); // A page whose editor appears late is waited for, not failed. let (_, result) = timeout( Duration::from_secs(90), run_turn( &harness.provider, &session, "late editor", Some(fixture_url("loading=1")), Vec::new(), ), ) .await .expect("the loading case timed out"); let outcome = result.expect("the late editor should still be driven"); assert!( outcome.text.starts_with("Echo: late editor"), "{}", outcome.text ); // A stale selector is reported as such. let broken = provider_toml(&fixture_url("")) .replace( "input_field = \"div#prompt-textarea\"", "input_field = \"div#this-does-not-exist\"", ) .replace("page_load = 20", "page_load = 4"); harness .provider .replace_config(ProviderConfig::parse(&broken, "fixture.toml").expect("broken config")) .await; let (_, result) = timeout( Duration::from_secs(60), run_turn(&harness.provider, &session, "hello", None, Vec::new()), ) .await .expect("the broken selector case timed out"); expect_code(result, "selector_missing"); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn two_tabs_of_one_browser_answer_concurrently() { let toml = provider_toml(&fixture_url("conv=1")); let Some(harness) = harness_with(&toml, |config| { config.browser.max_tabs = 2; }) .await else { eprintln!("no browser found, skipping"); return; }; let first = harness.provider.session(None).await.expect("first tab"); let second = harness.provider.session(None).await.expect("second tab"); let url = fixture_url("conv=1"); let outcome = timeout(Duration::from_secs(30), async { // Navigate both tabs at once... let (a, b) = tokio::join!( llm_bridge::browser::dom::goto(first.page(), &url, Duration::from_secs(10)), llm_bridge::browser::dom::goto(second.page(), &url, Duration::from_secs(10)), ); a.expect("the first tab did not navigate"); b.expect("the second tab did not navigate"); // ... then hammer both, the way the capture loop does. for _ in 0..20 { let (a, b) = tokio::join!( first.page().evaluate_expression("document.title"), second.page().evaluate_expression("document.title"), ); a.expect("the first tab stopped answering"); b.expect("the second tab stopped answering"); } }) .await; assert!( outcome.is_ok(), "two tabs of one browser stopped answering: {outcome:?}" ); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn two_tabs_serve_two_turns_in_parallel() { let toml = provider_toml(&fixture_url("conv=1&delay=5000")); let Some(harness) = harness_with(&toml, |config| { config.browser.max_tabs = 2; }) .await else { eprintln!("no browser found, skipping"); return; }; // Two tabs of the same account can be leased at the same time. let first = harness.provider.session(None).await.expect("first tab"); let second = harness.provider.session(None).await.expect("second tab"); assert_ne!(first.tab_slot(), second.tab_slot()); let stats = harness.provider.tab_stats(); assert_eq!(stats[0].1.max_tabs, 2); assert_eq!(stats[0].1.busy, 2); let started = std::time::Instant::now(); let (a, b) = tokio::join!( run_turn(&harness.provider, &first, "first", None, Vec::new()), run_turn(&harness.provider, &second, "second", None, Vec::new()), ); let elapsed = started.elapsed(); let a = a.1.expect("the first turn failed"); let b = b.1.expect("the second turn failed"); assert!(a.text.contains("first"), "{}", a.text); assert!(b.text.contains("second"), "{}", b.text); // Each generation takes five seconds: serving them one after the other // takes more than ten, two tabs take about six. assert!( elapsed < Duration::from_secs(9), "two tabs did not run in parallel: {elapsed:?}" ); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn a_busy_single_tab_account_refuses_a_second_request() { let toml = provider_toml(&fixture_url("conv=1")); let Some(harness) = harness_with(&toml, |config| { config.browser.max_tabs = 1; config.browser.busy_wait_s = 1; }) .await else { eprintln!("no browser found, skipping"); return; }; let held = harness.provider.session(None).await.expect("the only tab"); let started = std::time::Instant::now(); let err = harness .provider .session(None) .await .expect_err("the second request must not get a tab"); assert_eq!(err.code(), "provider_busy"); assert_eq!(err.http_status(), 429); assert!( started.elapsed() >= Duration::from_secs(1), "the request did not wait for the configured budget" ); drop(held); assert!(harness.provider.session(None).await.is_ok()); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn each_account_has_its_own_browser_and_tab_pool() { let mut toml = provider_toml(&fixture_url("conv=1&delay=3000")); toml.push_str("\n[[accounts]]\nid = \"un\"\ndefault = true\n\n[[accounts]]\nid = \"deux\"\n"); let Some(harness) = harness(&toml).await else { eprintln!("no browser found, skipping"); return; }; let accounts = harness.provider.accounts(); assert_eq!(accounts.len(), 2); assert_ne!(accounts[0].profile, accounts[1].profile); // max_tabs is 1 for both accounts, yet one tab per account is available at // the same time: the pools are per account, not per provider. let one = harness .provider .session(Some("un")) .await .expect("account un"); let two = harness .provider .session(Some("deux")) .await .expect("account deux"); assert_eq!(one.account(), Some("un")); assert_eq!(two.account(), Some("deux")); let started = std::time::Instant::now(); let (a, b) = tokio::join!( run_turn(&harness.provider, &one, "un", None, Vec::new()), run_turn(&harness.provider, &two, "deux", None, Vec::new()), ); let elapsed = started.elapsed(); assert!(a.1.is_ok(), "account un failed: {:?}", a.1.err()); assert!(b.1.is_ok(), "account deux failed: {:?}", b.1.err()); assert!( elapsed < Duration::from_secs(6), "the two accounts did not answer at the same time: {elapsed:?}" ); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn fixture_returns_markdown_and_never_the_toolbar_text() { let toml = provider_toml(&fixture_url("conv=1&rich=1")); let Some(harness) = harness(&toml).await else { eprintln!("no browser found, skipping"); return; }; let session = harness.provider.session(None).await.expect("session"); // Auto mode: the provider copy button first, the HTML conversion otherwise. let (_, result) = timeout( Duration::from_secs(90), run_turn(&harness.provider, &session, "rich answer", None, Vec::new()), ) .await .expect("the rich turn timed out"); let outcome = result.expect("the rich turn failed"); assert!( outcome.fidelity == "clipboard" || outcome.fidelity == "dom", "unexpected fidelity {:?}: {}", outcome.fidelity, outcome.text ); eprintln!("rich answer captured through the {} path", outcome.fidelity); // Structure the plain text capture used to lose. assert!(outcome.text.contains("## Answer"), "{}", outcome.text); assert!(outcome.text.contains("- first point"), "{}", outcome.text); assert!(outcome.text.contains("```rust"), "{}", outcome.text); assert!( outcome.text.contains("| Name | Value |"), "{}", outcome.text ); assert!(outcome.text.contains("| alpha | 1 |"), "{}", outcome.text); // The toolbar sits next to the answer, so its label is not answer text. assert!(!outcome.text.contains("Copy"), "{}", outcome.text); // Forcing the DOM conversion must give the same structure. let forced = provider_toml(&fixture_url("conv=1&rich=1")).replace( "poll_interval_ms = 100", "poll_interval_ms = 100\nmarkdown = \"dom\"", ); harness .provider .replace_config(ProviderConfig::parse(&forced, "fixture.toml").expect("forced config")) .await; let (_, result) = timeout( Duration::from_secs(90), run_turn(&harness.provider, &session, "dom answer", None, Vec::new()), ) .await .expect("the forced DOM turn timed out"); let forced_outcome = result.expect("the forced DOM turn failed"); assert_eq!(forced_outcome.fidelity, "dom"); assert!( forced_outcome.text.contains("## Answer"), "{}", forced_outcome.text ); assert!( forced_outcome.text.contains("```rust"), "{}", forced_outcome.text ); assert!( forced_outcome.text.contains("| alpha | 1 |"), "{}", forced_outcome.text ); assert!( !forced_outcome.text.contains("Copy"), "{}", forced_outcome.text ); harness.provider.shutdown().await.expect("shutdown"); } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn fixture_handles_rewrites_and_attachments() { let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else { eprintln!("no browser found, skipping"); return; }; let session = harness.provider.session(None).await.expect("session"); // A mid-stream rewrite must not corrupt the final answer. let (_, result) = timeout( Duration::from_secs(90), run_turn( &harness.provider, &session, "rewrite please", Some(fixture_url("rewrite=1")), Vec::new(), ), ) .await .expect("the rewrite case timed out"); let outcome = result.expect("the rewrite case failed"); assert!( outcome.text.starts_with("Echo: rewrite please"), "{}", outcome.text ); assert!(!outcome.text.contains("Draft answer"), "{}", outcome.text); // Attachments are pushed into the file input and acknowledged by the page. let dir = std::env::temp_dir().join("llm-gateway-e2e"); std::fs::create_dir_all(&dir).expect("temp dir"); let image = dir.join("pixel.png"); std::fs::write(&image, b"\x89PNG\r\n\x1a\n").expect("write png"); let (_, result) = timeout( Duration::from_secs(90), run_turn( &harness.provider, &session, "with an image", None, vec![Attachment { path: image, mime: "image/png".to_string(), kind: AttachmentKind::Image, }], ), ) .await .expect("the attachment case timed out"); let outcome = result.expect("the attachment case failed"); assert!(outcome.text.contains("pixel.png"), "{}", outcome.text); harness.provider.shutdown().await.expect("shutdown"); } /// Wait until something accepts TCP connections on a port. async fn wait_for_port(port: u16, budget: Duration) -> bool { let deadline = std::time::Instant::now() + budget; while std::time::Instant::now() < deadline { if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() { return true; } tokio::time::sleep(Duration::from_millis(200)).await; } false } /// Wait until the browser is really up: its debugging endpoint answers and the /// page we asked it to open is loaded. async fn wait_for_ready_browser(port: u16, expected_url: &str, budget: Duration) -> bool { let client = reqwest::Client::new(); let endpoint = format!("http://127.0.0.1:{port}/json/list"); let deadline = std::time::Instant::now() + budget; while std::time::Instant::now() < deadline { if let Ok(response) = client.get(&endpoint).send().await { if let Ok(targets) = response.json::>().await { let ready = targets.iter().any(|target| { target["type"] == "page" && target["url"] .as_str() .is_some_and(|url| url.contains(expected_url)) }); if ready { return true; } } } tokio::time::sleep(Duration::from_millis(250)).await; } false } /// Kills and reaps a browser process the test started, on every path. struct BrowserProcess { child: std::process::Child, } impl BrowserProcess { fn spawn(launch: &llm_bridge::browser::ExternalLaunch) -> Self { Self { child: launch.spawn().expect("spawning the browser"), } } fn is_running(&mut self) -> bool { self.child.try_wait().ok().flatten().is_none() } } impl Drop for BrowserProcess { fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); } } /// Ask the OS for a free port. fn free_port() -> u16 { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind an ephemeral port"); listener.local_addr().expect("local address").port() } #[tokio::test] #[ignore = "requires a real Chromium-based browser"] async fn attach_mode_uses_a_browser_the_user_started() { init_logs(); let Some(chrome) = resolve_executable(None, std::env::var("CHROME").ok()) else { eprintln!("no browser found, skipping"); return; }; let tmp = tempfile::tempdir().expect("temp dir"); let profile = tmp.path().join("attached-profile"); let port = free_port(); // The user starts the browser: no automation switch, no user agent spoofing. let launch = external_launch(Some(&chrome), &profile, &fixture_url("conv=1"), Some(port)); assert!(!launch.command_line().contains("enable-automation")); let mut browser = BrowserProcess::spawn(&launch); if !wait_for_port(port, Duration::from_secs(30)).await { panic!("the browser never opened its debugging port"); } // Attaching to a browser that is still starting up races with Chrome // creating its first tab: wait until the page is really there. if !wait_for_ready_browser(port, "chatgpt_mock.html", Duration::from_secs(30)).await { panic!("the browser never finished opening the fixture page"); } let Some(harness) = harness_with(&provider_toml(&fixture_url("conv=1")), |config| { config.browser.attach = true; config.browser.debug_port = port; }) .await else { return; }; let session = harness .provider .session(None) .await .expect("attach to the running browser"); // The gateway reuses a tab the browser exposes to us, and otherwise opens // one of its own in that same browser: either way the session, the cookies // and the window belong to the user and are never closed by the gateway. let url = session .page() .url() .await .unwrap_or_default() .unwrap_or_default(); assert!( !url.starts_with("devtools://"), "unexpected page attached: {url:?}" ); let (deltas, result) = timeout( Duration::from_secs(90), run_turn( &harness.provider, &session, "attached hello", None, Vec::new(), ), ) .await .expect("the attached turn timed out"); let outcome = result.expect("the attached turn failed"); assert!( outcome.text.starts_with("Echo: attached hello"), "{}", outcome.text ); assert!( deltas.len() >= 2, "expected streamed deltas, got {deltas:?}" ); // Shutting the gateway down must leave the user's browser alone. harness.provider.shutdown().await.expect("shutdown"); tokio::time::sleep(Duration::from_millis(500)).await; assert!( browser.is_running(), "attach mode must not close the browser it connected to" ); } #[test] fn the_fixture_configuration_is_valid() { let cfg = ProviderConfig::parse(&provider_toml(&fixture_url("")), "fixture.toml").unwrap(); assert!(cfg.is_usable(), "{:?}", cfg.problems); assert!(cfg.capabilities().supports_images); assert_eq!(cfg.file.timeouts.response_generation, 15); }