Add initial llm-bridge project
CI / fmt, clippy and tests (ubuntu-latest) (push) Successful in 21m0s
CI / browser end-to-end (Chromium + fixture) (push) Skipped
CI / fmt, clippy and tests (windows-latest) (push) Canceled after 0s

This commit is contained in:
2026-09-18 18:01:07 -04:00
commit 7b5aa6fcc9
58 changed files with 19914 additions and 0 deletions
+764
View File
@@ -0,0 +1,764 @@
//! HTTP integration tests against the mock backend: no browser, no network.
use std::sync::Arc;
use std::time::Duration;
use arc_swap::ArcSwap;
use axum::body::Body;
use axum::http::{HeaderMap, Request, StatusCode};
use axum::Router;
use http_body_util::BodyExt;
use llm_bridge::config::{CliOverrides, GlobalConfig, ScreenshotMode};
use llm_bridge::conversation::ThreadStore;
use llm_bridge::paths::Layout;
use llm_bridge::providers::{AccountInfo, ChatBackend, MockBackend};
use llm_bridge::server::{router, AppState, LogBuffer};
use serde_json::{json, Value};
use tower::ServiceExt;
struct Harness {
app: Router,
store: Arc<ThreadStore>,
state: AppState,
}
/// The two accounts of the mock ChatGPT provider.
fn mock_accounts() -> Vec<AccountInfo> {
vec![
AccountInfo {
id: "perso".to_string(),
email: Some("[email protected]".to_string()),
label: None,
default: true,
profile: "profiles/chatgpt/accounts/perso/user-data-dir".to_string(),
},
AccountInfo {
id: "pro".to_string(),
email: Some("[email protected]".to_string()),
label: Some("Work".to_string()),
default: false,
profile: "profiles/chatgpt/accounts/pro/user-data-dir".to_string(),
},
]
}
fn mock_backend(accounts: bool) -> MockBackend {
let backend = MockBackend::new(
vec![("chatgpt", "gpt-4o"), ("claude", "claude-sonnet-4")],
Duration::from_millis(1),
);
if accounts {
backend.with_accounts("chatgpt", mock_accounts())
} else {
backend
}
}
fn harness_with(config: GlobalConfig) -> Harness {
harness_with_backend(config, mock_backend(false))
}
fn harness_with_backend(config: GlobalConfig, backend: MockBackend) -> Harness {
let backend: Arc<dyn ChatBackend> = Arc::new(backend);
let store = Arc::new(ThreadStore::in_memory());
let layout = Layout::rooted_at(std::env::temp_dir().join("llm-gateway-tests"));
let state = AppState::new(
backend,
Arc::clone(&store),
layout,
Arc::new(ArcSwap::from_pointee(config)),
CliOverrides::default(),
LogBuffer::new(32),
);
Harness {
app: router(state.clone()),
store,
state,
}
}
fn harness() -> Harness {
harness_with(GlobalConfig::default())
}
/// A harness whose ChatGPT provider declares two accounts.
fn harness_with_accounts(config: GlobalConfig) -> Harness {
harness_with_backend(config, mock_backend(true))
}
async fn send(app: &Router, request: Request<Body>) -> (StatusCode, HeaderMap, String) {
let response = app
.clone()
.oneshot(request)
.await
.expect("router call failed");
let status = response.status();
let headers = response.headers().clone();
let bytes = response
.into_body()
.collect()
.await
.expect("body")
.to_bytes();
(status, headers, String::from_utf8_lossy(&bytes).to_string())
}
async fn get_json(app: &Router, path: &str) -> (StatusCode, Value) {
let request = Request::builder().uri(path).body(Body::empty()).unwrap();
let (status, _, body) = send(app, request).await;
(status, serde_json::from_str(&body).unwrap_or(Value::Null))
}
fn post_chat(body: Value) -> Request<Body> {
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap()
}
#[tokio::test]
async fn health_and_models_describe_every_provider() {
let harness = harness();
let (status, health) = get_json(&harness.app, "/health").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(health["status"], "ok");
assert_eq!(health["providers"].as_array().unwrap().len(), 2);
let (status, models) = get_json(&harness.app, "/v1/models").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(models["object"], "list");
let ids: Vec<String> = models["data"]
.as_array()
.unwrap()
.iter()
.map(|entry| entry["id"].as_str().unwrap().to_string())
.collect();
assert!(ids.contains(&"chatgpt".to_string()));
assert!(ids.contains(&"chatgpt/gpt-4o".to_string()));
assert!(ids.contains(&"claude/claude-sonnet-4".to_string()));
}
#[tokio::test]
async fn a_chat_completion_returns_the_openai_shape_and_a_conversation_id() {
let harness = harness();
let (status, headers, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hello"}],
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["object"], "chat.completion");
assert_eq!(value["model"], "chatgpt");
assert_eq!(value["choices"][0]["message"]["role"], "assistant");
assert_eq!(value["choices"][0]["finish_reason"], "stop");
let content = value["choices"][0]["message"]["content"].as_str().unwrap();
assert!(!content.is_empty());
assert!(value["usage"]["total_tokens"].as_u64().unwrap() >= 1);
assert!(value["id"].as_str().unwrap().starts_with("chatcmpl-"));
let conversation = headers
.get("x-conversation-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
assert!(conversation.starts_with("fp-"), "{conversation}");
assert!(headers.get("x-trace-id").is_some());
// The thread mapping was persisted for the next turn.
let record = harness
.store
.get(&conversation)
.await
.expect("thread recorded");
assert_eq!(record.provider, "chatgpt");
assert!(record.web_url.is_some());
}
#[tokio::test]
async fn the_same_conversation_is_reused_when_the_header_is_repeated() {
let harness = harness();
let request = |text: &str| {
post_chat(json!({
"model": "chatgpt",
"messages": [
{"role": "user", "content": "opening"},
{"role": "assistant", "content": "answer"},
{"role": "user", "content": text},
],
}))
};
let (_, headers, _) = send(&harness.app, request("second")).await;
let conversation = headers
.get("x-conversation-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
let (status, _, _) = send(&harness.app, request("third")).await;
assert_eq!(status, StatusCode::OK);
let record = harness.store.get(&conversation).await.unwrap();
assert_eq!(
record.turns, 2,
"the second call must reuse the stored thread"
);
}
#[tokio::test]
async fn streaming_emits_sse_chunks_and_terminates_with_done() {
let harness = harness();
let (status, headers, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hello"}],
"stream": true,
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let content_type = headers.get("content-type").unwrap().to_str().unwrap();
assert!(
content_type.starts_with("text/event-stream"),
"{content_type}"
);
assert!(body.contains("chat.completion.chunk"), "{body}");
assert!(body.contains(r#""role":"assistant""#), "{body}");
assert!(body.contains("data: [DONE]"), "{body}");
let deltas: String = body
.lines()
.filter_map(|line| line.strip_prefix("data: "))
.filter(|payload| *payload != "[DONE]")
.filter_map(|payload| serde_json::from_str::<Value>(payload).ok())
.filter_map(|chunk| {
chunk["choices"][0]["delta"]["content"]
.as_str()
.map(str::to_string)
})
.collect();
assert!(!deltas.is_empty());
assert!(deltas.contains("mock backend"), "{deltas}");
}
#[tokio::test]
async fn the_conversation_header_wins_over_the_fingerprint() {
let harness = harness();
let request = Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.header("x-conversation-id", "client-thread-42")
.body(Body::from(
json!({"model": "chatgpt", "messages": [{"role": "user", "content": "hi"}]})
.to_string(),
))
.unwrap();
let (status, headers, _) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(
headers.get("x-conversation-id").unwrap(),
"client-thread-42"
);
assert!(harness.store.get("client-thread-42").await.is_some());
}
#[tokio::test]
async fn unsupported_parameters_are_rejected_with_a_clear_error() {
let harness = harness();
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hi"}],
"tools": [{"type": "function", "function": {"name": "f"}}],
})),
)
.await;
assert_eq!(status, StatusCode::BAD_REQUEST);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["error"]["code"], "unsupported_parameter");
assert_eq!(value["error"]["param"], "tools");
assert_eq!(value["error"]["type"], "invalid_request_error");
// n = 0 is a client mistake, unlike n > 1 which is now supported.
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hi"}],
"n": 0,
})),
)
.await;
assert_eq!(status, StatusCode::BAD_REQUEST);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["error"]["param"], "n");
}
#[tokio::test]
async fn several_completions_come_back_as_several_choices() {
let harness = harness();
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hello"}],
"n": 3,
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
let choices = value["choices"].as_array().unwrap();
assert_eq!(choices.len(), 3);
for (index, choice) in choices.iter().enumerate() {
assert_eq!(choice["index"], index as u64);
assert_eq!(choice["message"]["role"], "assistant");
assert_eq!(choice["finish_reason"], "stop");
assert!(!choice["message"]["content"].as_str().unwrap().is_empty());
}
assert!(value["usage"]["total_tokens"].as_u64().unwrap() >= 1);
let warnings = value["x_gateway_warnings"].as_array().unwrap();
assert!(
warnings.iter().any(|w| w
.as_str()
.unwrap()
.contains("independent web conversations")),
"{warnings:?}"
);
}
#[tokio::test]
async fn n_is_clamped_to_the_configured_maximum() {
let mut config = GlobalConfig::default();
config.capture.max_variants = 2;
let harness = harness_with(config);
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hello"}],
"n": 9,
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["choices"].as_array().unwrap().len(), 2);
let warnings = value["x_gateway_warnings"].as_array().unwrap();
assert!(
warnings
.iter()
.any(|w| w.as_str().unwrap().contains("clamped to 2")),
"{warnings:?}"
);
}
#[tokio::test]
async fn streamed_choices_are_tagged_with_their_index() {
let harness = harness();
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hello"}],
"stream": true,
"n": 2,
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let chunks: Vec<Value> = body
.lines()
.filter_map(|line| line.strip_prefix("data: "))
.filter(|payload| *payload != "[DONE]")
.filter_map(|payload| serde_json::from_str::<Value>(payload).ok())
.collect();
assert!(body.contains("data: [DONE]"), "{body}");
let indexes: Vec<u64> = chunks
.iter()
.flat_map(|chunk| chunk["choices"].as_array().cloned().unwrap_or_default())
.filter_map(|choice| choice["index"].as_u64())
.collect();
assert!(indexes.contains(&0), "{indexes:?}");
assert!(indexes.contains(&1), "{indexes:?}");
let stops = chunks
.iter()
.flat_map(|chunk| chunk["choices"].as_array().cloned().unwrap_or_default())
.filter(|choice| choice["finish_reason"] == "stop")
.count();
assert_eq!(stops, 2, "each choice must finish: {body}");
}
#[tokio::test]
async fn an_account_can_be_selected_in_the_model_string() {
let harness = harness_with_accounts(GlobalConfig::default());
// The declared accounts show up in /v1/models and /health.
let (_, models) = get_json(&harness.app, "/v1/models").await;
let ids: Vec<String> = models["data"]
.as_array()
.unwrap()
.iter()
.map(|entry| entry["id"].as_str().unwrap().to_string())
.collect();
assert!(ids.contains(&"chatgpt@perso".to_string()), "{ids:?}");
assert!(ids.contains(&"chatgpt@pro".to_string()), "{ids:?}");
let (_, health) = get_json(&harness.app, "/health").await;
let chatgpt = health["providers"]
.as_array()
.unwrap()
.iter()
.find(|provider| provider["name"] == "chatgpt")
.unwrap();
assert_eq!(chatgpt["accounts"].as_array().unwrap().len(), 2);
// By account id, and by e-mail address.
for model in ["chatgpt@pro", "[email protected]@gmail.com"] {
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": model,
"messages": [{"role": "user", "content": "hello"}],
})),
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["model"], model);
}
}
#[tokio::test]
async fn an_unknown_account_is_a_404_listing_the_known_ones() {
let harness = harness_with_accounts(GlobalConfig::default());
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt@nobody",
"messages": [{"role": "user", "content": "hello"}],
})),
)
.await;
assert_eq!(status, StatusCode::NOT_FOUND);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["error"]["code"], "account_not_found");
let message = value["error"]["message"].as_str().unwrap();
assert!(message.contains("no account 'nobody'"), "{message}");
assert!(message.contains("perso"), "{message}");
assert!(
!message.contains("does not match any configured provider"),
"the message must name the account problem: {message}"
);
}
#[tokio::test]
async fn two_accounts_never_share_a_thread() {
let harness = harness_with_accounts(GlobalConfig::default());
let request = |model: &str| {
post_chat(json!({
"model": model,
"messages": [{"role": "user", "content": "hello"}],
}))
};
let (_, first, _) = send(&harness.app, request("chatgpt@perso")).await;
let (_, second, _) = send(&harness.app, request("chatgpt@pro")).await;
let first = first
.get("x-conversation-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
let second = second
.get("x-conversation-id")
.unwrap()
.to_str()
.unwrap()
.to_string();
assert_ne!(first, second, "the account is part of the thread identity");
}
#[tokio::test]
async fn an_unknown_model_is_a_404() {
let harness = harness();
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "gemini",
"messages": [{"role": "user", "content": "hi"}],
})),
)
.await;
assert_eq!(status, StatusCode::NOT_FOUND);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["error"]["code"], "model_not_found");
assert!(value["error"]["message"]
.as_str()
.unwrap()
.contains("chatgpt"));
}
#[tokio::test]
async fn ignored_parameters_are_reported_in_the_response() {
let harness = harness();
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hi"}],
"temperature": 0.1,
"max_tokens": 10,
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
let warnings = value["x_gateway_warnings"].as_array().unwrap();
assert!(warnings
.iter()
.any(|w| w.as_str().unwrap().contains("temperature")));
assert!(warnings
.iter()
.any(|w| w.as_str().unwrap().contains("max_tokens")));
}
#[tokio::test]
async fn warnings_can_be_switched_off() {
let mut config = GlobalConfig::default();
config.server.include_warnings = false;
let harness = harness_with(config);
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hi"}],
"top_p": 0.5,
})),
)
.await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
assert!(value.get("x_gateway_warnings").is_none());
}
#[tokio::test]
async fn the_legacy_completions_endpoint_is_supported() {
let harness = harness();
let request = Request::builder()
.method("POST")
.uri("/v1/completions")
.header("content-type", "application/json")
.body(Body::from(
json!({"model": "chatgpt", "prompt": "say ok"}).to_string(),
))
.unwrap();
let (status, headers, body) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["object"], "text_completion");
assert!(!value["choices"][0]["text"].as_str().unwrap().is_empty());
assert!(headers.get("x-conversation-id").is_some());
let request = Request::builder()
.method("POST")
.uri("/v1/completions")
.header("content-type", "application/json")
.body(Body::from(
json!({"model": "chatgpt", "prompt": "say ok", "stream": true}).to_string(),
))
.unwrap();
let (status, _, body) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::BAD_REQUEST);
assert!(body.contains("unsupported_parameter"));
}
#[tokio::test]
async fn provider_status_and_validation_are_exposed() {
let harness = harness();
let (status, body) = get_json(&harness.app, "/v1/providers/chatgpt/status").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body["state"], "available");
assert_eq!(body["provider"], "chatgpt");
let request = Request::builder()
.method("POST")
.uri("/v1/providers/chatgpt/validate")
.body(Body::empty())
.unwrap();
let (status, _, body) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["ok"], true);
assert_eq!(value["status"], "ok");
assert!(!value["steps"].as_array().unwrap().is_empty());
let (status, body) = get_json(&harness.app, "/v1/providers/nope/status").await;
assert_eq!(status, StatusCode::NOT_FOUND);
assert_eq!(body["error"]["code"], "model_not_found");
}
#[tokio::test]
async fn provider_status_and_validation_can_name_an_account() {
let harness = harness_with_accounts(GlobalConfig::default());
// A named account is probed, and the answer says which one.
let (status, body) = get_json(&harness.app, "/v1/providers/chatgpt@pro/status").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body["state"], "available");
assert_eq!(body["provider"], "chatgpt@pro");
// The bare provider name means "its default account", and says so rather
// than leaving the caller to guess which profile answered.
let (status, body) = get_json(&harness.app, "/v1/providers/chatgpt/status").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body["provider"], "chatgpt@perso");
let (status, body) = post_account_validate(&harness, "chatgpt@pro").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body["ok"], true);
assert_eq!(body["provider"], "chatgpt@pro");
// The status route keeps working for a provider that declares no account.
let (status, body) = get_json(&harness.app, "/v1/providers/claude/status").await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body["provider"], "claude");
}
#[tokio::test]
async fn an_unknown_account_is_a_404_on_the_status_routes() {
let harness = harness_with_accounts(GlobalConfig::default());
let (status, body) = get_json(&harness.app, "/v1/providers/chatgpt@nobody/status").await;
assert_eq!(status, StatusCode::NOT_FOUND);
assert_eq!(body["error"]["code"], "account_not_found");
assert!(body["error"]["message"].as_str().unwrap().contains("perso"));
let (status, body) = post_account_validate(&harness, "chatgpt@nobody").await;
assert_eq!(status, StatusCode::NOT_FOUND);
assert_eq!(body["error"]["code"], "account_not_found");
// A provider that declares no account refuses a named one instead of
// silently probing its only profile.
let (status, body) = get_json(&harness.app, "/v1/providers/claude@perso/status").await;
assert_eq!(status, StatusCode::NOT_FOUND);
assert_eq!(body["error"]["code"], "account_not_found");
}
/// POST /v1/providers/{owner}/validate and parse the JSON body.
async fn post_account_validate(harness: &Harness, owner: &str) -> (StatusCode, Value) {
let request = Request::builder()
.method("POST")
.uri(format!("/v1/providers/{owner}/validate"))
.body(Body::empty())
.unwrap();
let (status, _, body) = send(&harness.app, request).await;
(status, serde_json::from_str(&body).unwrap_or(Value::Null))
}
#[tokio::test]
async fn reload_reports_the_configuration_diff() {
let harness = harness();
let request = Request::builder()
.method("POST")
.uri("/v1/admin/reload")
.body(Body::empty())
.unwrap();
let (status, _, body) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::OK);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["config"], "reloaded");
}
#[tokio::test]
async fn an_api_key_protects_the_v1_routes_only() {
let mut config = GlobalConfig::default();
config.server.api_key = "s3cret".to_string();
let harness = harness_with(config);
let (status, _, body) = send(
&harness.app,
post_chat(json!({
"model": "chatgpt",
"messages": [{"role": "user", "content": "hi"}],
})),
)
.await;
assert_eq!(status, StatusCode::UNAUTHORIZED);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["error"]["code"], "invalid_api_key");
let request = Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.header("authorization", "Bearer s3cret")
.body(Body::from(
json!({"model": "chatgpt", "messages": [{"role": "user", "content": "hi"}]})
.to_string(),
))
.unwrap();
let (status, _, _) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::OK);
// /health stays open so that monitors and the dashboard can reach it.
let (status, _) = get_json(&harness.app, "/health").await;
assert_eq!(status, StatusCode::OK);
}
#[tokio::test]
async fn a_malformed_body_is_a_400_with_the_openai_error_shape() {
let harness = harness();
let request = Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.body(Body::from("{not json"))
.unwrap();
let (status, _, body) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::BAD_REQUEST);
let value: Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["error"]["code"], "invalid_request");
}
#[tokio::test]
async fn the_dashboard_is_served_and_exposes_metrics() {
let harness = harness();
let request = Request::builder()
.uri("/dashboard")
.body(Body::empty())
.unwrap();
let (status, _, body) = send(&harness.app, request).await;
assert_eq!(status, StatusCode::OK);
assert!(body.contains("llm-gateway"));
let (status, metrics) = get_json(&harness.app, "/metrics").await;
assert_eq!(status, StatusCode::OK);
assert!(metrics["requests_total"].is_number());
// The screenshot policy is only reachable through the configuration.
let config = harness.state.config();
assert_eq!(config.debug.screenshots, ScreenshotMode::OnError);
}
+898
View File
@@ -0,0 +1,898 @@
//! Browser end-to-end tests against the local ChatGPT fixture page.
//!
//! They need a real Chromium-based browser and are therefore ignored by default:
//!
//! cargo test --test e2e_browser -- --ignored --test-threads=1
//!
//! The fixture is a static HTML file that imitates the real page (editor, send
//! button, streaming indicator, login form, captcha, error banner), so the whole
//! capture pipeline can be exercised without a network or a login.
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use llm_bridge::browser::{external_launch, resolve_executable, Session};
use llm_bridge::config::{GlobalConfig, ScreenshotMode};
use llm_bridge::error::GatewayError;
use llm_bridge::paths::Layout;
use llm_bridge::providers::config::ProviderConfig;
use llm_bridge::providers::{
Attachment, AttachmentKind, LlmProvider, ProviderRegistry, TurnOutcome, TurnRequest,
WebUiProvider,
};
use llm_bridge::selftest;
use tokio::time::timeout;
/// Install a tracing subscriber once so the tests are diagnosable.
fn init_logs() {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string());
let _ = tracing_subscriber::fmt()
.with_env_filter(filter)
.with_test_writer()
.try_init();
});
}
/// Absolute file:// URL of the fixture, optionally with a query string.
fn fixture_url(query: &str) -> String {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests")
.join("fixtures")
.join("chatgpt_mock.html");
let path = path.to_string_lossy().replace('\\', "/");
let url = format!("file:///{path}");
if query.is_empty() {
url
} else {
format!("{url}?{query}")
}
}
/// Provider configuration pointing at the fixture.
fn provider_toml(base: &str) -> String {
format!(
r##"
[provider]
name = "fixture"
display_name = "ChatGPT fixture"
web_url = "{base}"
new_conversation_url = "{base}"
conversation_url_pattern = "#c=([0-9a-zA-Z-]{{8,}})"
default_models = ["fixture-1"]
enabled = true
[selectors]
input_field = "div#prompt-textarea"
send_button = "button[data-testid='send-button']"
response_container = "div[data-message-author-role='assistant']"
streaming_indicator = "button[data-testid='stop-button']"
file_input = "input[type='file']"
new_chat_button = "a[data-testid='create-new-chat-button']"
login_page_indicator = "form[action*='login']"
captcha_indicator = "div#challenge-form"
error_banner = "[data-testid='error-message'], div[role='alert']"
attachment_indicator = "div[data-testid='attachment-chip']"
copy_button = "button[data-testid='copy-button']"
[login]
url_patterns = ["login=1"]
[rate_limit]
text_patterns = ["too many requests"]
[timeouts]
page_load = 20
response_generation = 15
response_timeout = 30
[input]
poll_interval_ms = 100
quiet_ms = 500
[options]
supports_images = true
supports_streaming = true
supports_new_chat = true
"##
)
}
struct Harness {
registry: Arc<ProviderRegistry>,
provider: Arc<WebUiProvider>,
_tmp: tempfile::TempDir,
}
/// Boot a registry with a single fixture provider, or None when no browser exists.
async fn harness(toml: &str) -> Option<Harness> {
harness_with(toml, |_| {}).await
}
/// Same, with a chance to tweak the global configuration.
async fn harness_with(toml: &str, configure: impl FnOnce(&mut GlobalConfig)) -> Option<Harness> {
init_logs();
let chrome = resolve_executable(None, std::env::var("CHROME").ok())?;
let tmp = tempfile::tempdir().expect("temp dir");
let layout = Layout::rooted_at(tmp.path().join("state"));
layout.ensure().expect("layout");
// Only the fixture provider may be registered: the state directory is
// materialised with the providers shipped in the build.
if let Ok(entries) = std::fs::read_dir(layout.providers_dir()) {
for entry in entries.flatten() {
std::fs::remove_file(entry.path()).ok();
}
}
std::fs::write(layout.provider_config("fixture"), toml).expect("provider config");
let mut config = GlobalConfig::default();
config.browser.headless = true;
config.browser.executable = chrome.to_string_lossy().to_string();
config.capture.poll_interval_ms = 100;
config.capture.quiet_ms = 500;
config.capture.response_timeout_s = 30;
config.debug.screenshots = ScreenshotMode::OnError;
config.debug.dir = tmp.path().join("debug").to_string_lossy().to_string();
configure(&mut config);
let registry = Arc::new(ProviderRegistry::new(layout, config));
registry.reload().await.expect("registry reload");
let provider = registry
.get("fixture")
.expect("fixture provider registered");
Some(Harness {
registry,
provider,
_tmp: tmp,
})
}
/// Run one turn and return the deltas plus the final result.
async fn run_turn(
provider: &Arc<WebUiProvider>,
session: &Session,
prompt: &str,
reuse_url: Option<String>,
attachments: Vec<Attachment>,
) -> (Vec<String>, Result<TurnOutcome, GatewayError>) {
let request = TurnRequest {
prompt: prompt.to_string(),
attachments,
replay_history: reuse_url.is_none(),
reuse_url,
probe: false,
trace_id: "e2e".to_string(),
wait_for_tab: None,
};
let mut stream = match provider.run_turn(session.clone(), request).await {
Ok(stream) => stream,
Err(err) => return (Vec::new(), Err(err)),
};
let mut deltas = Vec::new();
loop {
match stream.recv().await {
Some(llm_bridge::providers::TurnEvent::Delta(delta)) => deltas.push(delta),
Some(llm_bridge::providers::TurnEvent::Completed(outcome)) => {
return (deltas, Ok(outcome))
}
Some(llm_bridge::providers::TurnEvent::Failed(err)) => return (deltas, Err(err)),
None => {
return (
deltas,
Err(GatewayError::internal(
"the turn ended without a terminal event",
)),
)
}
}
}
}
/// Assert that the error carries the expected machine readable code.
fn expect_code(result: Result<TurnOutcome, GatewayError>, code: &str) {
match result {
Ok(outcome) => panic!("expected {code}, got an answer: {}", outcome.text),
Err(err) => assert_eq!(err.code(), code, "unexpected error: {err}"),
}
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn fixture_full_pipeline() {
let Some(harness) = harness(&provider_toml(&fixture_url("conv=1"))).await else {
eprintln!("no browser found, skipping");
return;
};
assert_eq!(
harness.registry.all().len(),
1,
"only the fixture provider is configured"
);
let session = harness.provider.session(None).await.expect("session");
let (deltas, result) = timeout(
Duration::from_secs(90),
run_turn(
&harness.provider,
&session,
"hello fixture",
None,
Vec::new(),
),
)
.await
.expect("the first turn timed out");
let outcome = result.expect("the first turn failed");
assert!(
outcome.text.starts_with("Echo: hello fixture"),
"{}",
outcome.text
);
assert!(outcome.text.contains("fixture page"), "{}", outcome.text);
assert!(
deltas.len() >= 2,
"expected several streamed deltas, got {deltas:?}"
);
assert_eq!(deltas.concat().trim(), outcome.text.trim());
let web_id = outcome
.web_id
.clone()
.expect("the conversation id must be extracted");
assert!(
web_id.len() >= 8,
"suspicious conversation id {web_id:?} from {:?}",
outcome.web_url
);
// Continuing the thread reopens the same web conversation.
let reuse = outcome.web_url.clone().expect("web url");
let (_, second) = timeout(
Duration::from_secs(90),
run_turn(
&harness.provider,
&session,
"second question",
Some(reuse),
Vec::new(),
),
)
.await
.expect("the second turn timed out");
let second = second.expect("the second turn failed");
assert!(
second.text.starts_with("Echo: second question"),
"{}",
second.text
);
assert_eq!(
second.web_id, outcome.web_id,
"the web conversation changed"
);
// The provider passes its own validation. The validation opens its own
// tab, so the one this test holds is released first.
drop(session);
let cfg = harness.provider.cfg();
let report = selftest::validate_provider(harness.provider.as_ref(), &cfg, None)
.await
.expect("validation");
assert!(report.ok, "selftest failed: {report:?}");
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn fixture_selftest_works_on_a_brand_new_session() {
let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else {
eprintln!("no browser found, skipping");
return;
};
let session = harness.provider.session(None).await.expect("session");
// This is what the CLI does: a session that never navigated anywhere yet.
let start_page = session
.page()
.url()
.await
.unwrap_or_default()
.unwrap_or_default();
assert!(
start_page.is_empty() || start_page == "about:blank",
"a new session should start on about:blank, got {start_page:?}"
);
// The selftest acquires its own tab: release this one first, otherwise a
// single-tab provider (max_tabs = 1) answers "busy" instead.
drop(session);
let cfg = harness.provider.cfg();
let report = selftest::validate_provider(harness.provider.as_ref(), &cfg, None)
.await
.expect("validation on a fresh session");
assert!(report.ok, "selftest on a fresh session failed: {report:?}");
assert_eq!(report.status, "ok");
// The check must be quick: it opens the page instead of waiting for a
// selector to appear on about:blank.
assert!(
report.duration_ms < 20_000,
"selftest took {} ms",
report.duration_ms
);
let probe_step = report
.steps
.iter()
.find(|step| step.step == "page_probe")
.expect("page_probe step");
assert!(probe_step.ok, "page_probe failed: {probe_step:?}");
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn fixture_reports_login_captcha_error_and_rate_limit() {
let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else {
eprintln!("no browser found, skipping");
return;
};
let session = harness.provider.session(None).await.expect("session");
let cases = [
("login=1", "requires_login"),
("captcha=1", "captcha_required"),
("error=1", "upstream_error"),
("ratelimit=1", "upstream_rate_limit"),
];
for (query, code) in cases {
let url = fixture_url(query);
let (_, result) = timeout(
Duration::from_secs(60),
run_turn(&harness.provider, &session, "hello", Some(url), Vec::new()),
)
.await
.unwrap_or_else(|_| panic!("the {query} case timed out"));
expect_code(result, code);
}
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn fixture_timeouts_and_missing_selectors() {
// A page that never produces text trips the first-token budget.
let impatient = provider_toml(&fixture_url(""))
.replace("response_generation = 15", "response_generation = 3");
let Some(harness) = harness(&impatient).await else {
eprintln!("no browser found, skipping");
return;
};
let session = harness.provider.session(None).await.expect("session");
let (_, result) = timeout(
Duration::from_secs(60),
run_turn(
&harness.provider,
&session,
"hello",
Some(fixture_url("empty=1")),
Vec::new(),
),
)
.await
.expect("the empty case timed out");
expect_code(result, "upstream_timeout");
// A page whose editor appears late is waited for, not failed.
let (_, result) = timeout(
Duration::from_secs(90),
run_turn(
&harness.provider,
&session,
"late editor",
Some(fixture_url("loading=1")),
Vec::new(),
),
)
.await
.expect("the loading case timed out");
let outcome = result.expect("the late editor should still be driven");
assert!(
outcome.text.starts_with("Echo: late editor"),
"{}",
outcome.text
);
// A stale selector is reported as such.
let broken = provider_toml(&fixture_url(""))
.replace(
"input_field = \"div#prompt-textarea\"",
"input_field = \"div#this-does-not-exist\"",
)
.replace("page_load = 20", "page_load = 4");
harness
.provider
.replace_config(ProviderConfig::parse(&broken, "fixture.toml").expect("broken config"))
.await;
let (_, result) = timeout(
Duration::from_secs(60),
run_turn(&harness.provider, &session, "hello", None, Vec::new()),
)
.await
.expect("the broken selector case timed out");
expect_code(result, "selector_missing");
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn two_tabs_of_one_browser_answer_concurrently() {
let toml = provider_toml(&fixture_url("conv=1"));
let Some(harness) = harness_with(&toml, |config| {
config.browser.max_tabs = 2;
})
.await
else {
eprintln!("no browser found, skipping");
return;
};
let first = harness.provider.session(None).await.expect("first tab");
let second = harness.provider.session(None).await.expect("second tab");
let url = fixture_url("conv=1");
let outcome = timeout(Duration::from_secs(30), async {
// Navigate both tabs at once...
let (a, b) = tokio::join!(
llm_bridge::browser::dom::goto(first.page(), &url, Duration::from_secs(10)),
llm_bridge::browser::dom::goto(second.page(), &url, Duration::from_secs(10)),
);
a.expect("the first tab did not navigate");
b.expect("the second tab did not navigate");
// ... then hammer both, the way the capture loop does.
for _ in 0..20 {
let (a, b) = tokio::join!(
first.page().evaluate_expression("document.title"),
second.page().evaluate_expression("document.title"),
);
a.expect("the first tab stopped answering");
b.expect("the second tab stopped answering");
}
})
.await;
assert!(
outcome.is_ok(),
"two tabs of one browser stopped answering: {outcome:?}"
);
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn two_tabs_serve_two_turns_in_parallel() {
let toml = provider_toml(&fixture_url("conv=1&delay=5000"));
let Some(harness) = harness_with(&toml, |config| {
config.browser.max_tabs = 2;
})
.await
else {
eprintln!("no browser found, skipping");
return;
};
// Two tabs of the same account can be leased at the same time.
let first = harness.provider.session(None).await.expect("first tab");
let second = harness.provider.session(None).await.expect("second tab");
assert_ne!(first.tab_slot(), second.tab_slot());
let stats = harness.provider.tab_stats();
assert_eq!(stats[0].1.max_tabs, 2);
assert_eq!(stats[0].1.busy, 2);
let started = std::time::Instant::now();
let (a, b) = tokio::join!(
run_turn(&harness.provider, &first, "first", None, Vec::new()),
run_turn(&harness.provider, &second, "second", None, Vec::new()),
);
let elapsed = started.elapsed();
let a = a.1.expect("the first turn failed");
let b = b.1.expect("the second turn failed");
assert!(a.text.contains("first"), "{}", a.text);
assert!(b.text.contains("second"), "{}", b.text);
// Each generation takes five seconds: serving them one after the other
// takes more than ten, two tabs take about six.
assert!(
elapsed < Duration::from_secs(9),
"two tabs did not run in parallel: {elapsed:?}"
);
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn a_busy_single_tab_account_refuses_a_second_request() {
let toml = provider_toml(&fixture_url("conv=1"));
let Some(harness) = harness_with(&toml, |config| {
config.browser.max_tabs = 1;
config.browser.busy_wait_s = 1;
})
.await
else {
eprintln!("no browser found, skipping");
return;
};
let held = harness.provider.session(None).await.expect("the only tab");
let started = std::time::Instant::now();
let err = harness
.provider
.session(None)
.await
.expect_err("the second request must not get a tab");
assert_eq!(err.code(), "provider_busy");
assert_eq!(err.http_status(), 429);
assert!(
started.elapsed() >= Duration::from_secs(1),
"the request did not wait for the configured budget"
);
drop(held);
assert!(harness.provider.session(None).await.is_ok());
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn each_account_has_its_own_browser_and_tab_pool() {
let mut toml = provider_toml(&fixture_url("conv=1&delay=3000"));
toml.push_str("\n[[accounts]]\nid = \"un\"\ndefault = true\n\n[[accounts]]\nid = \"deux\"\n");
let Some(harness) = harness(&toml).await else {
eprintln!("no browser found, skipping");
return;
};
let accounts = harness.provider.accounts();
assert_eq!(accounts.len(), 2);
assert_ne!(accounts[0].profile, accounts[1].profile);
// max_tabs is 1 for both accounts, yet one tab per account is available at
// the same time: the pools are per account, not per provider.
let one = harness
.provider
.session(Some("un"))
.await
.expect("account un");
let two = harness
.provider
.session(Some("deux"))
.await
.expect("account deux");
assert_eq!(one.account(), Some("un"));
assert_eq!(two.account(), Some("deux"));
let started = std::time::Instant::now();
let (a, b) = tokio::join!(
run_turn(&harness.provider, &one, "un", None, Vec::new()),
run_turn(&harness.provider, &two, "deux", None, Vec::new()),
);
let elapsed = started.elapsed();
assert!(a.1.is_ok(), "account un failed: {:?}", a.1.err());
assert!(b.1.is_ok(), "account deux failed: {:?}", b.1.err());
assert!(
elapsed < Duration::from_secs(6),
"the two accounts did not answer at the same time: {elapsed:?}"
);
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn fixture_returns_markdown_and_never_the_toolbar_text() {
let toml = provider_toml(&fixture_url("conv=1&rich=1"));
let Some(harness) = harness(&toml).await else {
eprintln!("no browser found, skipping");
return;
};
let session = harness.provider.session(None).await.expect("session");
// Auto mode: the provider copy button first, the HTML conversion otherwise.
let (_, result) = timeout(
Duration::from_secs(90),
run_turn(&harness.provider, &session, "rich answer", None, Vec::new()),
)
.await
.expect("the rich turn timed out");
let outcome = result.expect("the rich turn failed");
assert!(
outcome.fidelity == "clipboard" || outcome.fidelity == "dom",
"unexpected fidelity {:?}: {}",
outcome.fidelity,
outcome.text
);
eprintln!("rich answer captured through the {} path", outcome.fidelity);
// Structure the plain text capture used to lose.
assert!(outcome.text.contains("## Answer"), "{}", outcome.text);
assert!(outcome.text.contains("- first point"), "{}", outcome.text);
assert!(outcome.text.contains("```rust"), "{}", outcome.text);
assert!(
outcome.text.contains("| Name | Value |"),
"{}",
outcome.text
);
assert!(outcome.text.contains("| alpha | 1 |"), "{}", outcome.text);
// The toolbar sits next to the answer, so its label is not answer text.
assert!(!outcome.text.contains("Copy"), "{}", outcome.text);
// Forcing the DOM conversion must give the same structure.
let forced = provider_toml(&fixture_url("conv=1&rich=1")).replace(
"poll_interval_ms = 100",
"poll_interval_ms = 100\nmarkdown = \"dom\"",
);
harness
.provider
.replace_config(ProviderConfig::parse(&forced, "fixture.toml").expect("forced config"))
.await;
let (_, result) = timeout(
Duration::from_secs(90),
run_turn(&harness.provider, &session, "dom answer", None, Vec::new()),
)
.await
.expect("the forced DOM turn timed out");
let forced_outcome = result.expect("the forced DOM turn failed");
assert_eq!(forced_outcome.fidelity, "dom");
assert!(
forced_outcome.text.contains("## Answer"),
"{}",
forced_outcome.text
);
assert!(
forced_outcome.text.contains("```rust"),
"{}",
forced_outcome.text
);
assert!(
forced_outcome.text.contains("| alpha | 1 |"),
"{}",
forced_outcome.text
);
assert!(
!forced_outcome.text.contains("Copy"),
"{}",
forced_outcome.text
);
harness.provider.shutdown().await.expect("shutdown");
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn fixture_handles_rewrites_and_attachments() {
let Some(harness) = harness(&provider_toml(&fixture_url(""))).await else {
eprintln!("no browser found, skipping");
return;
};
let session = harness.provider.session(None).await.expect("session");
// A mid-stream rewrite must not corrupt the final answer.
let (_, result) = timeout(
Duration::from_secs(90),
run_turn(
&harness.provider,
&session,
"rewrite please",
Some(fixture_url("rewrite=1")),
Vec::new(),
),
)
.await
.expect("the rewrite case timed out");
let outcome = result.expect("the rewrite case failed");
assert!(
outcome.text.starts_with("Echo: rewrite please"),
"{}",
outcome.text
);
assert!(!outcome.text.contains("Draft answer"), "{}", outcome.text);
// Attachments are pushed into the file input and acknowledged by the page.
let dir = std::env::temp_dir().join("llm-gateway-e2e");
std::fs::create_dir_all(&dir).expect("temp dir");
let image = dir.join("pixel.png");
std::fs::write(&image, b"\x89PNG\r\n\x1a\n").expect("write png");
let (_, result) = timeout(
Duration::from_secs(90),
run_turn(
&harness.provider,
&session,
"with an image",
None,
vec![Attachment {
path: image,
mime: "image/png".to_string(),
kind: AttachmentKind::Image,
}],
),
)
.await
.expect("the attachment case timed out");
let outcome = result.expect("the attachment case failed");
assert!(outcome.text.contains("pixel.png"), "{}", outcome.text);
harness.provider.shutdown().await.expect("shutdown");
}
/// Wait until something accepts TCP connections on a port.
async fn wait_for_port(port: u16, budget: Duration) -> bool {
let deadline = std::time::Instant::now() + budget;
while std::time::Instant::now() < deadline {
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() {
return true;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
false
}
/// Wait until the browser is really up: its debugging endpoint answers and the
/// page we asked it to open is loaded.
async fn wait_for_ready_browser(port: u16, expected_url: &str, budget: Duration) -> bool {
let client = reqwest::Client::new();
let endpoint = format!("http://127.0.0.1:{port}/json/list");
let deadline = std::time::Instant::now() + budget;
while std::time::Instant::now() < deadline {
if let Ok(response) = client.get(&endpoint).send().await {
if let Ok(targets) = response.json::<Vec<serde_json::Value>>().await {
let ready = targets.iter().any(|target| {
target["type"] == "page"
&& target["url"]
.as_str()
.is_some_and(|url| url.contains(expected_url))
});
if ready {
return true;
}
}
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
false
}
/// Kills and reaps a browser process the test started, on every path.
struct BrowserProcess {
child: std::process::Child,
}
impl BrowserProcess {
fn spawn(launch: &llm_bridge::browser::ExternalLaunch) -> Self {
Self {
child: launch.spawn().expect("spawning the browser"),
}
}
fn is_running(&mut self) -> bool {
self.child.try_wait().ok().flatten().is_none()
}
}
impl Drop for BrowserProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
/// Ask the OS for a free port.
fn free_port() -> u16 {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind an ephemeral port");
listener.local_addr().expect("local address").port()
}
#[tokio::test]
#[ignore = "requires a real Chromium-based browser"]
async fn attach_mode_uses_a_browser_the_user_started() {
init_logs();
let Some(chrome) = resolve_executable(None, std::env::var("CHROME").ok()) else {
eprintln!("no browser found, skipping");
return;
};
let tmp = tempfile::tempdir().expect("temp dir");
let profile = tmp.path().join("attached-profile");
let port = free_port();
// The user starts the browser: no automation switch, no user agent spoofing.
let launch = external_launch(Some(&chrome), &profile, &fixture_url("conv=1"), Some(port));
assert!(!launch.command_line().contains("enable-automation"));
let mut browser = BrowserProcess::spawn(&launch);
if !wait_for_port(port, Duration::from_secs(30)).await {
panic!("the browser never opened its debugging port");
}
// Attaching to a browser that is still starting up races with Chrome
// creating its first tab: wait until the page is really there.
if !wait_for_ready_browser(port, "chatgpt_mock.html", Duration::from_secs(30)).await {
panic!("the browser never finished opening the fixture page");
}
let Some(harness) = harness_with(&provider_toml(&fixture_url("conv=1")), |config| {
config.browser.attach = true;
config.browser.debug_port = port;
})
.await
else {
return;
};
let session = harness
.provider
.session(None)
.await
.expect("attach to the running browser");
// The gateway reuses a tab the browser exposes to us, and otherwise opens
// one of its own in that same browser: either way the session, the cookies
// and the window belong to the user and are never closed by the gateway.
let url = session
.page()
.url()
.await
.unwrap_or_default()
.unwrap_or_default();
assert!(
!url.starts_with("devtools://"),
"unexpected page attached: {url:?}"
);
let (deltas, result) = timeout(
Duration::from_secs(90),
run_turn(
&harness.provider,
&session,
"attached hello",
None,
Vec::new(),
),
)
.await
.expect("the attached turn timed out");
let outcome = result.expect("the attached turn failed");
assert!(
outcome.text.starts_with("Echo: attached hello"),
"{}",
outcome.text
);
assert!(
deltas.len() >= 2,
"expected streamed deltas, got {deltas:?}"
);
// Shutting the gateway down must leave the user's browser alone.
harness.provider.shutdown().await.expect("shutdown");
tokio::time::sleep(Duration::from_millis(500)).await;
assert!(
browser.is_running(),
"attach mode must not close the browser it connected to"
);
}
#[test]
fn the_fixture_configuration_is_valid() {
let cfg = ProviderConfig::parse(&provider_toml(&fixture_url("")), "fixture.toml").unwrap();
assert!(cfg.is_usable(), "{:?}", cfg.problems);
assert!(cfg.capabilities().supports_images);
assert_eq!(cfg.file.timeouts.response_generation, 15);
}
+387
View File
@@ -0,0 +1,387 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>ChatGPT fixture</title>
<!--
Local fake of the ChatGPT web UI used by the browser end-to-end tests.
The Rust driver opens this file with a file:// URL and drives it over CDP:
it inserts text through Input.insertText / execCommand / key events, clicks
button[data-testid=send-button], then polls the DOM until the answer settles.
DOM contract (selectors live in the test provider TOML):
- editor div#prompt-textarea.ProseMirror[contenteditable=true]
- send button button[data-testid=send-button] (disabled while empty)
- answer containers div[data-message-author-role=assistant] (one per answer)
the text lives in div.answer-body inside it, and the
toolbar (with the copy button) is a sibling of the
answer node, exactly like the real page: the capture
must not pick the word "Copy" up as answer text
- answer toolbar div.message-toolbar > button[data-testid=copy-button]
copies the answer as Markdown, like the real UI
- streaming indicator button[data-testid=stop-button] (only while generating)
- file input input[type=file] (hidden)
- attachment chips div[data-testid=attachment-chip]
- new chat a[data-testid=create-new-chat-button]
- login mode form[action="/auth/login"]
- captcha mode div#challenge-form
- error mode div[role=alert]
Query parameters:
conv=1 after the first send, set location.hash to #c=<uuid> and keep it
login=1 render the sign-in form instead of the editor
captcha=1 render a bot challenge
error=1 render an error banner; sending produces no answer
ratelimit=1 the page text contains "too many requests"; sending produces no answer
empty=1 sending shows the stop button but never writes any text
delay=<ms> minimum generation time (default 1200)
loading=1 the editor only appears after 3 seconds
rewrite=1 the answer is rewritten mid-stream before settling
rich=1 the answer is rendered as rich HTML (heading, list, code
block, table) and the copy button copies it as Markdown
-->
<style>
body { font: 15px/1.5 system-ui, sans-serif; margin: 0; background: #212121; color: #ececec; }
header { padding: 10px 16px; border-bottom: 1px solid #3a3a3a; display: flex; gap: 12px; }
#messages { padding: 16px; height: 320px; overflow: auto; }
.msg { max-width: 70%; margin: 8px 0; padding: 8px 12px; border-radius: 10px; white-space: pre-wrap; }
[data-message-author-role="user"] { background: #2f2f2f; margin-left: auto; }
[data-message-author-role="assistant"] { background: #303030; }
footer { padding: 12px 16px; border-top: 1px solid #3a3a3a; }
#prompt-textarea { min-height: 48px; padding: 10px; background: #303030; border-radius: 12px;
border: 1px solid #4a4a4a; outline: none; }
#prompt-textarea:focus { border-color: #7aa2f7; }
button { font: inherit; padding: 6px 12px; border-radius: 8px; border: 1px solid #4a4a4a;
background: #303030; color: #ececec; cursor: pointer; }
button[disabled] { opacity: .45; cursor: default; }
#attachments { display: flex; gap: 6px; flex-wrap: wrap; margin-bottom: 8px; }
[data-testid="attachment-chip"] { background: #3a3a3a; border-radius: 6px; padding: 2px 8px; font-size: 13px; }
.message-toolbar { display: flex; gap: 6px; margin: 2px 0 10px; }
.message-toolbar button { font-size: 12px; padding: 2px 8px; }
.answer-body table { border-collapse: collapse; margin: 6px 0; }
.answer-body th, .answer-body td { border: 1px solid #555; padding: 2px 8px; }
.answer-body pre { background: #1b1b1b; padding: 8px; border-radius: 6px; overflow: auto; }
#toolbar { display: flex; justify-content: space-between; align-items: center; margin-top: 8px; }
.banner { margin: 12px 16px; padding: 10px 12px; border-radius: 8px; background: #4a2020; }
#challenge-form { margin: 24px 16px; padding: 20px; background: #3a3a3a; border-radius: 10px; }
form { margin: 40px auto; width: 320px; padding: 24px; background: #2a2a2a; border-radius: 12px; }
input[type="email"], input[type="password"] { width: 100%; padding: 8px; margin: 4px 0 12px;
background: #303030; border: 1px solid #4a4a4a;
border-radius: 8px; color: inherit; }
</style>
</head>
<body>
<div id="app"></div>
<script>
(function () {
"use strict";
var params = new URLSearchParams(location.search);
var mode = function (name) { return params.get(name) === "1"; };
var state = {
generating: false,
token: 0,
attachments: [],
conversation: null,
answerCount: 0
};
// A reopened conversation keeps its identifier so that the driver can prove
// that it is continuing the same web thread.
var hashMatch = /#c=([0-9a-zA-Z-]+)/.exec(location.hash);
if (hashMatch) { state.conversation = hashMatch[1]; }
function answer(prompt) {
var trimmed = prompt.trim();
var text;
if (trimmed.toLowerCase().indexOf("ok") !== -1) {
text = "ok";
} else {
text = "Echo: " + trimmed + " — this answer was generated by the local ChatGPT fixture page.";
}
if (state.attachments.length) {
text = text + " (attached: " + state.attachments.join(", ") + ")";
}
return text;
}
function escapeHtml(value) {
return String(value)
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
// The rich rendering of one answer: a heading, a list, a fenced code block
// and a table, i.e. exactly the constructs a plain innerText capture loses.
function richHtml(prompt) {
return (
"<h2>Answer</h2>" +
"<p>Echo: <strong>" + escapeHtml(prompt.trim()) + "</strong></p>" +
"<ul><li>first point</li><li>second point</li></ul>" +
'<pre><code class="language-rust">fn main() {\n println!("hi");\n}</code></pre>' +
"<table><thead><tr><th>Name</th><th>Value</th></tr></thead>" +
"<tbody><tr><td>alpha</td><td>1</td></tr><tr><td>beta</td><td>2</td></tr></tbody></table>"
);
}
// What the real copy button puts in the clipboard: the Markdown rendering.
function richMarkdown(prompt) {
return (
"## Answer\n\n" +
"Echo: **" + prompt.trim() + "**\n\n" +
"- first point\n" +
"- second point\n\n" +
"```rust\n" +
'fn main() {\n println!("hi");\n}\n' +
"```\n\n" +
"| Name | Value |\n" +
"| --- | --- |\n" +
"| alpha | 1 |\n" +
"| beta | 2 |\n"
);
}
// Give one assistant message the toolbar the real page shows next to it.
// The copy button is a sibling of the answer node, not a child, so a capture
// that reads the whole turn would pick the word "Copy" up as answer text.
function attachToolbar(turn, prompt) {
var toolbar = document.createElement("div");
toolbar.className = "message-toolbar";
var copy = document.createElement("button");
copy.setAttribute("data-testid", "copy-button");
copy.setAttribute("aria-label", "Copy");
copy.textContent = "Copy";
copy.addEventListener("click", function (event) {
event.preventDefault();
var markdown = richMarkdown(prompt);
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(markdown)["catch"](function () {});
}
});
toolbar.appendChild(copy);
turn.appendChild(toolbar);
}
// Give one assistant message the toolbar the real page shows next to it.
// The copy button is a sibling of the answer node, not a child, so a capture
// that reads the whole turn would pick the word "Copy" up as answer text.
function attachToolbar(turn, prompt) {
var toolbar = document.createElement("div");
toolbar.className = "message-toolbar";
var copy = document.createElement("button");
copy.setAttribute("data-testid", "copy-button");
copy.setAttribute("aria-label", "Copy");
copy.textContent = "Copy";
copy.addEventListener("click", function (event) {
event.preventDefault();
var markdown = richMarkdown(prompt);
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(markdown)["catch"](function () {});
}
});
toolbar.appendChild(copy);
turn.appendChild(toolbar);
}
function render() {
var app = document.getElementById("app");
if (mode("login")) {
app.innerHTML =
'<form action="/auth/login" method="post">' +
'<h2>Sign in (fixture)</h2>' +
'<label>Email<input type="email" name="email" /></label>' +
'<label>Password<input type="password" name="password" /></label>' +
'<button type="submit">Continue</button>' +
'</form>';
return;
}
var banner = "";
if (mode("error")) {
banner = '<div role="alert">Something went wrong. Please try again.</div>';
}
if (mode("ratelimit")) {
banner += '<div class="banner">Too many requests right now, please try again later.</div>';
}
var challenge = mode("captcha") ? '<div id="challenge-form">Verify you are human</div>' : "";
app.innerHTML =
'<header><a data-testid="create-new-chat-button" href="#">New chat</a><span>ChatGPT fixture</span></header>' +
banner + challenge +
'<div id="messages"></div>' +
'<footer>' +
'<div id="attachments"></div>' +
'<div id="prompt-textarea" class="ProseMirror" contenteditable="true"></div>' +
'<div id="toolbar">' +
'<input type="file" multiple hidden />' +
'<button data-testid="send-button" disabled>Send</button>' +
'</div>' +
'</footer>';
var editor = document.getElementById("prompt-textarea");
var send = document.querySelector("[data-testid='send-button']");
var fileInput = document.querySelector("input[type='file']");
function refresh() { send.disabled = editor.innerText.trim().length === 0 || state.generating; }
editor.addEventListener("input", refresh);
editor.addEventListener("keyup", refresh);
editor.addEventListener("blur", refresh);
window.setInterval(refresh, 100);
editor.addEventListener("keydown", function (event) {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
submit();
}
});
send.addEventListener("click", function (event) { event.preventDefault(); submit(); });
fileInput.addEventListener("change", function () {
var strip = document.getElementById("attachments");
for (var i = 0; i < fileInput.files.length; i += 1) {
var file = fileInput.files[i];
state.attachments.push(file.name);
var chip = document.createElement("div");
chip.setAttribute("data-testid", "attachment-chip");
chip.textContent = file.name;
strip.appendChild(chip);
}
});
var newChat = document.querySelector("[data-testid='create-new-chat-button']");
newChat.addEventListener("click", function (event) {
event.preventDefault();
state.token += 1;
state.generating = false;
state.answerCount = 0;
state.attachments = [];
document.getElementById("messages").innerHTML = "";
document.getElementById("attachments").innerHTML = "";
if (state.conversation) { history.replaceState(null, "", location.pathname + location.search); }
state.conversation = null;
editor.innerText = "";
refresh();
});
refresh();
}
function stopButton() { return document.querySelector("[data-testid='stop-button']"); }
function setStopButton(visible) {
var existing = stopButton();
if (visible && !existing) {
var button = document.createElement("button");
button.setAttribute("data-testid", "stop-button");
button.textContent = "Stop";
button.addEventListener("click", function () { state.token += 1; state.generating = false; setStopButton(false); });
document.getElementById("toolbar").appendChild(button);
} else if (!visible && existing) {
existing.remove();
}
}
function submit() {
var editor = document.getElementById("prompt-textarea");
var prompt = editor.innerText;
if (!prompt.trim() || state.generating) { return; }
var messages = document.getElementById("messages");
var user = document.createElement("div");
user.setAttribute("data-message-author-role", "user");
user.textContent = prompt;
messages.appendChild(user);
editor.innerText = "";
editor.dispatchEvent(new Event("input", { bubbles: true }));
if (mode("conv") && !state.conversation) {
state.conversation = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now()));
location.hash = "#c=" + state.conversation;
}
if (mode("login") || mode("captcha")) { return; }
if (mode("error") || mode("ratelimit")) { return; }
state.generating = true;
state.token += 1;
var token = state.token;
setStopButton(true);
// The page wraps one exchange in a "turn" element, like the real UI: the
// answer node holds the text only, and the toolbar is its sibling.
var turn = document.createElement("div");
turn.className = "turn";
turn.setAttribute("data-testid", "conversation-turn");
var node = document.createElement("div");
node.setAttribute("data-message-author-role", "assistant");
var body = document.createElement("div");
body.className = "answer-body";
node.appendChild(body);
turn.appendChild(node);
messages.appendChild(turn);
state.answerCount += 1;
if (mode("rich")) {
attachToolbar(turn, prompt);
}
var full = answer(prompt);
var draft = mode("rewrite") ? "Draft answer being rewritten" : null;
var delay = parseInt(params.get("delay") || "1200", 10);
var steps = 22;
var written = "";
if (mode("empty")) {
window.setTimeout(function () {
if (token !== state.token) { return; }
state.generating = false;
setStopButton(false);
}, delay);
return;
}
var index = 0;
var timer = window.setInterval(function () {
if (token !== state.token) { window.clearInterval(timer); return; }
index += 1;
var target = full;
if (draft && index * 2 < steps) { target = draft; }
else if (draft && index === Math.floor(steps / 2) + 1) { written = ""; }
var chars = Math.ceil(target.length * (index / steps));
written = target.slice(0, chars);
body.innerText = written;
if (index >= steps) {
window.clearInterval(timer);
// The settled answer is rendered: the rich modes turn it into HTML,
// which is what a plain text capture cannot represent.
try {
if (mode("rich")) {
body.innerHTML = richHtml(prompt);
} else {
body.innerText = full;
}
} catch (e) {
body.innerText = full;
}
state.generating = false;
setStopButton(false);
var send = document.querySelector("[data-testid='send-button']");
if (send) { send.disabled = document.getElementById("prompt-textarea").innerText.trim().length === 0; }
}
}, Math.max(20, Math.floor(delay / steps)));
}
window.fixtureState = state;
if (mode("loading")) {
window.setTimeout(render, 3000);
} else {
render();
}
})();
</script>
</body>
</html>