141 lines
5.3 KiB
PowerShell
141 lines
5.3 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
End to end demonstration of llm-gateway.
|
|
|
|
.DESCRIPTION
|
|
Starts the API server on a spare port, runs a full cycle against it
|
|
(health, models, non-streaming completion, streaming completion, provider
|
|
status, validation, configuration reload) and stops the server.
|
|
|
|
Without -Mock the server talks to a real browser: log in first with
|
|
"llm-gateway login chatgpt" and make sure the provider is configured.
|
|
With -Mock everything runs in memory, no browser and no login needed.
|
|
|
|
.EXAMPLE
|
|
./demo.ps1 -Mock
|
|
./demo.ps1 -Port 8099
|
|
#>
|
|
param(
|
|
[switch]$Mock,
|
|
[int]$Port = 8099,
|
|
[string]$Provider = 'chatgpt'
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$root = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
Push-Location $root
|
|
|
|
$stateDir = Join-Path $root '.demo-state'
|
|
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
|
$env:LLM_GATEWAY_HOME = $stateDir
|
|
|
|
$backend = if ($Mock) { 'mock' } else { 'browser' }
|
|
$base = "http://127.0.0.1:$Port"
|
|
|
|
Write-Host "== building ==" -ForegroundColor Cyan
|
|
cargo build --quiet
|
|
if ($LASTEXITCODE -ne 0) { throw 'cargo build failed' }
|
|
$exe = Join-Path $root 'target/debug/llm-gateway.exe'
|
|
|
|
Write-Host "== starting llm-gateway (backend: $backend, port: $Port) ==" -ForegroundColor Cyan
|
|
$server = Start-Process -FilePath $exe -ArgumentList @('serve', '--port', "$Port", '--backend', $backend) -PassThru -NoNewWindow
|
|
|
|
try {
|
|
$health = $null
|
|
foreach ($attempt in 1..60) {
|
|
try {
|
|
$health = Invoke-RestMethod -Uri "$base/health" -TimeoutSec 2
|
|
break
|
|
} catch {
|
|
Start-Sleep -Milliseconds 500
|
|
}
|
|
}
|
|
if (-not $health) { throw "the server did not become healthy on $base" }
|
|
|
|
Write-Host "
|
|
== /health ==" -ForegroundColor Green
|
|
$health | ConvertTo-Json -Depth 6
|
|
|
|
Write-Host "
|
|
== /v1/models ==" -ForegroundColor Green
|
|
(Invoke-RestMethod -Uri "$base/v1/models").data | ForEach-Object { " " + $_.id }
|
|
|
|
$payload = @{
|
|
model = $Provider
|
|
messages = @(@{ role = 'user'; content = 'Say ok and nothing else.' })
|
|
} | ConvertTo-Json -Depth 5
|
|
|
|
Write-Host "
|
|
== POST /v1/chat/completions (non-streaming) ==" -ForegroundColor Green
|
|
$answer = Invoke-RestMethod -Uri "$base/v1/chat/completions" -Method Post -Body $payload -ContentType 'application/json'
|
|
" id : " + $answer.id
|
|
" model : " + $answer.model
|
|
" answer : " + $answer.choices[0].message.content
|
|
" usage : prompt " + $answer.usage.prompt_tokens + ", completion " + $answer.usage.completion_tokens
|
|
|
|
Write-Host "
|
|
== POST /v1/chat/completions (two completions) ==" -ForegroundColor Green
|
|
$variantsPayload = @{
|
|
model = $Provider
|
|
n = 2
|
|
messages = @(@{ role = 'user'; content = 'Answer with one word.' })
|
|
} | ConvertTo-Json -Depth 5
|
|
$variants = Invoke-RestMethod -Uri "$base/v1/chat/completions" -Method Post -Body $variantsPayload -ContentType 'application/json'
|
|
" choices : " + $variants.choices.Count
|
|
$variants.choices | ForEach-Object { " [" + $_.index + "] " + $_.message.content }
|
|
if ($variants.x_gateway_warnings) { " warning : " + $variants.x_gateway_warnings[0] }
|
|
|
|
$accounts = ($health.providers | Where-Object { $_.name -eq $Provider }).accounts
|
|
if ($accounts) {
|
|
Write-Host "
|
|
== accounts of $Provider ==" -ForegroundColor Green
|
|
$accounts | ForEach-Object {
|
|
" " + $_.id + " " + $_.email + $(if ($_.default) { " (default)" } else { "" })
|
|
}
|
|
$account = $accounts[0].id
|
|
$accountPayload = @{
|
|
model = "$Provider@$account"
|
|
messages = @(@{ role = 'user'; content = 'Say ok and nothing else.' })
|
|
} | ConvertTo-Json -Depth 5
|
|
$accounted = Invoke-RestMethod -Uri "$base/v1/chat/completions" -Method Post -Body $accountPayload -ContentType 'application/json'
|
|
" model $Provider@$account answered: " + $accounted.choices[0].message.content
|
|
}
|
|
|
|
Write-Host "
|
|
== POST /v1/chat/completions (streaming) ==" -ForegroundColor Green
|
|
$streamPayload = @{
|
|
model = $Provider
|
|
stream = $true
|
|
messages = @(@{ role = 'user'; content = 'Count from one to five.' })
|
|
} | ConvertTo-Json -Depth 5
|
|
$streamed = Invoke-WebRequest -Uri "$base/v1/chat/completions" -Method Post -Body $streamPayload -ContentType 'application/json'
|
|
$streamed.Content -split "`n" | Select-Object -First 12 | ForEach-Object { " " + $_ }
|
|
" ... ($((($streamed.Content -split "`n") | Measure-Object).Count) lines of SSE)"
|
|
|
|
Write-Host "
|
|
== GET /v1/providers/$Provider/status ==" -ForegroundColor Green
|
|
Invoke-RestMethod -Uri "$base/v1/providers/$Provider/status" | ConvertTo-Json -Depth 5
|
|
|
|
Write-Host "
|
|
== POST /v1/providers/$Provider/validate ==" -ForegroundColor Green
|
|
Invoke-RestMethod -Uri "$base/v1/providers/$Provider/validate" -Method Post | ConvertTo-Json -Depth 5
|
|
|
|
Write-Host "
|
|
== POST /v1/admin/reload ==" -ForegroundColor Green
|
|
Invoke-RestMethod -Uri "$base/v1/admin/reload" -Method Post | ConvertTo-Json -Depth 5
|
|
|
|
Write-Host "
|
|
== dashboard ==" -ForegroundColor Green
|
|
" $base/dashboard"
|
|
" state directory: $stateDir"
|
|
}
|
|
finally {
|
|
Write-Host "
|
|
== stopping the server ==" -ForegroundColor Cyan
|
|
if ($server -and -not $server.HasExited) {
|
|
Stop-Process -Id $server.Id -Force
|
|
$server.WaitForExit(5000) | Out-Null
|
|
}
|
|
Pop-Location
|
|
}
|