Projets/agent-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 4 Releases 34 Wiki Activity
Files
4c7866a25ad993b8de271df20916be9d766028fa
agent-manager/src/download.rs
T

697 lines
24 KiB
Rust
Raw Blame History

//! Downloading (with checksum verification), GitHub Releases resolution,
//! archive extraction and executable discovery.
use anyhow::{anyhow, bail, Context, Result};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::io::{IsTerminal, Read, Write};
use std::path::{Path, PathBuf};
pub const USER_AGENT: &str = concat!("agent-manager/", env!("CARGO_PKG_VERSION"));
// ---------------------------------------------------------------------------
// URL validation + download
// ---------------------------------------------------------------------------
/// Only http, https and file schemes are allowed.
pub fn validate_url(url: &str) -> Result<()> {
let lower = url.to_lowercase();
let ok = lower.starts_with("https://")
|| lower.starts_with("http://")
|| lower.starts_with("file://");
if !ok {
bail!(
"refusing URL with unsupported scheme (only http, https and file are allowed): {url}"
);
}
Ok(())
}
/// Download a URL to dest, verifying the optional sha256 checksum.
pub fn download(url: &str, dest: &Path, expected_sha256: Option<&str>, progress: bool) -> Result<()> {
validate_url(url)?;
if let Some(local) = url.strip_prefix("file://") {
let src = PathBuf::from(local);
std::fs::copy(&src, dest)
.with_context(|| format!("cannot copy {} to {}", src.display(), dest.display()))?;
verify_sha256(dest, expected_sha256)?;
return Ok(());
}
let resp = ureq::get(url)
.set("User-Agent", USER_AGENT)
.call()
.map_err(|e| {
match e {
ureq::Error::Status(code, r) => anyhow!(
"download failed for {url}: HTTP {code} {}",
r.status_text()
),
other => anyhow!("download failed for {url}: {other}"),
}
})?;
let total: Option<u64> = resp.header("content-length").and_then(|v| v.parse().ok());
let mut reader = resp.into_reader();
let mut file = std::fs::File::create(dest)
.with_context(|| format!("cannot create {}", dest.display()))?;
let mut buf = [0u8; 65536];
let mut done: u64 = 0;
let mut last_shown: u64 = 0;
let show = progress && std::io::stderr().is_terminal();
loop {
let n = reader
.read(&mut buf)
.with_context(|| format!("read error while downloading {url}"))?;
if n == 0 {
break;
}
file.write_all(&buf[..n])?;
done += n as u64;
if show && (done - last_shown >= 8 * 1024 * 1024 || done == total.unwrap_or(u64::MAX)) {
last_shown = done;
match total {
Some(t) => eprint!(
"\r downloading {:.1} / {:.1} MB",
done as f64 / 1e6,
t as f64 / 1e6
),
None => eprint!("\r downloaded {:.1} MB", done as f64 / 1e6),
}
std::io::stderr().flush().ok();
}
}
if show {
eprintln!();
}
verify_sha256(dest, expected_sha256)?;
Ok(())
}
fn verify_sha256(path: &Path, expected: Option<&str>) -> Result<()> {
let Some(exp) = expected else {
return Ok(());
};
let hex = exp
.strip_prefix("sha256:")
.or_else(|| exp.strip_prefix("SHA256:"))
.unwrap_or(exp)
.trim();
if hex.len() != 64 || !hex.chars().all(|c| c.is_ascii_hexdigit()) {
bail!("invalid sha256 checksum in config: {exp}");
}
let mut f = std::fs::File::open(path)?;
let mut hasher = Sha256::new();
std::io::copy(&mut f, &mut hasher)?;
let actual = to_hex(&hasher.finalize());
if !actual.eq_ignore_ascii_case(hex) {
bail!(
"checksum mismatch for {}: expected {hex}, got {actual}",
path.display()
);
}
Ok(())
}
pub fn to_hex(bytes: &[u8]) -> String {
let mut s = String::with_capacity(bytes.len() * 2);
for b in bytes {
s.push_str(&format!("{b:02x}"));
}
s
}
// ---------------------------------------------------------------------------
// GitHub Releases
// ---------------------------------------------------------------------------
#[derive(Debug, Deserialize)]
pub struct GhAsset {
pub name: String,
pub browser_download_url: String,
#[serde(default)]
pub size: u64,
}
#[derive(Debug, Deserialize)]
pub struct GhRelease {
pub tag_name: String,
#[serde(default)]
pub assets: Vec<GhAsset>,
}
/// Authorization header for api.github.com when GITHUB_TOKEN or GH_TOKEN
/// is set (raises the rate limit from 60 to 5000 requests/hour).
fn github_auth_header() -> Option<String> {
let token = std::env::var("GITHUB_TOKEN")
.or_else(|_| std::env::var("GH_TOKEN"))
.ok()
.filter(|t| !t.trim().is_empty())?;
Some(format!("Bearer {}", token.trim()))
}
pub fn latest_release(repo: &str) -> Result<GhRelease> {
latest_release_with_base(repo, None)
}
/// Resolve the latest release, with an optional API base URL override
/// (e.g. https://git.dracodev.net/api/v1 for a Gitea instance).
pub fn latest_release_with_base(repo: &str, api_base: Option<&str>) -> Result<GhRelease> {
let github = api_base.is_none() || api_base == Some("https://api.github.com");
match latest_release_api(repo, api_base) {
Ok(release) => Ok(release),
Err(e) if github && format!("{e:#}").contains("rate limit") => {
// The GitHub API is rate limited (60 unauthenticated
// requests/hour): fall back to the GitHub web pages, which are
// not limited. (Gitea has no equivalent web fallback.)
latest_release_web(repo)
}
Err(e) => Err(e),
}
}
fn latest_release_api(repo: &str, api_base: Option<&str>) -> Result<GhRelease> {
let base = api_base.unwrap_or("https://api.github.com");
let url = format!("{base}/repos/{repo}/releases/latest");
let mut req = ureq::get(&url).set("User-Agent", USER_AGENT);
if let Some(auth) = github_auth_header() {
req = req.set("Authorization", &auth);
}
let resp = req.call().map_err(|e| match e {
ureq::Error::Status(404, _) => {
anyhow!("no GitHub releases found for repository '{repo}'")
}
ureq::Error::Status(403, _) => anyhow!(
"GitHub API request failed for '{repo}': rate limit exceeded — set the GITHUB_TOKEN environment variable to raise the limit, or retry in a few minutes"
),
other => anyhow!("GitHub API request failed for '{repo}': {other}"),
})?;
let text = resp
.into_string()
.context("reading GitHub API response")?;
serde_json::from_str::<GhRelease>(&text).context("parsing GitHub API response")
}
/// Discover the latest release without the API: the /releases/latest page
/// redirects to the tagged page (tag from the final URL), then the
/// expanded_assets endpoint lists every asset. These web pages are not
/// subject to the API rate limit.
fn latest_release_web(repo: &str) -> Result<GhRelease> {
let latest = format!("https://github.com/{repo}/releases/latest");
let resp = ureq::get(&latest)
.set("User-Agent", USER_AGENT)
.call()
.map_err(|e| anyhow!("cannot reach GitHub releases page for '{repo}': {e}"))?;
let final_url = resp.get_url().to_string();
let tag = final_url
.rsplit('/')
.next()
.map(|s| s.to_string())
.filter(|s| !s.is_empty() && final_url.contains("/releases/tag/"))
.ok_or_else(|| {
anyhow!("no GitHub releases found for repository '{repo}' (no tagged release page)")
})?;
let assets_url = format!("https://github.com/{repo}/releases/expanded_assets/{tag}");
let resp = ureq::get(&assets_url)
.set("User-Agent", USER_AGENT)
.call()
.map_err(|e| anyhow!("cannot list GitHub release assets for '{repo}': {e}"))?;
let html = resp
.into_string()
.context("reading GitHub expanded_assets response")?;
let mut assets = parse_expanded_assets(&html, repo, &tag);
if assets.is_empty() {
anyhow::bail!(
"GitHub release '{repo}@{tag}' has no downloadable assets (web fallback)"
);
}
assets.sort_by(|a, b| a.name.cmp(&b.name));
assets.dedup_by(|a, b| a.name == b.name);
Ok(GhRelease {
tag_name: tag,
assets,
})
}
/// Extract the asset links from a GitHub expanded_assets HTML fragment.
fn parse_expanded_assets(html: &str, repo: &str, tag: &str) -> Vec<GhAsset> {
let marker = format!("/{repo}/releases/download/");
let mut assets = Vec::new();
let mut rest = html;
while let Some(pos) = rest.find(&marker) {
let start = pos + marker.len();
let end = rest[start..]
.find(['"', '\'', '<'])
.map(|i| start + i)
.unwrap_or(rest.len());
let raw = &rest[start..end];
// The href reads "download/<tag>/<asset-name>": keep the name only.
let raw = raw.strip_prefix(&format!("{tag}/")).unwrap_or(raw);
// GitHub sometimes hides zero-width characters inside hrefs.
let name = raw.replace('\u{200b}', "");
let name = decode_html_entities(&name);
if !name.is_empty() {
assets.push(GhAsset {
name: name.clone(),
browser_download_url: format!(
"https://github.com/{repo}/releases/download/{tag}/{name}"
),
size: 0,
});
}
rest = &rest[end..];
}
assets
}
fn decode_html_entities(s: &str) -> String {
s.replace("&amp;", "&")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&#34;", "\"")
.replace("&#32;", " ")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&nbsp;", " ")
}
pub struct PickedAsset {
pub name: String,
pub url: String,
pub tag: String,
}
/// Choose the release asset that best matches the current platform and arch.
pub fn pick_asset(
release: &GhRelease,
want: &[String],
exclude: &[String],
binary_hint: Option<&str>,
) -> Option<PickedAsset> {
let platform = if cfg!(windows) {
"windows"
} else if cfg!(target_os = "macos") {
"macos"
} else {
"linux"
};
let arch = if cfg!(target_arch = "aarch64") {
"aarch64"
} else {
"x86_64"
};
let mut best: Option<(u32, usize, &GhAsset)> = None;
for asset in &release.assets {
let lower = asset.name.to_lowercase();
// Skip checksums, signatures and desktop packages.
if lower.ends_with(".sha256")
|| lower.ends_with(".sha512")
|| lower.ends_with(".sig")
|| lower.ends_with(".asc")
|| lower.ends_with(".pem")
|| lower.contains("checksums")
|| lower.ends_with(".dmg")
|| lower.ends_with(".deb")
|| lower.ends_with(".rpm")
|| lower.ends_with(".msi")
|| lower.ends_with(".apk")
{
continue;
}
if !want.iter().all(|w| lower.contains(&w.to_lowercase())) {
continue;
}
if exclude.iter().any(|w| lower.contains(&w.to_lowercase())) {
continue;
}
let score = match platform_score(&lower, platform, arch) {
Some(s) => s,
None => continue,
};
let replace = match best {
None => true,
Some((s, _, _)) => score > s,
};
if replace {
best = Some((score, asset.name.len(), asset));
}
}
best.map(|(_, _, a)| {
let _ = binary_hint;
PickedAsset {
name: a.name.clone(),
url: a.browser_download_url.clone(),
tag: release.tag_name.clone(),
}
})
}
fn platform_score(lower: &str, platform: &str, arch: &str) -> Option<u32> {
let plat_ok = match platform {
"windows" => {
lower.contains("windows")
|| lower.ends_with(".exe")
|| lower.contains("win64")
|| lower.contains("win32")
|| lower.contains("msvc")
|| lower.contains("mingw")
|| lower.contains("pc-windows")
}
"macos" => {
lower.contains("darwin")
|| lower.contains("macos")
|| lower.contains("apple")
|| lower.contains("osx")
}
_ => {
lower.contains("linux")
|| lower.contains("unknown-linux")
|| lower.contains("manylinux")
}
};
if !plat_ok {
return None;
}
let arch_ok = match arch {
"aarch64" => lower.contains("aarch64") || lower.contains("arm64") || lower.contains("armv8"),
_ => {
lower.contains("x86_64")
|| lower.contains("amd64")
|| lower.contains("x64")
|| lower.contains("i686")
}
};
if !arch_ok && !lower.contains("universal") {
return None;
}
let mut s = 1u32;
if lower.contains("universal") {
s += 4;
}
if arch == "aarch64" && lower.contains("arm64") {
s += 3;
}
if arch == "x86_64" && (lower.contains("x86_64") || lower.contains("amd64")) {
s += 3;
}
if lower.contains("musl") {
s += 1;
}
if lower.contains("static") || lower.contains("portable") {
s += 1;
}
// Prefer plain archives (extracted to a single executable) over
// installer/setup programs, which cannot be used directly.
if lower.ends_with(".zip") || lower.ends_with(".tar.gz") || lower.ends_with(".tgz") {
s += 2;
}
if lower.contains("setup") || lower.contains("installer") || lower.contains("-install") {
s = s.saturating_sub(3);
}
if lower.contains("armv7")
|| lower.contains("armhf")
|| lower.contains("armv6")
|| lower.contains("i686")
|| lower.contains("i386")
{
s = s.saturating_sub(2);
}
Some(s)
}
// ---------------------------------------------------------------------------
// Archives and executables
// ---------------------------------------------------------------------------
/// What kind of archive a file is, sniffed from its content.
enum ArchiveKind {
Zip,
TarGz,
Tar,
Gz,
Plain,
}
/// Detect the archive type from the file's magic bytes, falling back to the
/// file name. Downloads are stored under a generic name ("download"), so
/// content sniffing is the reliable way to recognize a zip/tar.gz asset.
fn sniff_archive(path: &Path) -> Result<ArchiveKind> {
let mut head = [0u8; 512];
let n = std::fs::File::open(path)
.and_then(|mut f| std::io::Read::read(&mut f, &mut head))
.unwrap_or(0);
if n >= 4 && &head[0..4] == b"PK\x03\x04" {
return Ok(ArchiveKind::Zip);
}
if n >= 2 && head[0] == 0x1f && head[1] == 0x8b {
// gzip: peek at the decompressed stream to see whether it is a tar.
let f = std::fs::File::open(path)?;
let mut dec = flate2::read::GzDecoder::new(f);
let mut inner = [0u8; 512];
let m = std::io::Read::read(&mut dec, &mut inner).unwrap_or(0);
if m >= 262 && &inner[257..262] == b"ustar" {
return Ok(ArchiveKind::TarGz);
}
return Ok(ArchiveKind::Gz);
}
if n >= 262 && &head[257..262] == b"ustar" {
return Ok(ArchiveKind::Tar);
}
// Content is not an archive: keep the name-based detection for files
// that still carry a real archive extension.
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_lowercase();
if name.ends_with(".zip") {
return Ok(ArchiveKind::Zip);
}
if name.ends_with(".tar.gz") || name.ends_with(".tgz") {
return Ok(ArchiveKind::TarGz);
}
if name.ends_with(".tar") {
return Ok(ArchiveKind::Tar);
}
if name.ends_with(".gz") {
return Ok(ArchiveKind::Gz);
}
Ok(ArchiveKind::Plain)
}
/// Extract zip / tar.gz / tgz / tar / gz archives, detected by content.
/// Returns false when the file is not a recognized archive (caller treats
/// it as a plain binary).
pub fn extract_archive(path: &Path, dest: &Path) -> Result<bool> {
std::fs::create_dir_all(dest)?;
match sniff_archive(path)? {
ArchiveKind::Zip => {
extract_zip(path, dest)?;
Ok(true)
}
ArchiveKind::TarGz => {
let f = std::fs::File::open(path)?;
let dec = flate2::read::GzDecoder::new(f);
let mut ar = tar::Archive::new(dec);
ar.unpack(dest)
.with_context(|| format!("extracting {}", path.display()))?;
Ok(true)
}
ArchiveKind::Tar => {
let f = std::fs::File::open(path)?;
let mut ar = tar::Archive::new(f);
ar.unpack(dest)
.with_context(|| format!("extracting {}", path.display()))?;
Ok(true)
}
ArchiveKind::Gz => {
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_lowercase();
let inner = name.trim_end_matches(".gz").to_string();
let out = dest.join(inner);
let f = std::fs::File::open(path)?;
let mut dec = flate2::read::GzDecoder::new(f);
let mut o = std::fs::File::create(&out)?;
std::io::copy(&mut dec, &mut o)?;
Ok(true)
}
ArchiveKind::Plain => Ok(false),
}
}
fn extract_zip(path: &Path, dest: &Path) -> Result<()> {
let f = std::fs::File::open(path)?;
let mut zip = zip::ZipArchive::new(f)
.with_context(|| format!("opening zip archive {}", path.display()))?;
for i in 0..zip.len() {
let mut entry = zip.by_index(i)?;
let Some(enclosed) = entry.enclosed_name() else {
bail!("zip archive {} contains an unsafe path", path.display());
};
let out_path = dest.join(enclosed);
if entry.is_dir() {
std::fs::create_dir_all(&out_path)?;
} else {
if let Some(parent) = out_path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut out = std::fs::File::create(&out_path)?;
std::io::copy(&mut entry, &mut out)?;
}
}
Ok(())
}
/// Find the executable in a directory tree: exact name match first, then a
/// single candidate file, then a single executable file (Unix).
pub fn find_executable(dir: &Path, expected: Option<&str>) -> Result<PathBuf> {
let mut files = Vec::new();
collect_files(dir, &mut files);
if files.is_empty() {
bail!("no files found in {}", dir.display());
}
if let Some(exp) = expected {
for f in &files {
let name = f
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_string();
let stripped = name.strip_suffix(".exe").unwrap_or(&name);
if stripped.eq_ignore_ascii_case(exp) {
return Ok(f.clone());
}
}
}
if files.len() == 1 {
return Ok(files.remove(0));
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let exec: Vec<&PathBuf> = files
.iter()
.filter(|f| {
std::fs::metadata(f)
.map(|m| m.permissions().mode() & 0o111 != 0)
.unwrap_or(false)
})
.collect();
if exec.len() == 1 {
return Ok(exec[0].clone());
}
}
let names: Vec<String> = files
.iter()
.map(|f| format!(" {}", f.display()))
.collect();
bail!(
"cannot determine which file is the executable in {}; candidates:
{}",
dir.display(),
names.join("
")
)
}
fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) {
if let Ok(rd) = std::fs::read_dir(dir) {
let mut entries: Vec<PathBuf> = rd.flatten().map(|e| e.path()).collect();
entries.sort();
for p in entries {
if p.is_dir() {
collect_files(&p, out);
} else {
out.push(p);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn url_validation() {
assert!(validate_url("https://example.com/x").is_ok());
assert!(validate_url("http://example.com/x").is_ok());
assert!(validate_url("file:///tmp/x").is_ok());
assert!(validate_url("ftp://example.com/x").is_err());
assert!(validate_url("javascript:alert(1)").is_err());
}
#[test]
fn parses_expanded_assets_fragment() {
let html = r#"<div><a href="/owner/repo/releases/download/v1.2.3/tool-windows-x86_64.exe">x</a>
<a href="/owner/repo/releases/download/v1.2.3/tool-windows-x86_64&#32;copy.zip">y</a>
<span>ignore me</span>
<a href="/owner/repo/releases/download/v1.2.3/SHA256SUMS">z</a></div>"#;
let assets = parse_expanded_assets(html, "owner/repo", "v1.2.3");
let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect();
assert!(names.contains(&"tool-windows-x86_64.exe"), "{names:?}");
assert!(names.contains(&"tool-windows-x86_64 copy.zip"), "{names:?}");
assert!(names.contains(&"SHA256SUMS"), "{names:?}");
assert_eq!(assets.len(), 3);
assert_eq!(
assets[0].browser_download_url,
"https://github.com/owner/repo/releases/download/v1.2.3/tool-windows-x86_64.exe"
);
}
#[test]
fn sha256_verification() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("f.bin");
std::fs::write(&p, b"hello world").unwrap();
let good = to_hex(&Sha256::digest(b"hello world"));
verify_sha256(&p, Some(&good)).unwrap();
verify_sha256(&p, Some("0000000000000000000000000000000000000000000000000000000000000000"))
.unwrap_err();
}
#[test]
fn extracts_zip_detected_by_magic_bytes() {
// Downloads land under a generic name with no extension: only the
// PK magic bytes reveal that it is a zip.
let dir = tempfile::tempdir().unwrap();
let payload = dir.path().join("download");
{
let f = std::fs::File::create(&payload).unwrap();
let mut zip = zip::ZipWriter::new(f);
let opts = zip::write::FileOptions::default();
zip.start_file("zeroclaw.exe", opts).unwrap();
std::io::Write::write_all(&mut zip, b"MZfake").unwrap();
zip.finish().unwrap();
}
let out = dir.path().join("out");
assert!(extract_archive(&payload, &out).unwrap());
let found = find_executable(&out, Some("zeroclaw")).unwrap();
assert!(found.ends_with("zeroclaw.exe"), "{found:?}");
}
#[test]
fn extracts_and_finds() {
let dir = tempfile::tempdir().unwrap();
let payload = dir.path().join("app.tar.gz");
{
let f = std::fs::File::create(&payload).unwrap();
let enc = flate2::write::GzEncoder::new(f, flate2::Compression::default());
let mut ar = tar::Builder::new(enc);
let mut header = tar::Header::new_gnu();
header.set_size(6);
header.set_mode(0o755);
header.set_cksum();
ar.append_data(&mut header, "bin/app", &b"binary"[..]).unwrap();
ar.finish().unwrap();
}
let out = dir.path().join("out");
assert!(extract_archive(&payload, &out).unwrap());
let found = find_executable(&out, Some("app")).unwrap();
assert!(found.ends_with("bin/app"));
}
}
Reference in New Issue View Git Blame Copy Permalink