//! Downloading (with checksum verification), GitHub Releases resolution, //! archive extraction and executable discovery. use anyhow::{anyhow, bail, Context, Result}; use serde::Deserialize; use sha2::{Digest, Sha256}; use std::io::{IsTerminal, Read, Write}; use std::path::{Path, PathBuf}; pub const USER_AGENT: &str = concat!("agent-manager/", env!("CARGO_PKG_VERSION")); // --------------------------------------------------------------------------- // URL validation + download // --------------------------------------------------------------------------- /// Only http, https and file schemes are allowed. pub fn validate_url(url: &str) -> Result<()> { let lower = url.to_lowercase(); let ok = lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("file://"); if !ok { bail!( "refusing URL with unsupported scheme (only http, https and file are allowed): {url}" ); } Ok(()) } /// Download a URL to dest, verifying the optional sha256 checksum. pub fn download(url: &str, dest: &Path, expected_sha256: Option<&str>, progress: bool) -> Result<()> { validate_url(url)?; if let Some(local) = url.strip_prefix("file://") { let src = PathBuf::from(local); std::fs::copy(&src, dest) .with_context(|| format!("cannot copy {} to {}", src.display(), dest.display()))?; verify_sha256(dest, expected_sha256)?; return Ok(()); } let resp = ureq::get(url) .set("User-Agent", USER_AGENT) .call() .map_err(|e| { match e { ureq::Error::Status(code, r) => anyhow!( "download failed for {url}: HTTP {code} {}", r.status_text() ), other => anyhow!("download failed for {url}: {other}"), } })?; let total: Option = resp.header("content-length").and_then(|v| v.parse().ok()); let mut reader = resp.into_reader(); let mut file = std::fs::File::create(dest) .with_context(|| format!("cannot create {}", dest.display()))?; let mut buf = [0u8; 65536]; let mut done: u64 = 0; let mut last_shown: u64 = 0; let show = progress && std::io::stderr().is_terminal(); loop { let n = reader .read(&mut buf) .with_context(|| format!("read error while downloading {url}"))?; if n == 0 { break; } file.write_all(&buf[..n])?; done += n as u64; if show && (done - last_shown >= 8 * 1024 * 1024 || done == total.unwrap_or(u64::MAX)) { last_shown = done; match total { Some(t) => eprint!( "\r downloading {:.1} / {:.1} MB", done as f64 / 1e6, t as f64 / 1e6 ), None => eprint!("\r downloaded {:.1} MB", done as f64 / 1e6), } std::io::stderr().flush().ok(); } } if show { eprintln!(); } verify_sha256(dest, expected_sha256)?; Ok(()) } fn verify_sha256(path: &Path, expected: Option<&str>) -> Result<()> { let Some(exp) = expected else { return Ok(()); }; let hex = exp .strip_prefix("sha256:") .or_else(|| exp.strip_prefix("SHA256:")) .unwrap_or(exp) .trim(); if hex.len() != 64 || !hex.chars().all(|c| c.is_ascii_hexdigit()) { bail!("invalid sha256 checksum in config: {exp}"); } let mut f = std::fs::File::open(path)?; let mut hasher = Sha256::new(); std::io::copy(&mut f, &mut hasher)?; let actual = to_hex(&hasher.finalize()); if !actual.eq_ignore_ascii_case(hex) { bail!( "checksum mismatch for {}: expected {hex}, got {actual}", path.display() ); } Ok(()) } pub fn to_hex(bytes: &[u8]) -> String { let mut s = String::with_capacity(bytes.len() * 2); for b in bytes { s.push_str(&format!("{b:02x}")); } s } // --------------------------------------------------------------------------- // GitHub Releases // --------------------------------------------------------------------------- #[derive(Debug, Deserialize)] pub struct GhAsset { pub name: String, pub browser_download_url: String, #[serde(default)] pub size: u64, } #[derive(Debug, Deserialize)] pub struct GhRelease { pub tag_name: String, #[serde(default)] pub assets: Vec, } /// Authorization header for api.github.com when GITHUB_TOKEN or GH_TOKEN /// is set (raises the rate limit from 60 to 5000 requests/hour). fn github_auth_header() -> Option { let token = std::env::var("GITHUB_TOKEN") .or_else(|_| std::env::var("GH_TOKEN")) .ok() .filter(|t| !t.trim().is_empty())?; Some(format!("Bearer {}", token.trim())) } pub fn latest_release(repo: &str) -> Result { latest_release_with_base(repo, None) } /// Resolve the latest release, with an optional API base URL override /// (e.g. https://git.dracodev.net/api/v1 for a Gitea instance). pub fn latest_release_with_base(repo: &str, api_base: Option<&str>) -> Result { let github = api_base.is_none() || api_base == Some("https://api.github.com"); match latest_release_api(repo, api_base) { Ok(release) => Ok(release), Err(e) if github && format!("{e:#}").contains("rate limit") => { // The GitHub API is rate limited (60 unauthenticated // requests/hour): fall back to the GitHub web pages, which are // not limited. (Gitea has no equivalent web fallback.) latest_release_web(repo) } Err(e) => Err(e), } } fn latest_release_api(repo: &str, api_base: Option<&str>) -> Result { let base = api_base.unwrap_or("https://api.github.com"); let url = format!("{base}/repos/{repo}/releases/latest"); let mut req = ureq::get(&url).set("User-Agent", USER_AGENT); if let Some(auth) = github_auth_header() { req = req.set("Authorization", &auth); } let resp = req.call().map_err(|e| match e { ureq::Error::Status(404, _) => { anyhow!("no GitHub releases found for repository '{repo}'") } ureq::Error::Status(403, _) => anyhow!( "GitHub API request failed for '{repo}': rate limit exceeded — set the GITHUB_TOKEN environment variable to raise the limit, or retry in a few minutes" ), other => anyhow!("GitHub API request failed for '{repo}': {other}"), })?; let text = resp .into_string() .context("reading GitHub API response")?; serde_json::from_str::(&text).context("parsing GitHub API response") } /// Discover the latest release without the API: the /releases/latest page /// redirects to the tagged page (tag from the final URL), then the /// expanded_assets endpoint lists every asset. These web pages are not /// subject to the API rate limit. fn latest_release_web(repo: &str) -> Result { let latest = format!("https://github.com/{repo}/releases/latest"); let resp = ureq::get(&latest) .set("User-Agent", USER_AGENT) .call() .map_err(|e| anyhow!("cannot reach GitHub releases page for '{repo}': {e}"))?; let final_url = resp.get_url().to_string(); let tag = final_url .rsplit('/') .next() .map(|s| s.to_string()) .filter(|s| !s.is_empty() && final_url.contains("/releases/tag/")) .ok_or_else(|| { anyhow!("no GitHub releases found for repository '{repo}' (no tagged release page)") })?; let assets_url = format!("https://github.com/{repo}/releases/expanded_assets/{tag}"); let resp = ureq::get(&assets_url) .set("User-Agent", USER_AGENT) .call() .map_err(|e| anyhow!("cannot list GitHub release assets for '{repo}': {e}"))?; let html = resp .into_string() .context("reading GitHub expanded_assets response")?; let mut assets = parse_expanded_assets(&html, repo, &tag); if assets.is_empty() { anyhow::bail!( "GitHub release '{repo}@{tag}' has no downloadable assets (web fallback)" ); } assets.sort_by(|a, b| a.name.cmp(&b.name)); assets.dedup_by(|a, b| a.name == b.name); Ok(GhRelease { tag_name: tag, assets, }) } /// Extract the asset links from a GitHub expanded_assets HTML fragment. fn parse_expanded_assets(html: &str, repo: &str, tag: &str) -> Vec { let marker = format!("/{repo}/releases/download/"); let mut assets = Vec::new(); let mut rest = html; while let Some(pos) = rest.find(&marker) { let start = pos + marker.len(); let end = rest[start..] .find(['"', '\'', '<']) .map(|i| start + i) .unwrap_or(rest.len()); let raw = &rest[start..end]; // The href reads "download//": keep the name only. let raw = raw.strip_prefix(&format!("{tag}/")).unwrap_or(raw); // GitHub sometimes hides zero-width characters inside hrefs. let name = raw.replace('\u{200b}', ""); let name = decode_html_entities(&name); if !name.is_empty() { assets.push(GhAsset { name: name.clone(), browser_download_url: format!( "https://github.com/{repo}/releases/download/{tag}/{name}" ), size: 0, }); } rest = &rest[end..]; } assets } fn decode_html_entities(s: &str) -> String { s.replace("&", "&") .replace(""", "\"") .replace("'", "'") .replace(""", "\"") .replace(" ", " ") .replace("<", "<") .replace(">", ">") .replace(" ", " ") } pub struct PickedAsset { pub name: String, pub url: String, pub tag: String, } /// Choose the release asset that best matches the current platform and arch. pub fn pick_asset( release: &GhRelease, want: &[String], exclude: &[String], binary_hint: Option<&str>, ) -> Option { let platform = if cfg!(windows) { "windows" } else if cfg!(target_os = "macos") { "macos" } else { "linux" }; let arch = if cfg!(target_arch = "aarch64") { "aarch64" } else { "x86_64" }; let mut best: Option<(u32, usize, &GhAsset)> = None; for asset in &release.assets { let lower = asset.name.to_lowercase(); // Skip checksums, signatures and desktop packages. if lower.ends_with(".sha256") || lower.ends_with(".sha512") || lower.ends_with(".sig") || lower.ends_with(".asc") || lower.ends_with(".pem") || lower.contains("checksums") || lower.ends_with(".dmg") || lower.ends_with(".deb") || lower.ends_with(".rpm") || lower.ends_with(".msi") || lower.ends_with(".apk") { continue; } if !want.iter().all(|w| lower.contains(&w.to_lowercase())) { continue; } if exclude.iter().any(|w| lower.contains(&w.to_lowercase())) { continue; } let score = match platform_score(&lower, platform, arch) { Some(s) => s, None => continue, }; let replace = match best { None => true, Some((s, _, _)) => score > s, }; if replace { best = Some((score, asset.name.len(), asset)); } } best.map(|(_, _, a)| { let _ = binary_hint; PickedAsset { name: a.name.clone(), url: a.browser_download_url.clone(), tag: release.tag_name.clone(), } }) } fn platform_score(lower: &str, platform: &str, arch: &str) -> Option { let plat_ok = match platform { "windows" => { lower.contains("windows") || lower.ends_with(".exe") || lower.contains("win64") || lower.contains("win32") || lower.contains("msvc") || lower.contains("mingw") || lower.contains("pc-windows") } "macos" => { lower.contains("darwin") || lower.contains("macos") || lower.contains("apple") || lower.contains("osx") } _ => { lower.contains("linux") || lower.contains("unknown-linux") || lower.contains("manylinux") } }; if !plat_ok { return None; } let arch_ok = match arch { "aarch64" => lower.contains("aarch64") || lower.contains("arm64") || lower.contains("armv8"), _ => { lower.contains("x86_64") || lower.contains("amd64") || lower.contains("x64") || lower.contains("i686") } }; if !arch_ok && !lower.contains("universal") { return None; } let mut s = 1u32; if lower.contains("universal") { s += 4; } if arch == "aarch64" && lower.contains("arm64") { s += 3; } if arch == "x86_64" && (lower.contains("x86_64") || lower.contains("amd64")) { s += 3; } if lower.contains("musl") { s += 1; } if lower.contains("static") || lower.contains("portable") { s += 1; } // Prefer plain archives (extracted to a single executable) over // installer/setup programs, which cannot be used directly. if lower.ends_with(".zip") || lower.ends_with(".tar.gz") || lower.ends_with(".tgz") { s += 2; } if lower.contains("setup") || lower.contains("installer") || lower.contains("-install") { s = s.saturating_sub(3); } if lower.contains("armv7") || lower.contains("armhf") || lower.contains("armv6") || lower.contains("i686") || lower.contains("i386") { s = s.saturating_sub(2); } Some(s) } // --------------------------------------------------------------------------- // Archives and executables // --------------------------------------------------------------------------- /// What kind of archive a file is, sniffed from its content. enum ArchiveKind { Zip, TarGz, Tar, Gz, Plain, } /// Detect the archive type from the file's magic bytes, falling back to the /// file name. Downloads are stored under a generic name ("download"), so /// content sniffing is the reliable way to recognize a zip/tar.gz asset. fn sniff_archive(path: &Path) -> Result { let mut head = [0u8; 512]; let n = std::fs::File::open(path) .and_then(|mut f| std::io::Read::read(&mut f, &mut head)) .unwrap_or(0); if n >= 4 && &head[0..4] == b"PK\x03\x04" { return Ok(ArchiveKind::Zip); } if n >= 2 && head[0] == 0x1f && head[1] == 0x8b { // gzip: peek at the decompressed stream to see whether it is a tar. let f = std::fs::File::open(path)?; let mut dec = flate2::read::GzDecoder::new(f); let mut inner = [0u8; 512]; let m = std::io::Read::read(&mut dec, &mut inner).unwrap_or(0); if m >= 262 && &inner[257..262] == b"ustar" { return Ok(ArchiveKind::TarGz); } return Ok(ArchiveKind::Gz); } if n >= 262 && &head[257..262] == b"ustar" { return Ok(ArchiveKind::Tar); } // Content is not an archive: keep the name-based detection for files // that still carry a real archive extension. let name = path .file_name() .and_then(|n| n.to_str()) .unwrap_or("") .to_lowercase(); if name.ends_with(".zip") { return Ok(ArchiveKind::Zip); } if name.ends_with(".tar.gz") || name.ends_with(".tgz") { return Ok(ArchiveKind::TarGz); } if name.ends_with(".tar") { return Ok(ArchiveKind::Tar); } if name.ends_with(".gz") { return Ok(ArchiveKind::Gz); } Ok(ArchiveKind::Plain) } /// Extract zip / tar.gz / tgz / tar / gz archives, detected by content. /// Returns false when the file is not a recognized archive (caller treats /// it as a plain binary). pub fn extract_archive(path: &Path, dest: &Path) -> Result { std::fs::create_dir_all(dest)?; match sniff_archive(path)? { ArchiveKind::Zip => { extract_zip(path, dest)?; Ok(true) } ArchiveKind::TarGz => { let f = std::fs::File::open(path)?; let dec = flate2::read::GzDecoder::new(f); let mut ar = tar::Archive::new(dec); ar.unpack(dest) .with_context(|| format!("extracting {}", path.display()))?; Ok(true) } ArchiveKind::Tar => { let f = std::fs::File::open(path)?; let mut ar = tar::Archive::new(f); ar.unpack(dest) .with_context(|| format!("extracting {}", path.display()))?; Ok(true) } ArchiveKind::Gz => { let name = path .file_name() .and_then(|n| n.to_str()) .unwrap_or("") .to_lowercase(); let inner = name.trim_end_matches(".gz").to_string(); let out = dest.join(inner); let f = std::fs::File::open(path)?; let mut dec = flate2::read::GzDecoder::new(f); let mut o = std::fs::File::create(&out)?; std::io::copy(&mut dec, &mut o)?; Ok(true) } ArchiveKind::Plain => Ok(false), } } fn extract_zip(path: &Path, dest: &Path) -> Result<()> { let f = std::fs::File::open(path)?; let mut zip = zip::ZipArchive::new(f) .with_context(|| format!("opening zip archive {}", path.display()))?; for i in 0..zip.len() { let mut entry = zip.by_index(i)?; let Some(enclosed) = entry.enclosed_name() else { bail!("zip archive {} contains an unsafe path", path.display()); }; let out_path = dest.join(enclosed); if entry.is_dir() { std::fs::create_dir_all(&out_path)?; } else { if let Some(parent) = out_path.parent() { std::fs::create_dir_all(parent)?; } let mut out = std::fs::File::create(&out_path)?; std::io::copy(&mut entry, &mut out)?; } } Ok(()) } /// Find the executable in a directory tree: exact name match first, then a /// single candidate file, then a single executable file (Unix). pub fn find_executable(dir: &Path, expected: Option<&str>) -> Result { let mut files = Vec::new(); collect_files(dir, &mut files); if files.is_empty() { bail!("no files found in {}", dir.display()); } if let Some(exp) = expected { for f in &files { let name = f .file_name() .and_then(|n| n.to_str()) .unwrap_or("") .to_string(); let stripped = name.strip_suffix(".exe").unwrap_or(&name); if stripped.eq_ignore_ascii_case(exp) { return Ok(f.clone()); } } } if files.len() == 1 { return Ok(files.remove(0)); } #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; let exec: Vec<&PathBuf> = files .iter() .filter(|f| { std::fs::metadata(f) .map(|m| m.permissions().mode() & 0o111 != 0) .unwrap_or(false) }) .collect(); if exec.len() == 1 { return Ok(exec[0].clone()); } } let names: Vec = files .iter() .map(|f| format!(" {}", f.display())) .collect(); bail!( "cannot determine which file is the executable in {}; candidates: {}", dir.display(), names.join(" ") ) } fn collect_files(dir: &Path, out: &mut Vec) { if let Ok(rd) = std::fs::read_dir(dir) { let mut entries: Vec = rd.flatten().map(|e| e.path()).collect(); entries.sort(); for p in entries { if p.is_dir() { collect_files(&p, out); } else { out.push(p); } } } } #[cfg(test)] mod tests { use super::*; #[test] fn url_validation() { assert!(validate_url("https://example.com/x").is_ok()); assert!(validate_url("http://example.com/x").is_ok()); assert!(validate_url("file:///tmp/x").is_ok()); assert!(validate_url("ftp://example.com/x").is_err()); assert!(validate_url("javascript:alert(1)").is_err()); } #[test] fn parses_expanded_assets_fragment() { let html = r#"
x y ignore me z
"#; let assets = parse_expanded_assets(html, "owner/repo", "v1.2.3"); let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect(); assert!(names.contains(&"tool-windows-x86_64.exe"), "{names:?}"); assert!(names.contains(&"tool-windows-x86_64 copy.zip"), "{names:?}"); assert!(names.contains(&"SHA256SUMS"), "{names:?}"); assert_eq!(assets.len(), 3); assert_eq!( assets[0].browser_download_url, "https://github.com/owner/repo/releases/download/v1.2.3/tool-windows-x86_64.exe" ); } #[test] fn sha256_verification() { let dir = tempfile::tempdir().unwrap(); let p = dir.path().join("f.bin"); std::fs::write(&p, b"hello world").unwrap(); let good = to_hex(&Sha256::digest(b"hello world")); verify_sha256(&p, Some(&good)).unwrap(); verify_sha256(&p, Some("0000000000000000000000000000000000000000000000000000000000000000")) .unwrap_err(); } #[test] fn extracts_zip_detected_by_magic_bytes() { // Downloads land under a generic name with no extension: only the // PK magic bytes reveal that it is a zip. let dir = tempfile::tempdir().unwrap(); let payload = dir.path().join("download"); { let f = std::fs::File::create(&payload).unwrap(); let mut zip = zip::ZipWriter::new(f); let opts = zip::write::FileOptions::default(); zip.start_file("zeroclaw.exe", opts).unwrap(); std::io::Write::write_all(&mut zip, b"MZfake").unwrap(); zip.finish().unwrap(); } let out = dir.path().join("out"); assert!(extract_archive(&payload, &out).unwrap()); let found = find_executable(&out, Some("zeroclaw")).unwrap(); assert!(found.ends_with("zeroclaw.exe"), "{found:?}"); } #[test] fn extracts_and_finds() { let dir = tempfile::tempdir().unwrap(); let payload = dir.path().join("app.tar.gz"); { let f = std::fs::File::create(&payload).unwrap(); let enc = flate2::write::GzEncoder::new(f, flate2::Compression::default()); let mut ar = tar::Builder::new(enc); let mut header = tar::Header::new_gnu(); header.set_size(6); header.set_mode(0o755); header.set_cksum(); ar.append_data(&mut header, "bin/app", &b"binary"[..]).unwrap(); ar.finish().unwrap(); } let out = dir.path().join("out"); assert!(extract_archive(&payload, &out).unwrap()); let found = find_executable(&out, Some("app")).unwrap(); assert!(found.ends_with("bin/app")); } }