//! Remote catalogs (issues #60 #67, axes 7 & 9): fetching a catalog from //! an http(s) URL, validating its format, and caching it on disk the //! probe.rs way (keyed by URL + timestamp, TTL-bounded). Used both by //! 'am catalog update/add' and by config `include:` entries pointing at //! URLs. //! //! Rules: //! * only http, https and file:// URLs are accepted (download::validate_url); //! * a cached copy is served while it is fresh; on network failure the //! stale cache is used with a warning (offline support); //! * a malformed catalog is an error, never silently skipped. use crate::config::{self, Config}; use anyhow::{anyhow, bail, Context, Result}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; /// Default time-to-live of a cached remote catalog, in seconds (1 h). pub const DEFAULT_TTL_SECS: u64 = 3600; /// Base URL of the official catalog (the agent-manager repo on Gitea). pub const DEFAULT_CATALOG_URL: &str = "https://git.dracodev.net/Projets/agent-manager/raw/branch/main/config.yaml"; /// One cached catalog: the raw text, the URL it came from, and the fetch /// time (seconds since the epoch). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CachedCatalog { pub url: String, pub fetched_at: u64, pub text: String, } #[derive(Debug, Default, Serialize, Deserialize)] pub struct CatalogCache { pub entries: BTreeMap, } impl CatalogCache { fn load(state_dir: &Path) -> Self { std::fs::read_to_string(state_dir.join("catalog-cache.json")) .ok() .and_then(|t| serde_json::from_str(&t).ok()) .unwrap_or_default() } fn save(&self, state_dir: &Path) { if let Ok(text) = serde_json::to_string_pretty(self) { let _ = std::fs::write(state_dir.join("catalog-cache.json"), text); } } } /// Fetch the text of a catalog URL, using and refreshing the on-disk cache. /// `force` bypasses the TTL (used by 'am catalog update'). pub fn fetch(app: &crate::app::App, url: &str, force: bool) -> Result { let ttl = app .config .settings .catalog_cache_ttl_secs .unwrap_or(DEFAULT_TTL_SECS); fetch_with_cache(&app.events_dir(), url, force, ttl, Some(&app.log)) } /// Fetch a catalog URL during config loading (no App available yet): the /// cache lives in the state directory and the TTL is the default one. pub fn fetch_for_config(state_dir: &Path, url: &str) -> Result { fetch_with_cache(state_dir, url, false, DEFAULT_TTL_SECS, None) } /// Shared fetch + cache logic. fn fetch_with_cache( state_dir: &Path, url: &str, force: bool, ttl: u64, log: Option<&crate::output::Logger>, ) -> Result { crate::download::validate_url(url)?; let mut cache = CatalogCache::load(state_dir); let now = now_secs(); // Fresh cache hit: no network. if !force { if let Some(entry) = cache.entries.get(url) { if now.saturating_sub(entry.fetched_at) < ttl { return Ok(entry.text.clone()); } } } // Try the network; fall back to the stale cache on failure. let fresh = fetch_url_text(url); match fresh { Ok(text) => { cache.entries.insert( url.to_string(), CachedCatalog { url: url.to_string(), fetched_at: now, text: text.clone(), }, ); cache.save(state_dir); Ok(text) } Err(net_err) => { if let Some(entry) = cache.entries.get(url) { if let Some(log) = log { log.warn(&format!( "cannot refresh catalog {url} ({net_err:#}) — using the cached copy from {}", format_ts(entry.fetched_at) )); } Ok(entry.text.clone()) } else { Err(net_err) } } } } /// Fetch the raw text of an http(s) or file:// URL (no cache). fn fetch_url_text(url: &str) -> Result { if let Some(local) = url.strip_prefix("file://") { // Windows: "file:///C:/x" -> "C:/x" (drop the leading slash). let local = if cfg!(windows) && local.starts_with('/') { local.strip_prefix('/').unwrap_or(local) } else { local }; let p = PathBuf::from(local); return std::fs::read_to_string(&p) .with_context(|| format!("cannot read catalog file {}", p.display())); } let resp = ureq::get(url) .set("User-Agent", crate::download::USER_AGENT) .timeout(std::time::Duration::from_secs(30)) .call() .map_err(|e| match e { ureq::Error::Status(code, r) => anyhow!( "HTTP {code} {} while fetching {url}", r.status_text() ), other => anyhow!("cannot fetch {url}: {other}"), })?; resp.into_string() .with_context(|| format!("reading the catalog body of {url}")) } /// Parse and validate a catalog YAML text; returns the parsed Config. pub fn parse_catalog(text: &str) -> Result { let cfg: Config = serde_yaml::from_str(text) .map_err(|e| anyhow!("the remote catalog is not a valid config: {e}"))?; let problems = config::validate(&cfg); if !problems.is_empty() { bail!( "the remote catalog is invalid:\n - {}", problems.join("\n - ") ); } Ok(cfg) } /// Diff between the local effective catalog and a remote one: agent names /// that are new, removed, or changed (run command differs). pub struct CatalogDiff { pub added: Vec, pub removed: Vec, pub changed: Vec<(String, String, String)>, // (name, local run, remote run) } pub fn diff_catalogs(local: &Config, remote: &Config) -> CatalogDiff { let local_names: BTreeMap<&str, &str> = local .agents .iter() .map(|a| (a.name.as_str(), a.run.as_deref().unwrap_or(""))) .collect(); let remote_names: BTreeMap<&str, &str> = remote .agents .iter() .map(|a| (a.name.as_str(), a.run.as_deref().unwrap_or(""))) .collect(); let mut added = Vec::new(); let mut removed = Vec::new(); let mut changed = Vec::new(); for (name, run) in &remote_names { match local_names.get(name) { None => added.push((*name).to_string()), Some(local_run) if local_run != run => { changed.push(((*name).to_string(), (*local_run).to_string(), (*run).to_string())); } _ => {} } } for name in local_names.keys() { if !remote_names.contains_key(name) { removed.push((*name).to_string()); } } added.sort(); removed.sort(); changed.sort(); CatalogDiff { added, removed, changed, } } /// The official catalog URL: the configured value, or the Gitea default. pub fn official_url(app: &crate::app::App) -> String { app.config .settings .catalog_url .clone() .unwrap_or_else(|| DEFAULT_CATALOG_URL.to_string()) } fn now_secs() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0) } fn format_ts(secs: u64) -> String { let dt = chrono::DateTime::from_timestamp(secs as i64, 0); dt.map(|d| d.format("%Y-%m-%d %H:%M").to_string()) .unwrap_or_else(|| "unknown date".to_string()) } #[cfg(test)] mod tests { use super::*; const VALID: &str = "version: \"1.0\"\nagents:\n - name: alpha\n run: alpha\n install: { type: binary, url: \"https://x/a\" }\n"; #[test] fn parses_and_validates_a_remote_catalog() { let cfg = parse_catalog(VALID).unwrap(); assert_eq!(cfg.agents.len(), 1); assert_eq!(cfg.agents[0].name, "alpha"); } #[test] fn rejects_malformed_catalogs() { assert!(parse_catalog("not: [valid").is_err()); // Missing run on an installable agent is invalid. assert!(parse_catalog( "version: \"1.0\"\nagents:\n - name: x\n install: { type: npm, package: y }\n" ) .is_err()); } #[test] fn diff_detects_added_removed_changed() { let local: Config = serde_yaml::from_str( "version: \"1.0\"\nagents:\n - { name: keep, run: keep, install: { type: binary, url: \"https://x/k\" } }\n - { name: gone, run: gone, install: { type: binary, url: \"https://x/g\" } }\n - { name: tweaked, run: old, install: { type: binary, url: \"https://x/t\" } }\n", ) .unwrap(); let remote: Config = serde_yaml::from_str( "version: \"1.0\"\nagents:\n - { name: keep, run: keep, install: { type: binary, url: \"https://x/k\" } }\n - { name: fresh, run: fresh, install: { type: binary, url: \"https://x/f\" } }\n - { name: tweaked, run: new, install: { type: binary, url: \"https://x/t\" } }\n", ) .unwrap(); let d = diff_catalogs(&local, &remote); assert_eq!(d.added, vec!["fresh".to_string()]); assert_eq!(d.removed, vec!["gone".to_string()]); assert_eq!(d.changed.len(), 1); assert_eq!(d.changed[0].0, "tweaked"); assert_eq!(d.changed[0].1, "old"); assert_eq!(d.changed[0].2, "new"); } #[test] fn file_url_fetch_works_offline() { let dir = tempfile::tempdir().unwrap(); let p = dir.path().join("cat.yaml"); std::fs::write(&p, VALID).unwrap(); let url = format!("file://{}", p.display()); let text = fetch_url_text(&url).unwrap(); assert!(text.contains("alpha")); } #[test] fn cache_roundtrip() { let dir = tempfile::tempdir().unwrap(); let mut cache = CatalogCache::default(); cache.entries.insert( "https://x".to_string(), CachedCatalog { url: "https://x".to_string(), fetched_at: 42, text: "body".to_string(), }, ); cache.save(dir.path()); let back = CatalogCache::load(dir.path()); assert_eq!(back.entries["https://x"].text, "body"); } }