diff --git a/Cargo.lock b/Cargo.lock index 72b1d5e6a..ba6c7617d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8355,7 +8355,9 @@ name = "spin-templates" version = "3.1.0-pre0" dependencies = [ "anyhow", + "bytes", "dialoguer", + "flate2", "fs_extra", "heck 0.5.0", "indexmap 2.6.0", @@ -8367,10 +8369,12 @@ dependencies = [ "path-absolutize", "pathdiff", "regex", + "reqwest 0.12.9", "semver", "serde", "spin-common", "spin-manifest", + "tar", "tempfile", "tokio", "toml", diff --git a/crates/templates/Cargo.toml b/crates/templates/Cargo.toml index 998734c49..c0b8a07aa 100644 --- a/crates/templates/Cargo.toml +++ b/crates/templates/Cargo.toml @@ -6,9 +6,11 @@ edition = { workspace = true } [dependencies] anyhow = { workspace = true } +bytes = { workspace = true } dialoguer = "0.11" fs_extra = "1" heck = "0.5" +flate2 = "1" indexmap = { version = "2", features = ["serde"] } itertools = { workspace = true } lazy_static = "1" @@ -18,10 +20,12 @@ liquid-derive = "0.26" path-absolutize = "3" pathdiff = "0.2" regex = { workspace = true } +reqwest = { workspace = true } semver = "1" serde = { workspace = true } spin-common = { path = "../common" } spin-manifest = { path = "../manifest" } +tar = "0.4" tempfile = { workspace = true } tokio = { workspace = true, features = ["fs", "process", "rt", "macros"] } toml = { workspace = true } diff --git a/crates/templates/src/reader.rs b/crates/templates/src/reader.rs index 2ab2a1b35..70bf5622a 100644 --- a/crates/templates/src/reader.rs +++ b/crates/templates/src/reader.rs @@ -118,6 +118,7 @@ pub(crate) fn parse_manifest_toml(text: impl AsRef) -> anyhow::Result) -> Option { diff --git a/crates/templates/src/source.rs b/crates/templates/src/source.rs index 2edd87036..371d62f90 100644 --- a/crates/templates/src/source.rs +++ b/crates/templates/src/source.rs @@ -25,6 +25,12 @@ pub enum TemplateSource { /// Templates much be in a `/templates` directory under the specified /// root. File(PathBuf), + /// Install from a remote tarball. + /// + /// Templates should be in a `/templates` directory under the root of the tarball. + /// The implementation also allows for there to be a single root directory containing + /// the `templates` directory - this makes it compatible with GitHub release tarballs. + RemoteTar(Url), } /// Settings for installing templates from a Git repository. @@ -72,6 +78,9 @@ impl TemplateSource { None } } + Self::RemoteTar(url) => Some(crate::reader::RawInstalledFrom::RemoteTar { + url: url.to_string(), + }), } } @@ -96,6 +105,7 @@ impl TemplateSource { match self { Self::Git(git_source) => clone_local(git_source).await, Self::File(path) => check_local(path).await, + Self::RemoteTar(url) => download_untar_local(url).await, } } @@ -103,6 +113,7 @@ impl TemplateSource { match self { Self::Git { .. } => true, Self::File(_) => false, + Self::RemoteTar(_) => true, } } } @@ -192,6 +203,84 @@ async fn check_local(path: &Path) -> anyhow::Result { } } +/// Download a tarball to a temorary directory +async fn download_untar_local(url: &Url) -> anyhow::Result { + use bytes::buf::Buf; + + let temp_dir = tempdir()?; + let path = temp_dir.path().to_owned(); + + let resp = reqwest::get(url.clone()) + .await + .with_context(|| format!("Failed to download from {url}"))?; + let tar_content = resp + .bytes() + .await + .with_context(|| format!("Failed to download from {url}"))?; + + let reader = flate2::read::GzDecoder::new(tar_content.reader()); + let mut archive = tar::Archive::new(reader); + archive + .unpack(&path) + .context("Failed to unpack tar archive")?; + + let templates_root = bypass_gh_added_root(path); + + Ok(LocalTemplateSource { + root: templates_root, + _temp_dir: Some(temp_dir), + }) +} + +/// GitHub adds a prefix directory to release tarballs (e.g. spin-v3.0.0/...). +/// We try to locate the repo root within the unpacked tarball. +fn bypass_gh_added_root(unpack_dir: PathBuf) -> PathBuf { + // If the unpack dir directly contains a `templates` dir then we are done. + if has_templates_dir(&unpack_dir) { + return unpack_dir; + } + + let Ok(dirs) = unpack_dir.read_dir() else { + // If we can't traverse the unpack directory then return it and + // let the top level try to make sense of it. + return unpack_dir; + }; + + // Is there a single directory at the root? If not, we can't be in the GitHub situation: + // return the root of the unpacking. (The take(2) here is because we don't need to traverse + // the full list - we only care whether there is more than one.) + let dirs = dirs.filter_map(|de| de.ok()).take(2).collect::>(); + if dirs.len() != 1 { + return unpack_dir; + } + + // If we get here, there is a single directory (dirs has a single element). Look in it to see if it's a plausible repo root. + let candidate_repo_root = dirs[0].path(); + let Ok(mut candidate_repo_dirs) = candidate_repo_root.read_dir() else { + // Again, if it all goes awry, propose the base unpack directory. + return unpack_dir; + }; + let has_templates_dir = candidate_repo_dirs.any(is_templates_dir); + + if has_templates_dir { + candidate_repo_root + } else { + unpack_dir + } +} + +fn has_templates_dir(path: &Path) -> bool { + let Ok(mut dirs) = path.read_dir() else { + return false; + }; + + dirs.any(is_templates_dir) +} + +fn is_templates_dir(dir_entry: Result) -> bool { + dir_entry.is_ok_and(|d| d.file_name() == TEMPLATE_SOURCE_DIR) +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/templates/src/template.rs b/crates/templates/src/template.rs index a2204afd2..02fca2525 100644 --- a/crates/templates/src/template.rs +++ b/crates/templates/src/template.rs @@ -37,6 +37,7 @@ pub struct Template { enum InstalledFrom { Git(String), Directory(String), + RemoteTar(String), Unknown, } @@ -254,6 +255,7 @@ impl Template { match &self.installed_from { InstalledFrom::Git(repo) => repo, InstalledFrom::Directory(path) => path, + InstalledFrom::RemoteTar(url) => url, InstalledFrom::Unknown => "", } } @@ -625,6 +627,7 @@ fn read_install_record(layout: &TemplateLayout) -> InstalledFrom { match installed_from_text.and_then(parse_installed_from) { Some(RawInstalledFrom::Git { git }) => InstalledFrom::Git(git), Some(RawInstalledFrom::File { dir }) => InstalledFrom::Directory(dir), + Some(RawInstalledFrom::RemoteTar { url }) => InstalledFrom::RemoteTar(url), None => InstalledFrom::Unknown, } } diff --git a/src/commands/templates.rs b/src/commands/templates.rs index aa9367d09..34ff2fa13 100644 --- a/src/commands/templates.rs +++ b/src/commands/templates.rs @@ -15,6 +15,7 @@ use crate::build_info::*; const INSTALL_FROM_DIR_OPT: &str = "FROM_DIR"; const INSTALL_FROM_GIT_OPT: &str = "FROM_GIT"; +const INSTALL_FROM_TAR_OPT: &str = "FROM_TAR"; const UPGRADE_ONLY: &str = "GIT_URL"; const DEFAULT_TEMPLATES_INSTALL_PROMPT: &str = @@ -64,6 +65,7 @@ pub struct Install { long = "git", alias = "repo", conflicts_with = INSTALL_FROM_DIR_OPT, + conflicts_with = INSTALL_FROM_TAR_OPT, )] pub git: Option, @@ -76,9 +78,19 @@ pub struct Install { name = INSTALL_FROM_DIR_OPT, long = "dir", conflicts_with = INSTALL_FROM_GIT_OPT, + conflicts_with = INSTALL_FROM_TAR_OPT, )] pub dir: Option, + /// URL to a tarball in .tar.gz format containing the template(s) to install. + #[clap( + name = INSTALL_FROM_TAR_OPT, + long = "tar", + conflicts_with = INSTALL_FROM_GIT_OPT, + conflicts_with = INSTALL_FROM_DIR_OPT, + )] + pub tar_url: Option, + /// If present, updates existing templates instead of skipping. #[clap(long = "upgrade", alias = "update")] pub update: bool, @@ -119,16 +131,20 @@ impl Install { pub async fn run(self) -> Result<()> { let template_manager = TemplateManager::try_default() .context("Failed to construct template directory path")?; - let source = match (&self.git, &self.dir) { - (Some(git), None) => { + let source = match (&self.git, &self.dir, &self.tar_url) { + (Some(git), None, None) => { let git_url = infer_github(git); TemplateSource::try_from_git(git_url, &self.branch, SPIN_VERSION)? } - (None, Some(dir)) => { + (None, Some(dir), None) => { let abs_dir = dir.absolutize().map(|d| d.to_path_buf()); TemplateSource::File(abs_dir.unwrap_or_else(|_| dir.clone())) } - _ => anyhow::bail!("Exactly one of `git` and `dir` sources must be specified"), + (None, None, Some(tar_url)) => { + let url = url::Url::parse(tar_url).context("Invalid URL for remote tar")?; + TemplateSource::RemoteTar(url) + } + _ => anyhow::bail!("Exactly one of `git`, `dir`, or `tar` must be specified"), }; let reporter = ConsoleProgressReporter; @@ -204,6 +220,7 @@ impl Upgrade { git: self.git.clone(), branch: self.branch.clone(), dir: None, + tar_url: None, update: true, }; @@ -620,6 +637,7 @@ async fn install_default_templates() -> anyhow::Result<()> { git: Some(DEFAULT_TEMPLATE_REPO.to_owned()), branch: None, dir: None, + tar_url: None, update: false, }; install_cmd