fix(elan-dist): completely avoid using the Github API

src/download/src/lib.rs

@@ -163,9 +163,6 @@ pub mod curl {
                 let _ = handle.resume_from(0);
             }
 
-            // necessary for accessing Github API
-            handle.useragent("elan");
-
             // Take at most 30s to connect
             try!(handle.connect_timeout(Duration::new(30, 0)).chain_err(|| "failed to set connect timeout"));
 

src/elan-cli/self_update.rs

@@ -35,7 +35,6 @@ use errors::*;
 use elan_dist::dist;
 use elan_utils::utils;
 use flate2;
-use json;
 use same_file::Handle;
 use std::env;
 use std::env::consts::EXE_SUFFIX;
@@ -1268,15 +1267,7 @@ pub fn prepare_update() -> Result<Option<PathBuf>> {
     // Download available version
     info!("checking for self-updates");
 
-    let manifest_url = utils::parse_url("https://api.github.com/repos/Kha/elan/releases/latest")?;
-    let manifest_path = &tempdir.path().join("manifest");
-    try!(utils::download_file(&manifest_url,
-                              &manifest_path,
-                              None,
-                              &|_| ()));
-    let manifest_str = try!(utils::read_file("manifest", &manifest_path));
-    let man_json = json::parse(&manifest_str).expect("failed to parse manifest");
-    let tag = man_json["tag_name"].as_str().unwrap();
+    let tag = utils::fetch_latest_release_tag("Kha/elan")?;
     let available_version = &tag[1..];
 
     // If up-to-date

src/elan-dist/src/dist.rs

@@ -6,14 +6,13 @@ use prefix::InstallPrefix;
 use manifest::Component;
 use manifestation::{Manifestation};
 use download::{DownloadCfg};
+use notifications::Notification;
 
 use std::path::Path;
 use std::fmt;
 use std::env;
 
-use json;
 use regex::Regex;
-use sha2::{Sha256, Digest};
 
 pub const DEFAULT_DIST_SERVER: &'static str = "https://static.lean-lang.org";
 
@@ -336,20 +335,6 @@ impl ToolchainDesc {
             .ok_or(ErrorKind::InvalidToolchainName(name.to_string()).into())
     }
 
-    pub fn manifest_v1_url(&self, _dist_root: &str) -> String {
-        match (self.channel.as_str(), self.date.as_ref()) {
-            ("stable", None) => "https://api.github.com/repos/leanprover/lean/releases/latest".to_string(),
-            // HACK: prereleases don't have "/latest"
-            ("nightly", None) => "https://api.github.com/repos/leanprover/lean-nightly/releases".to_string(),
-            ("nightly", Some(date)) =>
-                format!("https://api.github.com/repos/leanprover/lean-nightly/releases/tags/nightly-{}",
-                        date),
-            (version, None) => format!("https://api.github.com/repos/leanprover/lean/releases/tags/v{}",
-                                    version),
-            _ => panic!("wat"),
-        }
-    }
-
     /// Either "$channel" or "channel-$date"
     pub fn manifest_name(&self) -> String {
         match self.date {
@@ -496,8 +481,8 @@ pub fn update_from_dist_<'a>(download: DownloadCfg<'a>,
     let toolchain_str = toolchain.to_string();
     let manifestation = try!(Manifestation::open(prefix.clone(), toolchain.target.clone()));
 
-    let manifest = match dl_v1_manifest(download, toolchain) {
-        Ok(m) => m,
+    let url = match toolchain_url(download, toolchain) {
+        Ok(url) => url,
         Err(Error(ErrorKind::Utils(elan_utils::ErrorKind::DownloadNotExists { .. }), _)) => {
             return Err(format!("no release found for '{}'", toolchain.manifest_name()).into());
         }
@@ -506,24 +491,17 @@ pub fn update_from_dist_<'a>(download: DownloadCfg<'a>,
         }
         Err(e) => {
             return Err(e).chain_err(|| {
-                format!("failed to download manifest for '{}'",
+                format!("failed to resolve latest version of '{}'",
                         toolchain.manifest_name())
             });
         }
     };
 
-    let mut hasher = Sha256::new();
-    for url in &manifest {
-        hasher.input(url.as_bytes());
-    }
-    let hash = format!("{:x}", hasher.result());
-    let partial_hash = &hash[0..20];
-
     if let Some(hash_file) = update_hash {
         if utils::is_file(hash_file) {
             if let Ok(contents) = utils::read_file("update hash", hash_file) {
-                if contents == partial_hash {
-                    // Skip download, update hash matches
+                if contents == url {
+                    // Skip download, url matches
                     return Ok(None);
                 }
             } /*else {
@@ -534,7 +512,7 @@ pub fn update_from_dist_<'a>(download: DownloadCfg<'a>,
         }*/
     }
 
-    match manifestation.update_v1(&manifest,
+    match manifestation.update_v1(&url,
                                   &download.temp_cfg,
                                   download.notify_handler.clone()) {
         Ok(()) => Ok(()),
@@ -545,20 +523,22 @@ pub fn update_from_dist_<'a>(download: DownloadCfg<'a>,
             })
         }
         Err(e) => Err(e),
-    }.map(|()| Some(partial_hash.to_string()))
+    }.map(|()| Some(url))
 }
 
-fn dl_v1_manifest<'a>(download: DownloadCfg<'a>, toolchain: &ToolchainDesc) -> Result<Vec<String>> {
-    let manifest_url = toolchain.manifest_v1_url(download.dist_root);
-    let manifest_file = try!(download.download_and_check(&manifest_url, ""));
-    let manifest_str = try!(utils::read_file("manifest", &manifest_file));
-    let mut man_json = json::parse(&manifest_str).expect("failed to parse manifest");
-    if toolchain.channel == "nightly" && toolchain.date.is_none() {
-        // HACK: no "/latest" for prereleases
-        man_json = man_json[0].take();
-    };
-    let urls = man_json["assets"].members().map(|j|
-        j["browser_download_url"].as_str().unwrap().to_string()).collect();
-
-    Ok(urls)
+fn toolchain_url<'a>(download: DownloadCfg<'a>, toolchain: &ToolchainDesc) -> Result<String> {
+    Ok(match (toolchain.date.as_ref(), toolchain.channel.as_str()) {
+        (None, version) if version == "stable" || version == "nightly" => {
+            (download.notify_handler)(Notification::DownloadingManifest(version));
+            let repo = if version == "stable" { "leanprover/lean" } else { "leanprover/lean-nightly" };
+            let release = utils::fetch_latest_release_tag(repo)?;
+            (download.notify_handler)(Notification::DownloadedManifest(version, Some(&release)));
+            format!("https://github.com/{}/releases/tag/{}", repo, release)
+        }
+        (Some(date), "nightly") =>
+            format!("https://github.com/leanprover/lean-nightly/releases/tag/nightly-{}", date),
+        (None, version) =>
+            format!("https://github.com/leanprover/lean/releases/tag/v{}", version),
+        _ => panic!("wat"),
+    })
 }

src/elan-dist/src/manifestation.rs

@@ -301,24 +301,9 @@ impl Manifestation {
 
     /// Installation using the legacy v1 manifest format
     pub fn update_v1(&self,
-                     new_manifest: &[String],
+                     url: &String,
                      temp_cfg: &temp::Cfg,
                      notify_handler: &Fn(Notification)) -> Result<()> {
-        let informal_target = match self.target_triple.0.as_str() {
-            "x86_64-unknown-linux-gnu" => Some("linux"),
-            "x86_64-apple-darwin" => Some("darwin"),
-            "x86_64-pc-windows-msvc" => Some("windows"),
-            _ => None,
-        };
-        let url = new_manifest.iter().find(|u|
-            informal_target.map(|t| u.contains(t)).unwrap_or(false));
-        if url.is_none() {
-            return Err(format!("binary package was not provided for '{}'",
-                               self.target_triple.to_string()).into());
-        }
-        // Only replace once. The cost is inexpensive.
-        let url = url.unwrap().replace(DEFAULT_DIST_SERVER, temp_cfg.dist_server.as_str());
-
         notify_handler(Notification::DownloadingComponent("lean",
                                                           &self.target_triple,
                                                           Some(&self.target_triple)));
@@ -332,6 +317,28 @@ impl Manifestation {
             notify_handler: notify_handler
         };
 
+        // find correct download on HTML page (AAAAH)
+        use std::fs;
+        use regex::Regex;
+        use std::io::Read;
+        let informal_target = match self.target_triple.0.as_str() {
+            "x86_64-unknown-linux-gnu" => Some("linux"),
+            "x86_64-apple-darwin" => Some("darwin"),
+            "x86_64-pc-windows-msvc" => Some("windows"),
+            _ => None,
+        };
+        let re = Regex::new(r#"/leanprover/[a-z-]+/releases/download/[^"]+"#).unwrap();
+        let download_page_file = dlcfg.download_and_check(&url, "")?;
+        let mut html = String::new();
+        fs::File::open(&download_page_file as &::std::path::Path)?.read_to_string(&mut html)?;
+        let url = re.find_iter(&html).map(|m| m.as_str().to_string()).find(|m|
+            informal_target.map(|t| m.as_str().contains(t)).unwrap_or(false));
+        if url.is_none() {
+            return Err(format!("binary package was not provided for '{}'",
+                               self.target_triple.to_string()).into());
+        }
+        let url = format!("https://github.com/{}", url.unwrap());
+
         let ext = if cfg!(target_os = "linux") { ".tar.gz" } else { ".zip" };
         let installer_file = try!(dlcfg.download_and_check(&url, ext));
 

src/elan-utils/Cargo.toml

@@ -18,6 +18,8 @@ semver = "0.9.0"
 sha2 = "0.7.0"
 toml = "0.4"
 url = "1.1"
+curl = "0.4.6"
+regex = "0.2.0"
 
 [target."cfg(windows)".dependencies]
 winapi = { version = "0.3", features = ["combaseapi", "errhandlingapi", "fileapi", "handleapi", 

src/elan-utils/src/lib.rs

@@ -9,6 +9,8 @@ extern crate url;
 extern crate toml;
 extern crate download;
 extern crate semver;
+extern crate curl;
+extern crate regex;
 
 #[cfg(windows)]
 extern crate winapi;

src/elan-utils/src/utils.rs

@@ -615,6 +615,32 @@ pub fn toolchain_sort<T: AsRef<str>>(v: &mut Vec<T>) {
     });
 }
 
+// fetch from HTML page instead of Github API to avoid rate limit
+pub fn fetch_latest_release_tag(repo_slug: &str) -> Result<String> {
+    use regex::Regex;
+    use curl::easy::Easy;
+
+    let latest_url = format!("https://github.com/{}/releases/latest", repo_slug);
+
+    let mut data = Vec::new();
+    let mut handle = Easy::new();
+    handle.url(&latest_url).unwrap();
+    {
+        let mut transfer = handle.transfer();
+        transfer.write_function(|new_data| {
+            data.extend_from_slice(new_data);
+            Ok(new_data.len())
+        }).unwrap();
+        transfer.perform().unwrap();
+    }
+    let redirect = ::std::str::from_utf8(&data).chain_err(|| "failed to decode release tag response")?;
+    let re = Regex::new(r#"/tag/([^"]+)"#).unwrap();
+    let capture = re.captures(&redirect);
+    match capture {
+        Some(cap) => Ok(cap.get(1).unwrap().as_str().to_string()),
+        None => Err("failed to parse latest release tag".into()),
+    }
+}
 
 #[cfg(test)]
 mod tests {