Merge pull request #30 from jupyterkat/update

Fixes and updates

crates/diffbot_lib/Cargo.toml

@@ -5,25 +5,19 @@ edition = "2021"
 license = "MIT"
 
 [dependencies]
-serde = { version = "1.0.188", features = ["derive"] }
-serde_json = "1.0.107"
-octocrab = "0.30.1"
-eyre = "0.6.8"
-derive_builder = "0.12.0"
-chrono = "0.4.30"
-reqwest = "0.11.20"
-futures-lite = "1.13.0"
-simple_logger = "4.2.0"
-sha2 = "0.10.7"
+serde = { version = "1.0.206", features = ["derive"] }
+octocrab = "0.39.0"
+eyre = "0.6.12"
+derive_builder = "0.20.0"
+chrono = "0.4.38"
+sha2 = "0.10.8"
 hmac = "0.12.1"
 hex = "0.4.3"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["fmt", "time"] }
-tracing-panic = "0.1.1"
-tracing-loki = "0.2.4"
+tracing-panic = "0.1.2"
+tracing-loki = "0.2.5"
 flume = "0.11.0"
+async-fs = "2.1.2"
 
-actix-web = "4.4.0"
-
-async-fs = "1.6.0"
-base64 = "0.21.4"
+actix-web = "4.9.0"

crates/diffbot_lib/src/github/github_api.rs

@@ -1,12 +1,8 @@
-use crate::github::github_types::{
-    CreateCheckRun, Output, RawCheckRun, Repository, UpdateCheckRunBuilder,
-};
-use eyre::{format_err, Context, Result};
-use octocrab::models::repos::Content;
+use crate::github::github_types::{CreateCheckRun, Output, RawCheckRun, UpdateCheckRunBuilder};
+use eyre::{Context, Result};
 use octocrab::models::InstallationId;
 use serde::{Deserialize, Serialize};
 use std::{future::Future, pin::Pin};
-use base64::{Engine as _, engine::general_purpose};
 
 pub struct GithubEvent(pub String, pub Option<Vec<u8>>);
 
@@ -205,50 +201,6 @@ impl CheckRun {
     }
 }
 
-async fn find_content<S: AsRef<str>>(
-    installation: &InstallationId,
-    repo: &Repository,
-    filename: S,
-    commit: S,
-) -> Result<Content> {
-    let (owner, repo) = repo.name_tuple();
-    let items = octocrab::instance()
-        .installation(*installation)
-        .repos(owner, repo)
-        .get_content()
-        .path(filename.as_ref())
-        .r#ref(commit.as_ref())
-        .send()
-        .await?
-        .take_items();
-
-    if items.len() > 1 {
-        return Err(format_err!("Directory given to find_content"));
-    }
-
-    items
-        .into_iter()
-        .next()
-        .ok_or_else(|| format_err!("No content was found"))
-}
-
-pub async fn download_url<S: AsRef<str>>(
-    installation: &InstallationId,
-    repo: &Repository,
-    filename: S,
-    commit: S,
-) -> Result<Vec<u8>> {
-    let target = find_content(installation, repo, filename, commit).await?;
-
-    let content = target.content
-        .ok_or_else(|| format_err!("File had no content!"))?
-        .replace("\n", "");
-
-    general_purpose::STANDARD
-        .decode(content)
-        .map_err(|decode_error| format_err!("DecodeError: {}", decode_error))
-}
-
 /* local test requires commenting out the .installation(...) call in find_content(), a valid github token with access, and the following dep: actix-rt = "2.9.0"
 
 #[actix_web::rt::test]

crates/icondiffbot2/Cargo.toml

@@ -6,35 +6,32 @@ license = "MIT"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde = { version = "1.0.188", features = ["derive"] }
-serde_json = "1.0.107"
-octocrab = "0.30.1"
+serde = { version = "1.0.206", features = ["derive"] }
+serde_json = "1.0.124"
+octocrab = "0.39.0"
 dmm-tools = { git = "https://github.com/jupyterkat/SpacemanDMM/" }
-dreammaker = { git = "https://github.com/jupyterkat/SpacemanDMM/" }
-once_cell = "1.18.0"
-jsonwebtoken = "8.3.0"
-reqwest = "0.11.20"
+jsonwebtoken = "9.3.0"
+reqwest = "0.12.5"
 diffbot_lib = { path = "../diffbot_lib" }
-eyre = "0.6.8"
+eyre = "0.6.12"
 simple-eyre = "0.3.1"
-ndarray = "0.15.6"
-rayon = "1.7.0"
-toml = "0.8.0"
-ahash = "0.8.3"
-hashbrown = { version = "0.14.0", features = ["rayon"] }
-hex = { version = "0.4.3", features = ["serde"] }
-tracing-loki = "0.2.4"
+rayon = "1.10.0"
+toml = "0.8.19"
+ahash = "0.8.11"
+hashbrown = { version = "0.14.5", features = ["rayon"] }
+tracing-loki = "0.2.5"
 flume = "0.11.0"
+mysql_async = "0.34.1"
+time = "0.3.36"
+secrecy = "0.8.0"
+percent-encoding = "2.3.1"
 
-actix-web = "4.4.0"
-actix-files = "0.6.2"
-
-mysql_async = "0.32.2"
-time = "0.3.28"
+actix-web = "4.9.0"
+actix-files = "0.6.6"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = "0.5.4"
+tikv-jemallocator = "0.6.0"
 
 [dependencies.tracing]
-version = "0.1.37"
+version = "0.1.40"
 features = ["attributes"]

crates/icondiffbot2/config.example.toml

@@ -5,6 +5,9 @@ blacklist_contact = "Good luck!"
 # Logging level (Optional, defaults to info), Valid values are "info", "warn", "trace", "error", "debug"
 logging = "info"
 
+# Summary message, the message that goes before the diff, (Optional, defaults to below)
+# summary_msg = "*Please file any issues [here](https://github.com/spacestation13/BYONDDiffBots/issues).*\n\nIcons with diff:"
+
 # Mysql db url, the bots will write down a table that contains necessary information,
 # for automated deletion (Optional)
 # db_url = ""

crates/icondiffbot2/src/downloading.rs

@@ -0,0 +1,72 @@
+use diffbot_lib::github::github_types::Repository;
+use eyre::Result;
+use octocrab::models::repos::Content;
+use octocrab::models::InstallationId;
+use secrecy::ExposeSecret;
+
+//https://url.spec.whatwg.org/#c0-control-percent-encode-set
+const PATH_ENCODING: &percent_encoding::AsciiSet = &percent_encoding::CONTROLS
+    //query
+    .add(b' ')
+    .add(b'"')
+    .add(b'<')
+    .add(b'>')
+    .add(b'`')
+    //path
+    .add(b'?')
+    .add(b'}')
+    .add(b'{');
+
+async fn find_content<S: AsRef<str>>(
+    installation: &InstallationId,
+    repo: &Repository,
+    filename: S,
+    commit: S,
+) -> Result<Content> {
+    let (owner, repo) = repo.name_tuple();
+    let items = octocrab::instance()
+        .installation(*installation)
+        .repos(owner, repo)
+        .get_content()
+        .path(
+            percent_encoding::percent_encode(filename.as_ref().as_bytes(), PATH_ENCODING)
+                .to_string(),
+        )
+        .r#ref(commit.as_ref())
+        .send()
+        .await?
+        .take_items();
+
+    if items.len() > 1 {
+        return Err(eyre::eyre!("Directory given to find_content"));
+    }
+
+    items
+        .into_iter()
+        .next()
+        .ok_or_else(|| eyre::eyre!("No content was found"))
+}
+
+pub async fn download_url<S: AsRef<str>>(
+    installation: &InstallationId,
+    repo: &Repository,
+    filename: S,
+    commit: S,
+    client: reqwest::Client,
+) -> Result<Vec<u8>> {
+    let target = find_content(installation, repo, filename, commit).await?;
+
+    let download_url = target
+        .download_url
+        .as_ref()
+        .ok_or_else(|| eyre::eyre!("No download URL given by GitHub"))?;
+    let (_, token) = octocrab::instance()
+        .installation_and_token(*installation)
+        .await?;
+    let response = client
+        .get(download_url)
+        .bearer_auth(token.expose_secret())
+        .send()
+        .await?;
+    Ok(response.bytes().await?.to_vec())
+}

crates/icondiffbot2/src/job_processor.rs

@@ -17,7 +17,7 @@ use std::{
 };
 
 #[tracing::instrument]
-pub fn do_job(job: Job) -> Result<CheckOutputs> {
+pub fn do_job(job: Job, client: reqwest::Client) -> Result<CheckOutputs> {
     let handle = actix_web::rt::Runtime::new()?;
 
     handle.block_on(async { job.check_run.mark_started().await })?;
@@ -28,16 +28,20 @@ pub fn do_job(job: Job) -> Result<CheckOutputs> {
         .iter()
         .map(|dmi| {
             (
-                sha_to_iconfile(&job, &dmi.filename, status_to_sha(&job, &dmi.status)),
+                sha_to_iconfile(
+                    &job,
+                    &dmi.filename,
+                    status_to_sha(&job, &dmi.status),
+                    client.clone(),
+                ),
                 dmi,
             )
         })
-        .map(|(file, dmi)| -> Result<()> {
+        .try_for_each(|(file, dmi)| -> Result<()> {
             let states = render(&job, file?)?;
             map.insert(dmi.filename.as_str(), states);
             Ok(())
-        })
-        .collect::<Result<()>>()?;
+        })?;
 
     map.build()
 }
@@ -70,7 +74,7 @@ fn render(
                     env!("CARGO_MANIFEST_DIR"),
                     "/templates/diff_line_error.txt"
                 )),
-                error = format_args!("Before icon render failed:\n```{e:?}```"),
+                error = format_args!("Before icon render failed:\n{e:?}"),
             )],
         )),
 
@@ -122,26 +126,24 @@ fn render(
                 .metadata
                 .states
                 .iter()
-                .map(|(name, vec)| {
+                .flat_map(|(name, vec)| {
                     vec.iter()
                         .enumerate()
                         .map(|(duplication_index, _)| (duplication_index, name.as_str()))
                         .collect::<Vec<_>>()
                 })
-                .flatten()
                 .collect();
             let after_states: HashSet<(usize, &str), ahash::RandomState> = after
                 .icon
                 .metadata
                 .states
                 .iter()
-                .map(|(name, vec)| {
+                .flat_map(|(name, vec)| {
                     vec.iter()
                         .enumerate()
                         .map(|(duplication_index, _)| (duplication_index, name.as_str()))
                         .collect::<Vec<_>>()
                 })
-                .flatten()
                 .collect();
 
             let prefix = format!("{}/{}", job.installation, job.pull_request);

crates/icondiffbot2/src/main.rs

@@ -1,3 +1,4 @@
+mod downloading;
 mod github_processor;
 mod job_processor;
 mod runner;
@@ -10,8 +11,8 @@ use diffbot_lib::{
 };
 use mysql_async::prelude::Queryable;
 use octocrab::OctocrabBuilder;
-use once_cell::sync::OnceCell;
 use serde::Deserialize;
+use std::sync::OnceLock;
 use std::{
     fs::File,
     io::Read,
@@ -64,6 +65,8 @@ pub struct Config {
     pub blacklist_contact: String,
     #[serde(default = "default_log_level")]
     pub logging: String,
+    #[serde(default = "default_msg")]
+    pub summary_msg: String,
     pub secret: Option<String>,
     pub db_url: Option<String>,
     pub grafana_loki: Option<GrafanaLoki>,
@@ -73,7 +76,11 @@ fn default_log_level() -> String {
     "info".to_string()
 }
 
-static CONFIG: OnceCell<Config> = OnceCell::new();
+fn default_msg() -> String {
+    "*Please file any issues [here](https://github.com/spacestation13/BYONDDiffBots/issues).*\n\nIcons with diff:".to_string()
+}
+
+static CONFIG: OnceLock<Config> = OnceLock::new();
 // static FLAME_LAYER_GUARD: OnceCell<tracing_flame::FlushGuard<std::io::BufWriter<File>>> =
 // OnceCell::new();
 
@@ -87,6 +94,10 @@ fn init_config(path: &Path) -> eyre::Result<&'static Config> {
     Ok(CONFIG.get().unwrap())
 }
 
+fn read_config() -> &'static Config {
+    CONFIG.get().unwrap()
+}
+
 // fn init_global_subscriber() {
 //     use tracing_subscriber::prelude::*;
 
@@ -109,7 +120,7 @@ fn read_key(path: &Path) -> Vec<u8> {
         File::open(path).unwrap_or_else(|_| panic!("Unable to find file {}", path.display()));
 
     let mut key = Vec::new();
-    let _ = key_file
+    _ = key_file
         .read_to_end(&mut key)
         .unwrap_or_else(|_| panic!("Failed to read key {}", path.display()));
 
@@ -151,6 +162,7 @@ async fn main() -> eyre::Result<()> {
             .build()
             .expect("fucked up octocrab"),
     );
+    let reqwest_client = reqwest::Client::new();
 
     async_fs::create_dir_all("./images").await.unwrap();
 
@@ -178,7 +190,11 @@ async fn main() -> eyre::Result<()> {
         .await?;
     }
 
-    actix_web::rt::spawn(runner::handle_jobs("IconDiffBot2", job_receiver));
+    actix_web::rt::spawn(runner::handle_jobs(
+        "IconDiffBot2",
+        job_receiver,
+        reqwest_client,
+    ));
 
     let job_sender: DataJobSender = actix_web::web::Data::new(job_sender);