diff --git a/Cargo.lock b/Cargo.lock index fd532bef..0006daa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,6 +1195,7 @@ version = "0.1.0" dependencies = [ "actix-web", "async-fs", + "base64 0.21.4", "chrono", "derive_builder", "eyre", @@ -2345,6 +2346,7 @@ dependencies = [ "once_cell", "path-absolutize", "rayon", + "secrecy", "serde", "serde_json", "simple-eyre", diff --git a/crates/diffbot_lib/Cargo.toml b/crates/diffbot_lib/Cargo.toml index a810a8d4..1da41d08 100644 --- a/crates/diffbot_lib/Cargo.toml +++ b/crates/diffbot_lib/Cargo.toml @@ -26,3 +26,4 @@ flume = "0.11.0" actix-web = "4.4.0" async-fs = "1.6.0" +base64 = "0.21.4" diff --git a/crates/diffbot_lib/src/github/github_api.rs b/crates/diffbot_lib/src/github/github_api.rs index 8b570551..83dc51ed 100644 --- a/crates/diffbot_lib/src/github/github_api.rs +++ b/crates/diffbot_lib/src/github/github_api.rs @@ -1,14 +1,12 @@ use crate::github::github_types::{ CreateCheckRun, Output, RawCheckRun, Repository, UpdateCheckRunBuilder, }; -use async_fs::File; use eyre::{format_err, Context, Result}; -use futures_lite::io::AsyncWriteExt; use octocrab::models::repos::Content; use octocrab::models::InstallationId; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; use std::{future::Future, pin::Pin}; +use base64::{Engine as _, engine::general_purpose}; pub struct GithubEvent(pub String, pub Option>); @@ -207,8 +205,6 @@ impl CheckRun { } } -static DOWNLOAD_DIR: &str = "download"; - async fn find_content>( installation: &InstallationId, repo: &Repository, @@ -244,34 +240,57 @@ pub async fn download_url>( ) -> Result> { let target = find_content(installation, repo, filename, commit).await?; - let download_url = target - .download_url - .as_ref() - .ok_or_else(|| format_err!("No download URL given by GitHub"))?; - - let response = reqwest::get(download_url).await?; + let content = target.content + .ok_or_else(|| format_err!("File had no content!"))? + .replace("\n", ""); - Ok(response.bytes().await?.to_vec()) + general_purpose::STANDARD + .decode(content) + .map_err(|decode_error| format_err!("DecodeError: {}", decode_error)) } -pub async fn download_file>( - installation: &InstallationId, - repo: &Repository, - filename: S, - commit: S, -) -> Result { - let target = find_content(installation, repo, &filename, &commit).await?; - - let mut path = PathBuf::new(); - path.push("."); - path.push(DOWNLOAD_DIR); - path.push(&target.sha); - path.set_extension("dmi"); // Method should have an IDB qualifier due to being a shared crate - - async_fs::create_dir_all(path.parent().unwrap()).await?; - let mut file = File::create(&path).await?; - - let data = download_url(installation, repo, &filename, &commit).await?; - file.write_all(&data).await?; - Ok(path) +/* local test requires commenting out the .installation(...) call in find_content(), a valid github token with access, and the following dep: actix-rt = "2.9.0" + +#[actix_web::rt::test] +async fn test_private_repo_file_download() { + octocrab::initialise(octocrab::OctocrabBuilder::new() + .personal_token("lol".to_owned()) + .build() + .unwrap()); + + let bytes = download_url( + &InstallationId(0), + &Repository{ + url: "https://api.github.com/repos/Cyberboss/tgstation-private-test".to_owned(), + id:0, + }, ".tgs.yml", "140c79189849ea616f09b3484f8930211d3705cd").await.unwrap(); + + let text = std::str::from_utf8(bytes.as_slice()).unwrap(); + assert_eq!(r#"# This file is used by TGS (https://github.com/tgstation/tgstation-server) clients to quickly initialize a server instance for the codebase +# The format isn't documented anywhere but hopefully we never have to change it. If there are questions, contact the TGS maintainer Cyberboss/@Dominion#0444 +version: 1 +# The BYOND version to use (kept in sync with dependencies.sh by the "TGS Test Suite" CI job) +# Must be interpreted as a string, keep quoted +byond: "514.1588" +# Folders to create in "/Configuration/GameStaticFiles/" +static_files: + # Config directory should be static + - name: config + # This implies the folder should be pre-populated with contents from the repo + populate: true + # Data directory must be static + - name: data +# String dictionary. The value is the location of the file in the repo to upload to TGS. The key is the name of the file to upload to "/Configuration/EventScripts/" +# This one is for Linux hosted servers +linux_scripts: + PreCompile.sh: tools/tgs_scripts/PreCompile.sh + WatchdogLaunch.sh: tools/tgs_scripts/WatchdogLaunch.sh + InstallDeps.sh: tools/tgs_scripts/InstallDeps.sh +# Same as above for Windows hosted servers +windows_scripts: + PreCompile.bat: tools/tgs_scripts/PreCompile.bat +# The security level the game should be run at +security: Trusted +"#, text) } +*/ diff --git a/crates/mapdiffbot2/Cargo.toml b/crates/mapdiffbot2/Cargo.toml index c48e7c35..006b63dd 100644 --- a/crates/mapdiffbot2/Cargo.toml +++ b/crates/mapdiffbot2/Cargo.toml @@ -39,6 +39,7 @@ tokio = { version = "1.32.0", features = ["io-util", "rt"] } mysql_async = "0.32.2" time = "0.3.28" +secrecy = "0.8.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.5.4" diff --git a/crates/mapdiffbot2/src/git_operations.rs b/crates/mapdiffbot2/src/git_operations.rs index ac83ede6..85098b85 100644 --- a/crates/mapdiffbot2/src/git_operations.rs +++ b/crates/mapdiffbot2/src/git_operations.rs @@ -21,11 +21,11 @@ pub fn fetch_and_get_branches<'a>( remote .fetch( - &[base_branch_name], + &[base_branch_name, head_branch_name], Some(FetchOptions::new().prune(git2::FetchPrune::On)), None, ) - .wrap_err("Fetching base")?; + .wrap_err("Fetching base and head")?; let fetch_head = repo .find_reference("FETCH_HEAD") .wrap_err("Getting FETCH_HEAD")?; @@ -67,14 +67,6 @@ pub fn fetch_and_get_branches<'a>( .resolve_reference_from_short_name(base_branch_name) .wrap_err("Getting the base reference")?; - remote - .fetch( - &[head_branch_name], - Some(FetchOptions::new().prune(git2::FetchPrune::On)), - None, - ) - .wrap_err("Fetching head")?; - let fetch_head = repo .find_reference("FETCH_HEAD") .wrap_err("Getting FETCH_HEAD")?; diff --git a/crates/mapdiffbot2/src/job_processor.rs b/crates/mapdiffbot2/src/job_processor.rs index be5db1f1..2f61b266 100644 --- a/crates/mapdiffbot2/src/job_processor.rs +++ b/crates/mapdiffbot2/src/job_processor.rs @@ -1,5 +1,6 @@ use eyre::{Context, Result}; use path_absolutize::Absolutize; +use secrecy::ExposeSecret; use std::path::Path; use std::path::PathBuf; @@ -342,12 +343,16 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result { let base = &job.base; let head = &job.head; - let repo = format!("https://github.com/{}", job.repo.full_name()); - let repo_dir: PathBuf = ["./repos/", &job.repo.full_name()].iter().collect(); let handle = actix_web::rt::Runtime::new()?; + let (_, secret_token) = handle.block_on(octocrab::instance() + .installation_and_token(job.installation))?; + + let repo_dir: PathBuf = ["./repos/", &job.repo.full_name()].iter().collect(); - if !repo_dir.exists() { + let url = format!("https://x-access-token:{}@github.com/{}", secret_token.expose_secret(), job.repo.full_name()); + let clone_required = !repo_dir.exists(); + if clone_required { tracing::debug!("Directory {:?} doesn't exist, creating dir", repo_dir); std::fs::create_dir_all(&repo_dir)?; handle.block_on(async { @@ -358,7 +363,7 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result { }; let _ = job.check_run.set_output(output).await; // we don't really care if updating the job fails, just continue }); - clone_repo(&repo, &repo_dir).wrap_err("Cloning repo")?; + clone_repo(&url, &repo_dir).wrap_err("Cloning repo")?; } let non_abs_directory: PathBuf = [ @@ -392,6 +397,10 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result { let repository = git2::Repository::open(&repo_dir).wrap_err("Opening repository")?; + if !clone_required { + repository.remote_set_url("origin", &url)?; + } + let mut remote = repository.find_remote("origin")?; remote @@ -412,7 +421,7 @@ pub fn do_job(job: Job, blob_client: Azure) -> Result { (&added_files, &modified_files, &removed_files), (&repository, &job.base.r#ref), (&repo_dir, output_directory, blob_client), - job.pull_request, + job.pull_request ) .wrap_err("") {