From fb0705bf0d66cf033b2429a812e3edeb8493553e Mon Sep 17 00:00:00 2001 From: Erik Reinert <4638629+erikreinert@users.noreply.github.com> Date: Sat, 8 Jun 2024 12:49:32 -0700 Subject: [PATCH] Implements basic local and remote sources support (#17) * feat: support local directories * feat(example): add support for local compressed file in vorpal example - Updated `example/rust/vorpal.rs` to include a new example for building from a local compressed file. - Changed the name of the existing example from "example-source" to "example-local-directory". - Modified `src/service/proxy/package/mod.rs` to simplify the call to `store::unpack_source` by removing unnecessary conversion to `PathBuf`. * refactor(example): rename package and remove local compressed file example Renamed the package from "example-local-directory" to "example-rust" for better clarity. Removed the example of a local compressed file package to streamline the code and reduce redundancy. * feat: add new dependencies and update package preparation and build process - Added new dependencies in Cargo.toml and Cargo.lock including `futures-util`, `git2`, `reqwest`, `tokio-stream`, `url`, and others. - Updated `PackageService` to handle streaming `PrepareRequest`. - Modified `prepare` function to handle different source types (Git, HTTP, Local) and prepare the source accordingly. - Implemented `prepare_source` function to sign and send source data in chunks. - Updated `build` function to handle new source preparation and build process. - Renamed `vorpal.rs` to `build.rs` and updated its content to reflect new build process. - Updated `flake.nix` to include new dependencies and build inputs. - Refactored store module functions to use consistent naming and paths. - Added new functions in notary module for signing data. - Updated service modules to use new store path functions and handle new build process. - Improved error handling and logging throughout the codebase. * refactor(service): simplify error handling in build preparation Simplified the error handling in the build preparation module by using the `map_err` method for more concise and readable code. Removed redundant error logging statements. This change improves code maintainability and readability. * feat: add rust-overlay and cargo-udeps to flake configuration - Added `rust-overlay` input to `flake.nix` and `flake.lock`. - Included `cargo-udeps` in the list of inherited packages. - Updated `devShells` to include `cargo-udeps` in `nativeBuildInputs`. - Configured `rust-overlay` to follow `nixpkgs`. - Added `cargo-nightly` application to `process-compose` configuration. * refactor: rename build.rs to vorpal.rs and update references - Renamed `build.rs` to `vorpal.rs` in `example/rust` directory. - Updated `Cargo.toml` to reflect the new binary name and path. - Modified `flake.nix` to include `clippy` and `rustfmt` in `nativeBuildInputs`. - Replaced `.clone()` with dereferencing in `src/command/mod.rs`. - Simplified function calls by removing unnecessary references in `src/notary/mod.rs`, `src/service/build/build/mod.rs`, `src/service/build/prepare/mod.rs`, `src/service/proxy/package/mod.rs`, and `src/store/mod.rs`. - Improved code readability by using shorthand syntax for increment operations and string concatenations. * refactor: rename modules and update build script - Renamed `build/mod.rs` to `run_build.rs` - Renamed `prepare/mod.rs` to `run_prepare.rs` - Renamed `build/sandbox_default.rs` to `sandbox_default.rs` - Updated `service.rs` to use new module names - Simplified build script generation in `run_build.rs` - Removed redundant error logging in `run_build.rs` - Changed function parameters from `String` to `&str` in `proxy/package/mod.rs` and `store/mod.rs` - Added unit tests for store directory paths in `store/mod.rs` - Updated `flake.nix` to include clippy checks and tests in the build phase and removed clippy from devShells. * refactor: update flake.nix and run_prepare.rs for improved build process - Updated `checkPhase` in `flake.nix` to include `cargo fmt --check --verbose` and modified `cargo test` command to use `--locked --all-features --all-targets`. - Added `rustfmt` to `nativeBuildInputs` in `flake.nix`. - Removed `rustfmt` from `devShells.default.nativeBuildInputs` in `flake.nix`. - Removed `cargo-nightly` app configuration from `apps` in `flake.nix`. - Simplified the creation of `PrepareResponse` in `run_prepare.rs`. * chore: update dependencies - Bump `addr2line` from 0.21.0 to 0.22.0 - Bump `anstyle-query` from 1.0.3 to 1.1.0 - Bump `hyper` from 0.14.28 to 0.14.29 - Bump `backtrace` from 0.3.71 to 0.3.72 - Bump `cc` from 1.0.98 to 1.0.99 - Bump `clap` from 4.5.4 to 4.5.6 - Bump `clap_builder` from 4.5.2 to 4.5.6 - Bump `clap_derive` from 4.5.4 to 4.5.5 - Bump `clap_lex` from 0.7.0 to 0.7.1 - Bump `gimli` from 0.28.1 to 0.29.0 - Bump `object` from 0.32.2 to 0.35.0 - Bump `proc-macro2` from 1.0.84 to 1.0.85 - Bump `tar` from 0.4.40 to 0.4.41 - Bump `tokio` from 1.37.0 to 1.38.0 - Bump `tokio-macros` from 2.2.0 to 2.3.0 - Bump `utf8parse` from 0.2.1 to 0.2.2 Remove `cargo-udeps` from `flake.nix` and update `cargoSha256`. --- Cargo.lock | 763 ++++++++++++++++-- Cargo.toml | 8 +- api/v0/build/build.proto | 17 +- api/v0/package/package.proto | 2 +- example/rust/Cargo.lock | 699 +++++++++++++++- example/rust/Cargo.toml | 2 +- example/rust/vorpal.rs | 62 +- flake.lock | 57 +- flake.nix | 21 +- src/command/mod.rs | 4 +- src/notary/mod.rs | 13 +- src/service/build/mod.rs | 12 +- .../build/{build/mod.rs => run_build.rs} | 72 +- .../build/{prepare/mod.rs => run_prepare.rs} | 114 +-- .../build/{build => }/sandbox_default.rs | 0 src/service/build/service.rs | 10 +- src/service/proxy/mod.rs | 13 +- src/service/proxy/package/mod.rs | 283 +++++-- src/store/mod.rs | 113 ++- 19 files changed, 1941 insertions(+), 324 deletions(-) rename src/service/build/{build/mod.rs => run_build.rs} (79%) rename src/service/build/{prepare/mod.rs => run_prepare.rs} (50%) rename src/service/build/{build => }/sandbox_default.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 80d91423..32f4f19a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ "windows-sys 0.52.0", ] @@ -132,6 +132,12 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" @@ -149,9 +155,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -175,8 +181,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -185,9 +191,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -204,6 +210,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -241,6 +253,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" @@ -255,9 +273,14 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -267,9 +290,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.4" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "a9689a29b593160de5bc4aacab7b5d54fb52231de70122626c178e6a368994c7" dependencies = [ "clap_builder", "clap_derive", @@ -277,9 +300,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "2e5387378c84f6faa26890ebf9f0a92989f8873d4d380467bcd0d8d8620424df" dependencies = [ "anstream", "anstyle", @@ -289,9 +312,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck", "proc-macro2", @@ -301,9 +324,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "colorchoice" @@ -317,6 +340,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -419,6 +458,15 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -461,7 +509,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -487,6 +535,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -502,6 +574,17 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -521,9 +604,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -549,9 +634,24 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + +[[package]] +name = "git2" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" +dependencies = [ + "bitflags 2.5.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] [[package]] name = "globset" @@ -588,7 +688,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", "indexmap 2.2.6", "slab", "tokio", @@ -649,6 +768,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -656,7 +786,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -674,17 +827,17 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -696,18 +849,84 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "ignore" version = "0.4.22" @@ -744,6 +963,18 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "infer" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865e8a58ae8e24d2c4412c31344afa1d302a3740ad67528c10f50d6876cdcf55" + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -765,6 +996,24 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -780,6 +1029,20 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libgit2-sys" +version = "0.16.2+1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.8" @@ -807,12 +1070,48 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.21" @@ -863,6 +1162,23 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -922,9 +1238,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -935,12 +1251,79 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.1", + "smallvec", + "windows-targets 0.52.5", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1088,9 +1471,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -1196,6 +1579,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "redox_users" version = "0.4.5" @@ -1236,6 +1628,48 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rsa" version = "0.9.6" @@ -1290,6 +1724,22 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustversion" version = "1.0.17" @@ -1311,6 +1761,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.203" @@ -1342,6 +1830,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1455,11 +1955,32 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" dependencies = [ "filetime", "libc", @@ -1514,17 +2035,33 @@ dependencies = [ "syn", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -1544,15 +2081,25 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -1586,12 +2133,12 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64", + "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -1748,17 +2295,43 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" @@ -1790,9 +2363,13 @@ dependencies = [ "clap", "dirs", "flate2", + "futures-util", + "git2", "hex", + "infer", "prost", "rand", + "reqwest", "rsa", "rusqlite", "sha256", @@ -1800,8 +2377,10 @@ dependencies = [ "tempfile", "tera", "tokio", + "tokio-stream", "tonic", "tonic-build", + "url", "uuid", "walkdir", ] @@ -1831,6 +2410,82 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.8" @@ -1979,6 +2634,16 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xattr" version = "1.3.1" diff --git a/Cargo.toml b/Cargo.toml index 95c02a7d..6745c225 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,17 +12,23 @@ anyhow = "1.0" clap = { version = "4.5", features = ["derive"] } dirs = "5.0" flate2 = "1.0" +futures-util = "0.3" +git2 = "0.18" hex = "0.4" +infer = "0.3" prost = "0.12" rand = "0.8" rsa = { version = "0.9", features = ["sha2"] } rusqlite = { version = "0.31", features = ["bundled"] } sha256 = "1.5" +reqwest = { version = "0.12", features = ["json"] } tar = "0.4" tempfile = "3.10" tera = { version = "1", default-features = false } -tokio = { version = "1.37", features = ["macros", "process", "rt-multi-thread"] } +tokio = { version = "1.37", features = ["full"] } +tokio-stream = "0.1" tonic = "0.11" +url = "2.5" uuid = { version = "1.8", features = ["v7"] } walkdir = "2.5" diff --git a/api/v0/build/build.proto b/api/v0/build/build.proto index 7559715a..f9d711c5 100644 --- a/api/v0/build/build.proto +++ b/api/v0/build/build.proto @@ -6,6 +6,20 @@ service BuildService { rpc Package (PackageRequest) returns (PackageResponse); } +enum PackageSourceKind { + UNKNOWN = 0; + LOCAL = 1; + HTTP = 2; + GIT = 3; +} + +message PackageSource { + PackageSourceKind kind = 1; + optional string hash = 2; + repeated string ignore_paths = 3; + string uri = 4; +} + message PackageResponse { string source_id = 1; string source_hash = 2; @@ -14,9 +28,8 @@ message PackageResponse { message PackageRequest { repeated PackageResponse build_deps = 1; repeated PackageResponse install_deps = 2; - repeated string ignore_paths = 3; string build_phase = 4; string install_phase = 5; string name = 6; - string source = 7; + PackageSource source = 7; } diff --git a/api/v0/package/package.proto b/api/v0/package/package.proto index 07f3d17f..e47ce976 100644 --- a/api/v0/package/package.proto +++ b/api/v0/package/package.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package vorpal.package.v0; service PackageService { - rpc Prepare (PrepareRequest) returns (PrepareResponse); + rpc Prepare (stream PrepareRequest) returns (PrepareResponse); rpc Build (BuildRequest) returns (BuildResponse); } diff --git a/example/rust/Cargo.lock b/example/rust/Cargo.lock index a6d2dacf..f3d23b59 100644 --- a/example/rust/Cargo.lock +++ b/example/rust/Cargo.lock @@ -132,6 +132,12 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" @@ -149,9 +155,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -175,8 +181,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -204,6 +210,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -241,6 +253,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" @@ -258,6 +276,11 @@ name = "cc" version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -317,6 +340,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -419,6 +458,15 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -461,7 +509,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -487,6 +535,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -502,6 +574,17 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -521,9 +604,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -553,6 +638,21 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "git2" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" +dependencies = [ + "bitflags 2.5.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "globset" version = "0.4.14" @@ -588,7 +688,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", "indexmap 2.2.6", "slab", "tokio", @@ -649,6 +768,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -656,7 +786,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -682,9 +835,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -696,18 +849,84 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "ignore" version = "0.4.22" @@ -744,6 +963,18 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "infer" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865e8a58ae8e24d2c4412c31344afa1d302a3740ad67528c10f50d6876cdcf55" + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -765,6 +996,24 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -780,6 +1029,20 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libgit2-sys" +version = "0.16.2+1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.8" @@ -807,12 +1070,48 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.21" @@ -863,6 +1162,23 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -935,12 +1251,79 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.1", + "smallvec", + "windows-targets 0.52.5", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1196,6 +1579,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "redox_users" version = "0.4.5" @@ -1236,6 +1628,48 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rsa" version = "0.9.6" @@ -1290,6 +1724,22 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustversion" version = "1.0.17" @@ -1311,6 +1761,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.203" @@ -1342,6 +1830,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1455,6 +1955,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" version = "0.4.40" @@ -1514,6 +2035,21 @@ dependencies = [ "syn", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.38.0" @@ -1525,6 +2061,7 @@ dependencies = [ "libc", "mio", "num_cpus", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -1553,6 +2090,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -1586,12 +2133,12 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64", + "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -1748,12 +2295,38 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -1790,9 +2363,13 @@ dependencies = [ "clap", "dirs", "flate2", + "futures-util", + "git2", "hex", + "infer", "prost", "rand", + "reqwest", "rsa", "rusqlite", "sha256", @@ -1800,8 +2377,10 @@ dependencies = [ "tempfile", "tera", "tokio", + "tokio-stream", "tonic", "tonic-build", + "url", "uuid", "walkdir", ] @@ -1840,6 +2419,82 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.8" @@ -1988,6 +2643,16 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xattr" version = "1.3.1" diff --git a/example/rust/Cargo.toml b/example/rust/Cargo.toml index 7c31cefb..e3da98c3 100644 --- a/example/rust/Cargo.toml +++ b/example/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2021" name = "vorpal-example" version = "0.1.0" -edition = "2021" [[bin]] name = "vorpal" diff --git a/example/rust/vorpal.rs b/example/rust/vorpal.rs index 2efaf882..afc281e7 100644 --- a/example/rust/vorpal.rs +++ b/example/rust/vorpal.rs @@ -1,61 +1,37 @@ use anyhow::Result; use std::env; use vorpal::api::build_service_client::BuildServiceClient; -use vorpal::api::PackageRequest; +use vorpal::api::{PackageRequest, PackageSource, PackageSourceKind}; #[tokio::main] pub async fn main() -> Result<(), anyhow::Error> { let mut client = BuildServiceClient::connect("http://[::1]:23151").await?; - let foo = client + let example_rust = client .package(PackageRequest { build_deps: Vec::new(), - build_phase: "echo \"foo\" >> foo.txt && cat foo.txt".to_string(), - ignore_paths: vec![ - ".direnv".to_string(), - ".git".to_string(), - "target".to_string(), - ], + build_phase: r#" + cat vorpal.rs + "# + .to_string(), + install_phase: r#" + mkdir -p $OUTPUT + cp vorpal.rs $OUTPUT/build.rs + "# + .to_string(), install_deps: Vec::new(), - install_phase: "cp foo.txt $OUTPUT".to_string(), - name: "foo".to_string(), - source: env::current_dir()?.to_string_lossy().to_string(), + name: "example-rust".to_string(), + source: Some(PackageSource { + hash: None, + ignore_paths: vec![".git".to_string(), "target".to_string()], + kind: PackageSourceKind::Local.into(), + uri: env::current_dir()?.to_string_lossy().to_string(), + }), }) .await? .into_inner(); - let bar = client - .package(PackageRequest { - build_deps: Vec::new(), - build_phase: "echo \"bar\" >> bar.txt && cat bar.txt".to_string(), - ignore_paths: vec![ - ".direnv".to_string(), - ".git".to_string(), - "target".to_string(), - ], - install_deps: Vec::new(), - install_phase: "cp bar.txt $OUTPUT".to_string(), - name: "bar".to_string(), - source: env::current_dir()?.to_string_lossy().to_string(), - }) - .await? - .into_inner(); - - client - .package(PackageRequest { - build_deps: vec![foo], - build_phase: "echo \"baz\" >> baz.txt && cat baz.txt".to_string(), - ignore_paths: vec![ - ".direnv".to_string(), - ".git".to_string(), - "target".to_string(), - ], - install_deps: vec![bar], - install_phase: "mkdir -p $OUTPUT && cp baz.txt $OUTPUT/baz.txt".to_string(), - name: "baz".to_string(), - source: env::current_dir()?.to_string_lossy().to_string(), - }) - .await?; + println!("{:?}", example_rust); Ok(()) } diff --git a/flake.lock b/flake.lock index 6a9bad06..97ffbe2b 100644 --- a/flake.lock +++ b/flake.lock @@ -17,6 +17,24 @@ "type": "indirect" } }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1717196966, @@ -64,7 +82,44 @@ "inputs": { "flake-parts": "flake-parts", "nixpkgs": "nixpkgs", - "process-compose-flake": "process-compose-flake" + "process-compose-flake": "process-compose-flake", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1717813066, + "narHash": "sha256-wqbRwq3i7g5EHIui0bIi84mdqZ/It1AXBSLJ5tafD28=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "6dc3e45fe4aee36efeed24d64fc68b1f989d5465", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 6d51f30d..a74753af 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,8 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; + inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; + inputs.rust-overlay.url = "github:oxalica/rust-overlay"; outputs = inputs @ { flake-parts, @@ -17,15 +19,28 @@ perSystem = { config, pkgs, + system, ... }: let - inherit (pkgs) grpcurl just protobuf rustPlatform; + inherit (pkgs) clippy darwin grpcurl just lib openssl pkg-config protobuf rustfmt rustPlatform; + inherit (darwin.apple_sdk.frameworks) CoreServices SystemConfiguration Security; inherit (rustPlatform) buildRustPackage; in { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + overlays = [inputs.rust-overlay.overlays.default]; + }; + packages = { default = buildRustPackage { - cargoSha256 = "sha256-l8A+eH+UeLd+IccoT4T67wDW0ya5M89tsQKwtQnbwog="; - nativeBuildInputs = [protobuf]; + buildInputs = [openssl] ++ lib.optionals pkgs.stdenv.isDarwin [CoreServices SystemConfiguration Security]; + cargoSha256 = "sha256-aBfWKqOlRxAOSkRoL+vX4mCbz6jZ7+XVp3uHwS8N4TY="; + checkPhase = '' + ${pkgs.cargo}/bin/cargo clippy -- -D warnings + ${pkgs.rust-bin.nightly.latest.default}/bin/cargo fmt --check --verbose + ${pkgs.cargo}/bin/cargo test --locked --all-features --all-targets + ''; + nativeBuildInputs = [clippy pkg-config protobuf rustfmt]; pname = "vorpal"; src = ./.; version = "0.1.0"; diff --git a/src/command/mod.rs b/src/command/mod.rs index 1a1da299..89184d02 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -48,10 +48,10 @@ pub async fn run() -> Result<(), anyhow::Error> { match &cli.command { Command::Service(service) => match service { Service::Build(build) => match build { - Build::Start { port } => build::start(port.clone()).await, + Build::Start { port } => build::start(*port).await, }, Service::Proxy(proxy) => match proxy { - Proxy::Start { port } => proxy::start(port.clone()).await, + Proxy::Start { port } => proxy::start(*port).await, }, }, } diff --git a/src/notary/mod.rs b/src/notary/mod.rs index 0e0fa680..5daf2c0c 100644 --- a/src/notary/mod.rs +++ b/src/notary/mod.rs @@ -1,7 +1,11 @@ use crate::store; use rand; +use rand::rngs::OsRng; use rsa::pkcs8::LineEnding; use rsa::pkcs8::{DecodePrivateKey, DecodePublicKey, EncodePrivateKey, EncodePublicKey}; +use rsa::pss::{Signature, SigningKey}; +use rsa::sha2::Sha256; +use rsa::signature::RandomizedSigner; use rsa::{RsaPrivateKey, RsaPublicKey}; use tokio::fs; @@ -28,7 +32,7 @@ pub fn generate_keys() -> Result<(), anyhow::Error> { pub async fn get_private_key() -> Result { let key_data = fs::read(store::get_private_key_path()).await?; let key = std::str::from_utf8(&key_data)?; - Ok(RsaPrivateKey::from_pkcs8_pem(&key)?) + Ok(RsaPrivateKey::from_pkcs8_pem(key)?) } pub async fn get_public_key() -> Result { @@ -36,3 +40,10 @@ pub async fn get_public_key() -> Result { let key = std::str::from_utf8(&key_data)?; Ok(RsaPublicKey::from_public_key_pem(key)?) } + +pub async fn sign(source_data: &[u8]) -> Result { + let private_key = get_private_key().await?; + let signing_key = SigningKey::::new(private_key); + let mut signing_rng = OsRng; + Ok(signing_key.sign_with_rng(&mut signing_rng, source_data)) +} diff --git a/src/service/build/mod.rs b/src/service/build/mod.rs index 3da85782..61d4ab0b 100644 --- a/src/service/build/mod.rs +++ b/src/service/build/mod.rs @@ -5,19 +5,21 @@ use crate::store; use anyhow::Result; use tokio::fs; use tonic::transport::Server; -mod build; -mod prepare; + +mod run_build; +mod run_prepare; +mod sandbox_default; pub mod service; pub async fn start(port: u16) -> Result<(), anyhow::Error> { - let vorpal_dir = store::get_home_path(); + let vorpal_dir = store::get_home_dir_path(); if !vorpal_dir.exists() { fs::create_dir_all(&vorpal_dir).await?; } println!("Vorpal directory: {:?}", vorpal_dir); - let store_dir = store::get_store_path(); + let store_dir = store::get_store_dir_path(); if !store_dir.exists() { fs::create_dir_all(&store_dir).await?; } @@ -27,7 +29,7 @@ pub async fn start(port: u16) -> Result<(), anyhow::Error> { let private_key_path = store::get_private_key_path(); let public_key_path = store::get_public_key_path(); if !private_key_path.exists() && !public_key_path.exists() { - let key_dir = store::get_key_path(); + let key_dir = store::get_private_key_path(); fs::create_dir_all(&key_dir).await?; println!("Key directory: {:?}", key_dir); notary::generate_keys()?; diff --git a/src/service/build/build/mod.rs b/src/service/build/run_build.rs similarity index 79% rename from src/service/build/build/mod.rs rename to src/service/build/run_build.rs index c7858b5a..d97b5fed 100644 --- a/src/service/build/build/mod.rs +++ b/src/service/build/run_build.rs @@ -1,16 +1,15 @@ use crate::api::{BuildRequest, BuildResponse}; use crate::database; +use crate::service::build::sandbox_default; use crate::store; use std::os::unix::fs::PermissionsExt; -use tempfile::tempdir; +use tempfile::TempDir; use tera::Tera; use tokio::fs; use tokio::process::Command; use tonic::{Request, Response, Status}; use walkdir::WalkDir; -mod sandbox_default; - pub async fn run(request: Request) -> Result, Status> { let message = request.into_inner(); @@ -37,7 +36,7 @@ pub async fn run(request: Request) -> Result) -> Result (), @@ -88,41 +86,46 @@ pub async fn run(request: Request) -> Result>() .join("\n"); let install_phase_steps = message .install_phase .trim() - .split("\n") + .split('\n') .map(|line| line.trim()) .collect::>() .join("\n"); - let mut automation_script: Vec = Vec::new(); - automation_script.push("#!/bin/bash".to_string()); - automation_script.push("set -e pipefail".to_string()); - automation_script.push("echo \"Starting build phase\"".to_string()); - automation_script.push(build_phase_steps); - automation_script.push("echo \"Finished build phase\"".to_string()); - automation_script.push("echo \"Starting install phase\"".to_string()); - automation_script.push(install_phase_steps); - automation_script.push("echo \"Finished install phase\"".to_string()); - let automation_script = automation_script.join("\n"); - let automation_script_path = source_temp_vorpal_dir.join("automation.sh"); - fs::write(&automation_script_path, &automation_script).await?; + let automation_script: Vec = vec![ + "#!/bin/bash".to_string(), + "set -e pipefail".to_string(), + "echo \"Starting build phase\"".to_string(), + build_phase_steps, + "echo \"Finished build phase\"".to_string(), + "echo \"Starting install phase\"".to_string(), + install_phase_steps, + "echo \"Finished install phase\"".to_string(), + ]; + + let automation_script_data = automation_script.join("\n"); + + println!("Build script: {}", automation_script_data); + + let automation_script_path = source_temp_vorpal_dir.join("automation.sh"); - println!("Build script: {}", automation_script); + fs::write(&automation_script_path, automation_script_data).await?; let metadata = fs::metadata(&automation_script_path).await?; let mut permissions = metadata.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&automation_script_path, permissions).await?; let os_type = std::env::consts::OS; if os_type != "macos" { - eprintln!("Unsupported OS: {}", os_type); return Err(Status::internal("Unsupported OS (currently only macOS)")); } @@ -148,25 +151,26 @@ pub async fn run(request: Request) -> Result 0 { - eprintln!("Build stderr: {:?}", sandbox_stderr); return Err(Status::internal("Build failed")); } - println!("{}", sandbox_stdout.trim()); - - if !sandbox_output_path.is_file() { + if sandbox_output_path.is_dir() { for entry in WalkDir::new(&sandbox_output_path) { let entry = match entry { Ok(entry) => entry, @@ -186,16 +190,14 @@ pub async fn run(request: Request) -> Result files, - Err(e) => { - eprintln!("Failed to get sandbox output files: {:?}", e); + Err(_) => { return Err(Status::internal("Failed to get sandbox output files")); } }; match store::compress_files(&store_output_path, &store_output_tar, &store_output_files) { Ok(_) => (), - Err(e) => { - eprintln!("Failed to compress sandbox output: {:?}", e); + Err(_) => { return Err(Status::internal("Failed to compress sandbox output")); } }; @@ -203,7 +205,7 @@ pub async fn run(request: Request) -> Result) -> Result, Status> { - let message = request.into_inner(); +pub async fn run( + mut stream: Streaming, +) -> Result, Status> { + let mut source_data: Vec = Vec::new(); - let source_dir_path = store::get_source_dir_path(&message.source_name, &message.source_hash); - let source_tar_path = store::get_source_tar_path(&source_dir_path); + let mut source_hash = String::new(); + let mut source_name = String::new(); + let mut source_signature = String::new(); + let mut source_chunks = 0; - let public_key = match notary::get_public_key().await { - Ok(key) => key, - Err(e) => { - eprintln!("Failed to get public key: {:?}", e); - return Err(Status::internal("Failed to get public key")); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| Status::internal(format!("Stream error: {}", e)))?; + + println!("Chunk size: {}", chunk.source_data.len()); + + if source_chunks == 0 { + source_hash = chunk.source_hash; + source_name = chunk.source_name; + source_signature = chunk.source_signature; + source_chunks += 1; } - }; + + source_data.extend_from_slice(&chunk.source_data); + } + + println!("Processed chunks: {}", source_chunks); + + let public_key = notary::get_public_key() + .await + .map_err(|_| Status::internal("Failed to get public key"))?; + let verifying_key = VerifyingKey::::new(public_key); - let source_signature_decode = match hex::decode(message.source_signature) { - Ok(data) => data, - Err(_) => return Err(Status::internal("hex decode of signature failed")), - }; + if source_signature.is_empty() { + return Err(Status::internal("Source signature is empty")); + } - let source_signature = match Signature::try_from(source_signature_decode.as_slice()) { - Ok(signature) => signature, - Err(e) => { - eprintln!("Failed to decode signature: {:?}", e); - return Err(Status::internal("Failed to decode signature")); - } - }; + let signature_decode = hex::decode(source_signature) + .map_err(|_| Status::internal("hex decode of signature failed"))?; - match verifying_key.verify(&message.source_data, &source_signature) { - Ok(_) => (), - Err(e) => { - eprintln!("Failed to verify signature: {:?}", e); - return Err(Status::internal("Failed to verify signature")); - } - }; + let signature = Signature::try_from(signature_decode.as_slice()) + .map_err(|_| Status::internal("Failed to decode signature"))?; - let db_path = store::get_database_path(); - let db = match database::connect(db_path) { - Ok(conn) => conn, - Err(e) => { - eprintln!("Failed to connect to database: {:?}", e); - return Err(Status::internal("Failed to connect to database")); - } - }; + verifying_key + .verify(&source_data, &signature) + .map_err(|_| Status::internal("Failed to verify signature"))?; + + let db = database::connect(store::get_database_path()) + .map_err(|_| Status::internal("Failed to connect to database"))?; + + if source_hash.is_empty() { + return Err(Status::internal("Source hash is empty")); + } + + if source_name.is_empty() { + return Err(Status::internal("Source name is empty")); + } + + let source_dir_path = store::get_source_dir_path(&source_name, &source_hash); + let source_tar_path = store::get_source_tar_path(&source_name, &source_hash); if !source_tar_path.exists() { let mut source_tar = File::create(&source_tar_path)?; - match source_tar.write_all(&message.source_data) { + match source_tar.write_all(&source_data) { Ok(_) => { let metadata = fs::metadata(&source_tar_path).await?; let mut permissions = metadata.permissions(); permissions.set_mode(0o444); fs::set_permissions(source_tar_path.clone(), permissions).await?; let file_name = source_tar_path.file_name().unwrap(); - println!("Source file: {}", file_name.to_string_lossy()); + println!("Source tar: {}", file_name.to_string_lossy()); } Err(e) => eprintln!("Failed source file: {}", e), } @@ -80,7 +97,7 @@ pub async fn run(request: Request) -> Result files, Err(e) => { eprintln!("Failed to get source files: {}", e); @@ -88,6 +105,8 @@ pub async fn run(request: Request) -> Result hashes, Err(e) => { @@ -96,7 +115,7 @@ pub async fn run(request: Request) -> Result hash, Err(e) => { eprintln!("Failed to get source hash: {}", e); @@ -104,12 +123,15 @@ pub async fn run(request: Request) -> Result (), Err(e) => { eprintln!("Failed to insert source: {:?}", e); @@ -120,7 +142,7 @@ pub async fn run(request: Request) -> Result source.id, Err(e) => { eprintln!("Failed to find source: {:?}", e); @@ -133,9 +155,7 @@ pub async fn run(request: Request) -> Result eprintln!("Failed to close database: {:?}", e), } - let response = PrepareResponse { - source_id: source_id, - }; + let response = PrepareResponse { source_id }; Ok(Response::new(response)) } diff --git a/src/service/build/build/sandbox_default.rs b/src/service/build/sandbox_default.rs similarity index 100% rename from src/service/build/build/sandbox_default.rs rename to src/service/build/sandbox_default.rs diff --git a/src/service/build/service.rs b/src/service/build/service.rs index 50b6c42f..64f487ce 100644 --- a/src/service/build/service.rs +++ b/src/service/build/service.rs @@ -1,8 +1,8 @@ use crate::api::package_service_server::PackageService; use crate::api::{BuildRequest, BuildResponse, PrepareRequest, PrepareResponse}; -use crate::service::build::{build, prepare}; +use crate::service::build::{run_build, run_prepare}; use anyhow::Result; -use tonic::{Request, Response, Status}; +use tonic::{Request, Response, Status, Streaming}; #[derive(Debug, Default)] pub struct Package {} @@ -11,15 +11,15 @@ pub struct Package {} impl PackageService for Package { async fn prepare( &self, - request: Request, + request: Request>, ) -> Result, Status> { - prepare::run(request).await + run_prepare::run(request.into_inner()).await } async fn build( &self, request: Request, ) -> Result, Status> { - build::run(request).await + run_build::run(request).await } } diff --git a/src/service/proxy/mod.rs b/src/service/proxy/mod.rs index 1bc94ed3..dd0eef5c 100644 --- a/src/service/proxy/mod.rs +++ b/src/service/proxy/mod.rs @@ -8,21 +8,14 @@ mod package; mod service; pub async fn start(port: u16) -> Result<(), anyhow::Error> { - let vorpal_dir = store::get_home_path(); + let vorpal_dir = store::get_home_dir_path(); if !vorpal_dir.exists() { fs::create_dir_all(&vorpal_dir).await?; } println!("Vorpal directory: {:?}", vorpal_dir); - let package_dir = store::get_package_path(); - if !package_dir.exists() { - fs::create_dir_all(&package_dir).await?; - } - - println!("Package directory: {:?}", package_dir); - - let store_dir = store::get_store_path(); + let store_dir = store::get_store_dir_path(); if !store_dir.exists() { fs::create_dir_all(&store_dir).await?; } @@ -32,7 +25,7 @@ pub async fn start(port: u16) -> Result<(), anyhow::Error> { let private_key_path = store::get_private_key_path(); let public_key_path = store::get_public_key_path(); if !private_key_path.exists() && !public_key_path.exists() { - let key_dir = store::get_key_path(); + let key_dir = store::get_private_key_path(); fs::create_dir_all(&key_dir).await?; println!("Key directory: {:?}", key_dir); notary::generate_keys()?; diff --git a/src/service/proxy/package/mod.rs b/src/service/proxy/package/mod.rs index 32aa9ace..7d78f6cd 100644 --- a/src/service/proxy/package/mod.rs +++ b/src/service/proxy/package/mod.rs @@ -1,46 +1,45 @@ use crate::api::package_service_client::PackageServiceClient; -use crate::api::{BuildRequest, PackageRequest, PackageResponse, PrepareRequest}; +use crate::api::{ + BuildRequest, PackageRequest, PackageResponse, PackageSource, PackageSourceKind, PrepareRequest, +}; use crate::notary; use crate::store; -use rand::rngs::OsRng; -use rsa::pss::SigningKey; -use rsa::sha2::Sha256; -use rsa::signature::RandomizedSigner; +use git2::build::RepoBuilder; +use git2::{Cred, RemoteCallbacks}; +use reqwest; use std::fs::Permissions; use std::os::unix::fs::PermissionsExt; -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; +use tempfile::{tempdir, NamedTempFile}; use tokio::fs; -use tokio::fs::File; +use tokio::fs::{copy, create_dir_all, read, set_permissions, write, File}; use tokio::io::AsyncWriteExt; +use tokio_stream; use tonic::{Request, Response, Status}; +use url::Url; pub async fn run(request: Request) -> Result, Status> { - let package_dir = store::get_package_path(); - if !package_dir.exists() { - return Err(Status::internal("Package directory does not exist")); - } + let req = request.into_inner(); - let r = request.into_inner(); + let req_source = req.source.as_ref().ok_or_else(|| { + eprintln!("Source is required"); + Status::invalid_argument("Source is required") + })?; - println!("Preparing: {}", r.name); + println!("Preparing: {}", req.name); - let (source_id, source_hash) = match prepare(package_dir, &r).await { - Ok((id, hash)) => (id, hash), - Err(e) => { - eprintln!("Failed to prepare: {:?}", e); - return Err(Status::internal("Failed to prepare")); - } - }; + let (source_id, source_hash) = prepare(&req.name, req_source).await.map_err(|e| { + eprintln!("Failed to prepare: {:?}", e); + Status::internal(e.to_string()) + })?; - println!("Building: {}-{}", r.name, source_hash); + println!("Building: {}-{}", req.name, source_hash); - match build(source_id, &source_hash, &r).await { - Ok(_) => (), - Err(e) => { - eprintln!("Failed to build: {:?}", e); - return Err(Status::internal("Failed to build")); - } - } + build(source_id, &source_hash, &req).await.map_err(|e| { + eprintln!("Failed to build: {:?}", e); + Status::internal(e.to_string()) + })?; let response = PackageResponse { source_id: source_id.to_string(), @@ -50,49 +49,197 @@ pub async fn run(request: Request) -> Result Result<(i32, String), anyhow::Error> { - let source = Path::new(&request.source).canonicalize()?; - let source_ignore_paths = request - .ignore_paths - .iter() - .map(|i| Path::new(i).to_path_buf()) - .collect::>(); - let source_files = store::get_file_paths(&source, &source_ignore_paths)?; - let source_files_hashes = store::get_file_hashes(&source_files)?; - let source_hash = store::get_source_hash(&source_files_hashes)?; - let source_dir_name = store::get_package_dir_name(&request.name, &source_hash); - let source_dir = package_dir.join(&source_dir_name).with_extension("source"); - let source_tar = source_dir.with_extension("source.tar.gz"); - if !source_tar.exists() { - println!("Creating source tar: {:?}", source_tar); - store::compress_files(&source, &source_tar, &source_files)?; - fs::set_permissions(&source_tar, Permissions::from_mode(0o444)).await?; - } + let data = read(source_tar).await?; - println!("Source file: {}", source_tar.display()); + let signature = notary::sign(&data).await?; - let private_key = notary::get_private_key().await?; - let signing_key = SigningKey::::new(private_key); - let mut signing_rng = OsRng; - let source_data = fs::read(&source_tar).await?; - let source_signature = signing_key.sign_with_rng(&mut signing_rng, &source_data); + println!("Source tar signature: {}", signature); - println!("Source signature: {}", source_signature.to_string()); + let mut request_chunks = vec![]; + let request_chunks_size = 2 * 1024 * 1024; // default grpc limit + + for chunk in data.chunks(request_chunks_size) { + request_chunks.push(PrepareRequest { + source_data: chunk.to_vec(), + source_hash: source_hash.to_string(), + source_name: source_name.to_string(), + source_signature: signature.to_string(), + }); + } + + println!("Request chunks: {}", request_chunks.len()); - let request = tonic::Request::new(PrepareRequest { - source_data, - source_hash: source_hash.to_string(), - source_name: request.name.to_string(), - source_signature: source_signature.to_string(), - }); let mut client = PackageServiceClient::connect("http://[::1]:15323").await?; - let response = client.prepare(request).await?; - let response_data = response.into_inner(); + let response = client.prepare(tokio_stream::iter(request_chunks)).await?; + let res = response.into_inner(); + + println!("Source ID: {}", res.source_id); - Ok((response_data.source_id, source_hash)) + Ok((res.source_id, source_hash.to_string())) +} + +async fn prepare(name: &str, source: &PackageSource) -> Result<(i32, String), anyhow::Error> { + let work_dir = tempdir()?; + let workdir_path = work_dir.path().canonicalize()?; + + println!("Preparing working dir: {:?}", workdir_path); + + if source.kind == PackageSourceKind::Unknown as i32 { + return Err(anyhow::anyhow!("Unknown source kind")); + } + + if source.kind == PackageSourceKind::Git as i32 { + let mut builder = RepoBuilder::new(); + + if source.uri.starts_with("git://") { + let mut callbacks = RemoteCallbacks::new(); + + callbacks.credentials(|_url, username_from_url, _allowed_types| { + Cred::ssh_key( + username_from_url.unwrap(), + None, + Path::new(&format!( + "{}/.ssh/id_rsa", + dirs::home_dir().unwrap().display() + )), + None, + ) + }); + + let mut fetch_options = git2::FetchOptions::new(); + + fetch_options.remote_callbacks(callbacks); + + builder.fetch_options(fetch_options); + } + + let _ = builder.clone(&source.uri, &workdir_path)?; + } + + if source.kind == PackageSourceKind::Http as i32 { + println!("Downloading source: {:?}", &source.uri); + + let url = Url::parse(&source.uri)?; + + if url.scheme() != "http" && url.scheme() != "https" { + return Err(anyhow::anyhow!("Invalid HTTP source URL")); + } + + let response = reqwest::get(url.as_str()).await?.bytes().await?; + let response_bytes = response.as_ref(); + + if let Some(source_kind) = infer::get(response_bytes) { + println!("Preparing source kind: {:?}", source_kind); + + match source_kind.mime_type() { + "application/gzip" => { + let temp_file = NamedTempFile::new()?; + write(&temp_file, response_bytes).await?; + store::unpack_source(&workdir_path, temp_file.path())?; + println!("Prepared packed source: {:?}", workdir_path); + } + _ => { + let source_file_name = url.path_segments().unwrap().last(); + let source_file = source_file_name.unwrap(); + write(&source_file, response_bytes).await?; + println!("Prepared source file: {:?}", source_file); + } + } + } + } + + if source.kind == PackageSourceKind::Local as i32 { + let source_path = Path::new(&source.uri).canonicalize()?; + + println!("Preparing source path: {:?}", source_path); + + if let Ok(Some(source_kind)) = infer::get_from_path(&source_path) { + println!("Preparing source kind: {:?}", source_kind); + + if source_kind.mime_type() == "application/gzip" { + println!("Preparing packed source: {:?}", work_dir); + store::unpack_source(&workdir_path, &source_path)?; + } + } + + if source_path.is_file() { + let dest = workdir_path.join(source_path.file_name().unwrap()); + copy(&source_path, &dest).await?; + println!( + "Preparing source file: {:?} -> {:?}", + source_path.display(), + dest.display() + ); + } + + if source_path.is_dir() { + let files = store::get_file_paths(&source_path, &source.ignore_paths)?; + + if files.is_empty() { + return Err(anyhow::anyhow!("No source files found")); + } + + for src in &files { + if src.is_dir() { + let dest = workdir_path.join(src.strip_prefix(&source_path)?); + create_dir_all(dest).await?; + continue; + } + + let dest = workdir_path.join(src.file_name().unwrap()); + copy(src, &dest).await?; + println!( + "Preparing source file: {:?} -> {:?}", + src.display(), + dest.display() + ); + } + } + } + + // At this point, any source URI should be a local file path + + let workdir_files = store::get_file_paths(&workdir_path.to_path_buf(), &source.ignore_paths)?; + + if workdir_files.is_empty() { + return Err(anyhow::anyhow!("No source files found")); + } + + println!("Preparing source files: {:?}", workdir_files); + + let workdir_files_hashes = store::get_file_hashes(&workdir_files)?; + let workdir_hash = store::get_source_hash(&workdir_files_hashes)?; + + if workdir_hash.is_empty() { + return Err(anyhow::anyhow!("Failed to get source hash")); + } + + println!("Source hash: {}", workdir_hash); + + if let Some(request_hash) = &source.hash { + if &workdir_hash != request_hash { + let message = format!("Hash mismatch: {} != {}", request_hash, workdir_hash); + return Err(anyhow::anyhow!("{}", message)); + } + } + + let source_tar = NamedTempFile::new()?; + let source_tar_path = source_tar.path().canonicalize()?; + + println!("Creating source tar: {:?}", source_tar); + + store::compress_files(&workdir_path, &source_tar_path, &workdir_files)?; + + set_permissions(&source_tar, Permissions::from_mode(0o444)).await?; + + println!("Source tar: {}", source_tar_path.display()); + + prepare_source(name, &workdir_hash, &source_tar_path).await } async fn build( @@ -112,8 +259,8 @@ async fn build( let response_data = response.into_inner(); if response_data.is_compressed { - let store_path = store::get_store_path(); - let store_path_dir_name = store::get_package_dir_name(&request.name, package_hash); + let store_path = store::get_store_dir_path(); + let store_path_dir_name = store::get_store_dir_name(&request.name, package_hash); let store_path_dir = store_path.join(&store_path_dir_name); let store_path_tar = store_path_dir.with_extension("tar.gz"); @@ -126,6 +273,7 @@ async fn build( println!("Using existing tar: {}", store_path_tar.display()); fs::create_dir_all(&store_path_dir).await?; + store::unpack_source(&store_path_dir, &store_path_tar)?; println!("Unpacked source: {}", store_path_dir.display()); @@ -134,7 +282,7 @@ async fn build( } let mut store_tar = File::create(&store_path_tar).await?; - match store_tar.write_all(&response_data.package_data).await { + match store_tar.write(&response_data.package_data).await { Ok(_) => { let metadata = fs::metadata(&store_path_tar).await?; let mut permissions = metadata.permissions(); @@ -151,6 +299,7 @@ async fn build( println!("Stored tar: {}", store_path_tar.display()); fs::create_dir_all(&store_path_dir).await?; + store::unpack_source(&store_path_dir, &store_path_tar)?; println!("Unpacked source: {}", store_path_dir.display()); diff --git a/src/store/mod.rs b/src/store/mod.rs index 112f7aa2..31838e69 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -3,6 +3,7 @@ use flate2::read::GzDecoder; use flate2::write::GzEncoder; use flate2::Compression; use sha256::{digest, try_digest}; +use std::env; use std::fs; use std::fs::File; use std::io::BufReader; @@ -10,69 +11,77 @@ use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use tar::Archive; use tar::Builder; +use uuid::Uuid; use walkdir::WalkDir; -pub fn get_home_path() -> PathBuf { +pub fn get_home_dir_path() -> PathBuf { dirs::home_dir() .expect("Home directory not found") .join(".vorpal") } -pub fn get_key_path() -> PathBuf { - get_home_path().join("key") +pub fn get_key_dir_path() -> PathBuf { + get_home_dir_path().join("key") } -pub fn get_package_path() -> PathBuf { - get_home_path().join("package") -} - -pub fn get_store_path() -> PathBuf { - get_home_path().join("store") +pub fn get_store_dir_path() -> PathBuf { + get_home_dir_path().join("store") } pub fn get_database_path() -> PathBuf { - get_home_path().join("vorpal.db") + get_home_dir_path().join("vorpal.db") } pub fn get_private_key_path() -> PathBuf { - get_key_path().join("private").with_extension("pem") + get_key_dir_path().join("private").with_extension("pem") } pub fn get_public_key_path() -> PathBuf { - get_key_path().join("public").with_extension("pem") + get_key_dir_path().join("public").with_extension("pem") } -pub fn get_source_tar_path(source_dir: &Path) -> PathBuf { - source_dir - .join(source_dir.with_extension("source.tar.gz")) - .to_path_buf() +pub fn get_temp_dir_path() -> PathBuf { + env::temp_dir().join(Uuid::now_v7().to_string()) } -pub fn get_package_dir_name(name: &str, hash: &str) -> String { +pub fn get_store_dir_name(name: &str, hash: &str) -> String { format!("{}-{}", name, hash) } -pub fn get_source_dir_path(source_name: &String, source_hash: &String) -> PathBuf { - let store_dir = get_store_path(); +pub fn get_source_tar_path(source_name: &str, source_hash: &str) -> PathBuf { + let store_dir = get_store_dir_path(); + let store_dir_name = get_store_dir_name(source_name, source_hash); + store_dir + .join(store_dir_name) + .with_extension("source.tar.gz") +} + +pub fn get_source_dir_path(source_name: &str, source_hash: &str) -> PathBuf { + let store_dir = get_store_dir_path(); store_dir - .join(&get_package_dir_name(source_name, source_hash)) - .with_extension("package") + .join(get_store_dir_name(source_name, source_hash)) + .with_extension("source") .to_path_buf() } -pub fn get_file_paths(source: &Path, ignore_paths: &[PathBuf]) -> Result> { - let mut files: Vec = WalkDir::new(&source) +pub fn get_file_paths(source: &PathBuf, ignore_paths: &[String]) -> Result> { + let source_ignore_paths = ignore_paths + .iter() + .map(|i| Path::new(i).to_path_buf()) + .collect::>(); + + let mut files: Vec = WalkDir::new(source) .into_iter() .filter_map(|entry| { let entry = entry.ok()?; let path = entry.path(); - if ignore_paths + if source_ignore_paths .iter() - .any(|i| path.strip_prefix(&source).unwrap().starts_with(i)) + .any(|i| path.strip_prefix(source).unwrap().starts_with(i)) { return None; } - Some(path.canonicalize().ok()?) + path.canonicalize().ok() }) .collect(); @@ -106,21 +115,23 @@ pub fn get_source_hash(hashes: &[(PathBuf, String)]) -> Result { let mut combined = String::new(); for (_, hash) in hashes { - combined.push_str(&hash); + combined.push_str(hash); } Ok(digest(combined)) } pub fn compress_files( - source: &Path, - source_tar: &Path, + source: &PathBuf, + source_output: &Path, source_files: &[PathBuf], ) -> Result { - let tar = File::create(source_tar)?; + let tar = File::create(source_output)?; let tar_encoder = GzEncoder::new(tar.try_clone()?, Compression::default()); let mut tar_builder = Builder::new(tar_encoder); + println!("Compressing: {}", source.display()); + for path in source_files { if path == source { continue; @@ -144,7 +155,7 @@ pub fn compress_files( pub fn set_files_permissions(files: &[PathBuf]) -> Result<(), anyhow::Error> { for file in files { - let permissions = fs::metadata(&file)?.permissions(); + let permissions = fs::metadata(file)?.permissions(); if permissions.mode() & 0o111 != 0 { fs::set_permissions(file, fs::Permissions::from_mode(0o555))?; } else { @@ -155,11 +166,45 @@ pub fn set_files_permissions(files: &[PathBuf]) -> Result<(), anyhow::Error> { Ok(()) } -pub fn unpack_source(target_dir: &Path, source_tar: &Path) -> Result<(), anyhow::Error> { - let tar_gz = File::open(&source_tar)?; +pub fn unpack_source(target_dir: &PathBuf, source_tar: &Path) -> Result<(), anyhow::Error> { + let tar_gz = File::open(source_tar)?; let buf_reader = BufReader::new(tar_gz); let gz_decoder = GzDecoder::new(buf_reader); let mut archive = Archive::new(gz_decoder); - archive.unpack(&target_dir)?; + archive.unpack(target_dir)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn home_dir_path() { + assert_eq!(get_home_dir_path().file_name().unwrap(), ".vorpal"); + } + + #[test] + fn key_dir_path() { + assert_eq!(get_key_dir_path().file_name().unwrap(), "key"); + } + + #[test] + fn key_dir_path_home() { + let home_dir = get_home_dir_path(); + let key_dir = get_key_dir_path(); + assert!(key_dir.starts_with(home_dir)); + } + + #[test] + fn store_dir_path() { + assert_eq!(get_store_dir_path().file_name().unwrap(), "store"); + } + + #[test] + fn store_dir_path_home() { + let home_dir = get_home_dir_path(); + let store_dir = get_key_dir_path(); + assert!(store_dir.starts_with(home_dir)); + } +}