From ae678ea2f8c6f6a7df445f87f0fff6c944541cfe Mon Sep 17 00:00:00 2001 From: Nicholas Farshidmehr Date: Mon, 1 Aug 2022 10:58:49 -0400 Subject: [PATCH] feat: add integration with drawbridge - Rewrite much of the frontend scripting code. - Change some messaging around file limits. - Create a more interactive experience preparing workloads. - Add handling code for drawbridge, including pulling a Enarx.toml preview. - Add server code for pulling Enarx.toml if the frontend fails to do it. - Add some initial examples (some of which are broken at the moment). - Change the "Deploy workload" positioning, size, and color. Signed-off-by: Nicholas Farshidmehr --- Cargo.lock | 8 + Cargo.nix | 9 ++ Cargo.toml | 3 +- examples.txt | 10 ++ src/auth/user.rs | 7 +- src/jobs.rs | 98 ++++++++++-- src/main.rs | 154 +++++++++++++++--- src/templates.rs | 4 +- templates/idx.html | 379 ++++++++++++++++++++++++++++++++++++++------- templates/job.html | 6 +- 10 files changed, 571 insertions(+), 107 deletions(-) create mode 100644 examples.txt diff --git a/Cargo.lock b/Cargo.lock index 279ae89..227a0c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,8 @@ dependencies = [ "axum", "clap", "enarx-config", + "humansize", + "lazy_static", "num_cpus", "once_cell", "openidconnect", @@ -528,6 +530,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humansize" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026" + [[package]] name = "hyper" version = "0.14.20" diff --git a/Cargo.nix b/Cargo.nix index 68b0484..d7a064b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -211,6 +211,8 @@ in axum = rustPackages."registry+https://github.com/rust-lang/crates.io-index".axum."0.5.13" { inherit profileName; }; clap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.2.14" { inherit profileName; }; enarx_config = rustPackages."registry+https://github.com/rust-lang/crates.io-index".enarx-config."0.6.1" { inherit profileName; }; + humansize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".humansize."1.1.1" { inherit profileName; }; + lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; num_cpus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; }; once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.13.0" { inherit profileName; }; openidconnect = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openidconnect."2.3.2" { inherit profileName; }; @@ -741,6 +743,13 @@ in src = fetchCratesIo { inherit name version; sha256 = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".humansize."1.1.1" = overridableMkRustCrate (profileName: rec { + name = "humansize"; + version = "1.1.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.20" = overridableMkRustCrate (profileName: rec { name = "hyper"; version = "0.14.20"; diff --git a/Cargo.toml b/Cargo.toml index dc2dfa8..e255ab4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,4 +22,5 @@ num_cpus = "1.13.1" serde = { version = "1.0.136", default-features = false } serde_json = { version = "1.0.82", default-features = false } enarx-config = { version = "0.6.1", default-features = false } - +humansize = { version = "1.1.1", default-features = false } +lazy_static = { version = "1.4.0", default-features = false } diff --git a/examples.txt b/examples.txt new file mode 100644 index 0000000..f16aeda --- /dev/null +++ b/examples.txt @@ -0,0 +1,10 @@ +examples/cryptle-rust:0.1.0 +examples/echo-tcp-rust-mio:0.1.0 +examples/echo-tcp-rust-tokio:0.1.0 +examples/fibonacci-c:0.2.0 +examples/fibonacci-cpp:0.2.0 +examples/fibonacci-go:0.2.0 +examples/fibonacci-rust:0.2.0 +examples/fibonacci-zig:0.3.0 +examples/greenhouse-monitor-csharp:0.1.0 +examples/http-rust-tokio:0.1.0 diff --git a/src/auth/user.rs b/src/auth/user.rs index 2f61869..057eb5c 100644 --- a/src/auth/user.rs +++ b/src/auth/user.rs @@ -8,21 +8,16 @@ use axum::async_trait; use axum::extract::{FromRequest, RequestParts}; use axum::response::Response; use once_cell::sync::Lazy; -use reqwest::{Client, ClientBuilder}; use serde::Deserialize; use tokio::sync::RwLock; use tokio::time::sleep; use tracing::error; use crate::reference::Ref; +use crate::HTTP; use super::Session; -static HTTP: Lazy = Lazy::new(|| { - const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); - ClientBuilder::new().user_agent(USER_AGENT).build().unwrap() -}); - const STAR_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60); static STAR: Lazy>> = Lazy::new(|| HashMap::new().into()); diff --git a/src/jobs.rs b/src/jobs.rs index 9890609..ca4433e 100644 --- a/src/jobs.rs +++ b/src/jobs.rs @@ -1,14 +1,38 @@ use std::process::Stdio; +use std::str::FromStr; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; + +use anyhow::anyhow; use tempfile::NamedTempFile; use tokio::io::AsyncReadExt; use tokio::process::{Child, Command}; +use tracing::{debug, error}; use uuid::Uuid; static COUNT: AtomicUsize = AtomicUsize::new(0); +#[derive(Debug)] +pub enum WorkloadType { + Drawbridge, + Browser, +} + +impl FromStr for WorkloadType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "drawbridge" => WorkloadType::Drawbridge, + "browser" => WorkloadType::Browser, + _ => return Err(anyhow!("Unknown workload type {s}")), + }) + } +} + pub enum Standard { Output, Error, @@ -19,8 +43,10 @@ pub enum Standard { pub struct Job { pub uuid: Uuid, exec: Child, - wasm: NamedTempFile, - toml: NamedTempFile, + workload_type: WorkloadType, + slug: Option, + wasm: Option, + toml: Option, } impl Drop for Job { @@ -34,24 +60,66 @@ impl Job { COUNT.load(Ordering::SeqCst) } - pub fn new(cmd: String, wasm: NamedTempFile, toml: NamedTempFile) -> std::io::Result { - let uuid = Uuid::new_v4(); - let exec = Command::new(cmd) - .arg("run") - .arg("--wasmcfgfile") - .arg(toml.path()) - .arg(wasm.path()) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; + pub fn new( + cmd: String, + workload_type: String, + slug: Option, + wasm: Option, + toml: Option, + ) -> Result { + let workload_type = WorkloadType::from_str(&workload_type).map_err(|e| { + debug!("Failed to parse workload type: {e}"); + StatusCode::BAD_REQUEST.into_response() + })?; + let exec = match workload_type { + WorkloadType::Drawbridge => { + let slug = slug + .as_ref() + .ok_or_else(|| StatusCode::BAD_REQUEST.into_response())?; + Command::new(cmd) + .arg("deploy") + .arg(slug) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|e| { + error!("failed to spawn process: {e}"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + })? + } + WorkloadType::Browser => { + let wasm = wasm + .as_ref() + .ok_or_else(|| StatusCode::BAD_REQUEST.into_response())?; + let toml = toml + .as_ref() + .ok_or_else(|| StatusCode::BAD_REQUEST.into_response())?; + Command::new(cmd) + .arg("run") + .arg("--wasmcfgfile") + .arg(toml.path()) + .arg(wasm.path()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|e| { + error!("failed to spawn process: {e}"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + })? + } + }; COUNT.fetch_add(1, Ordering::SeqCst); Ok(Self { - uuid, + uuid: Uuid::new_v4(), exec, + workload_type, + slug, wasm, toml, }) diff --git a/src/main.rs b/src/main.rs index e3fd94a..7c569e4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,7 @@ use std::io::Write; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::time::Duration; -use axum::extract::Multipart; +use axum::extract::{Multipart, Query}; use axum::http::StatusCode; use axum::response::{IntoResponse, Redirect}; use axum::routing::{get, post}; @@ -27,15 +27,32 @@ use axum::{Router, Server}; use anyhow::{bail, Context as _}; use clap::Parser; +use humansize::{file_size_opts as options, FileSize}; +use lazy_static::lazy_static; +use once_cell::sync::Lazy; +use reqwest::{Client, ClientBuilder}; +use serde::Deserialize; use tokio::time::{sleep, timeout}; use tower_http::trace::TraceLayer; use tracing::{debug, error, info}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +static HTTP: Lazy = Lazy::new(|| { + const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); + ClientBuilder::new().user_agent(USER_AGENT).build().unwrap() +}); + // TODO: raise this when this is fixed: https://github.com/profianinc/benefice/issues/75 const READ_TIMEOUT: Duration = Duration::from_millis(500); const TOML_MAX: usize = 256 * 1024; // 256 KiB +lazy_static! { + static ref EXAMPLES: Vec<&'static str> = include_str!("../examples.txt") + .lines() + .filter(|line| !line.is_empty()) + .collect::>(); +} + #[derive(Debug, Default)] struct Data { job: Option, @@ -127,25 +144,40 @@ impl Args { #[derive(Copy, Clone, Debug)] struct Limits { + /// Size in megabytes size_limit_default: usize, + /// Size in megabytes size_limit_starred: usize, timeout_default: Duration, timeout_starred: Duration, } impl Limits { - pub fn decide(&self, star: bool) -> (Duration, usize) { - let size = match star { - false => self.size_limit_default, - true => self.size_limit_starred, - }; + pub fn time_to_live(&self, star: bool) -> Duration { + if star { + self.timeout_default + } else { + self.timeout_starred + } + } - let ttl = match star { - false => self.timeout_default, - true => self.timeout_starred, + /// Get the maximum allowed wasm size in bytes. + pub fn size(&self, star: bool) -> usize { + let size_megabytes = if star { + self.size_limit_default + } else { + self.size_limit_starred }; + size_megabytes * 1024 * 1024 + } - (ttl, size) + pub fn size_human(&self, star: bool) -> String { + self.size(star) + .file_size(options::CONVENTIONAL) + .unwrap_or_else(|e| { + error!("Failed to get human readable size string: {e}"); + "?".to_string() + }) } } @@ -201,6 +233,7 @@ async fn main() -> anyhow::Result<()> { .init(); let app = Router::new() + .route("/enarx_toml_fallback", get(enarx_toml_fallback)) .route( "/out", post(move |user| reader(user, jobs::Standard::Output)), @@ -236,14 +269,14 @@ async fn root_get(user: Option>>, limits: Limits) -> impl I } }; - let (ttl, size) = limits.decide(star); - let tmpl = IdxTemplate { toml: enarx_config::CONFIG_TEMPLATE, + examples: EXAMPLES.as_slice(), user, star, - size, - ttl: ttl.as_secs(), + size: limits.size(star), + size_human: limits.size_human(star), + ttl: limits.time_to_live(star).as_secs(), }; HtmlTemplate(tmpl).into_response() @@ -257,7 +290,9 @@ async fn root_post( limits: Limits, jobs: usize, ) -> impl IntoResponse { - let (ttl, size) = limits.decide(user.read().await.is_starred("enarx/enarx").await); + let star = user.read().await.is_starred("enarx/enarx").await; + let ttl = limits.time_to_live(star); + let size = limits.size(star); if user.read().await.data.job.is_some() { return Err(Redirect::to("/").into_response()); @@ -267,6 +302,8 @@ async fn root_post( return Err(redirect::too_many_workloads().into_response()); } + let mut workload_type = None; + let mut slug = None; let mut wasm = None; let mut toml = None; @@ -276,6 +313,40 @@ async fn root_post( .map_err(|_| StatusCode::BAD_REQUEST.into_response())? { match field.name() { + Some("workloadType") => { + if field.content_type().is_some() { + return Err(StatusCode::BAD_REQUEST.into_response()); + } + + if workload_type.is_some() { + return Err(StatusCode::BAD_REQUEST.into_response()); + } + + workload_type = Some( + field + .text() + .await + .map_err(|_| StatusCode::BAD_REQUEST.into_response())?, + ); + } + + Some("slug") => { + if field.content_type().is_some() { + return Err(StatusCode::BAD_REQUEST.into_response()); + } + + if slug.is_some() { + return Err(StatusCode::BAD_REQUEST.into_response()); + } + + slug = Some( + field + .text() + .await + .map_err(|_| StatusCode::BAD_REQUEST.into_response())?, + ); + } + Some("wasm") => { if Some("application/wasm") != field.content_type() { return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()); @@ -295,7 +366,7 @@ async fn root_post( .map_err(|_| StatusCode::BAD_REQUEST.into_response())? { len += chunk.len(); - if len > size * 1024 * 1024 { + if len > size { return Err(StatusCode::PAYLOAD_TOO_LARGE.into_response()); } @@ -340,8 +411,7 @@ async fn root_post( } } - let wasm = wasm.ok_or_else(|| StatusCode::BAD_REQUEST.into_response())?; - let toml = toml.ok_or_else(|| StatusCode::BAD_REQUEST.into_response())?; + let workload_type = workload_type.ok_or_else(|| StatusCode::BAD_REQUEST.into_response())?; // Create the new job and get an identifier. let uuid = { @@ -355,11 +425,7 @@ async fn root_post( return Err(redirect::too_many_workloads().into_response()); } - let job = jobs::Job::new(command, wasm, toml).map_err(|e| { - error!("failed to spawn process: {e}"); - StatusCode::INTERNAL_SERVER_ERROR.into_response() - })?; - + let job = jobs::Job::new(command, workload_type, slug, wasm, toml)?; let uuid = job.uuid; lock.data = Data { job: Some(job) }; uuid @@ -386,6 +452,48 @@ async fn root_post( Ok((StatusCode::SEE_OTHER, [("Location", "/")])) } +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct EnarxTomlFallbackParams { + repo: String, + tag: String, +} + +async fn enarx_toml_fallback( + _user: Ref>, + Query(params): Query, +) -> Result { + let EnarxTomlFallbackParams { repo, tag } = params; + let response = HTTP + .get(&format!( + "https://store.profian.com/api/v0.2.0/{repo}/_tag/{tag}/tree/Enarx.toml" + )) + .send() + .await; + let response = response.map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to get Enarx.toml from repo: {e}"), + ) + })?; + let status_code = response.status(); + let body = response.text().await.map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to get string contents of Enarx.toml in response body: {e}"), + ) + })?; + + match status_code { + StatusCode::OK => Ok(body), + StatusCode::NOT_FOUND => Err(( + StatusCode::NOT_FOUND, + format!("Couldn\'t find file in package '{repo}' with tag '{tag}'"), + )), + status_code => Err((status_code, body)), + } +} + async fn root_delete(user: Ref>) -> StatusCode { let mut lock = user.write().await; diff --git a/src/templates.rs b/src/templates.rs index c027080..4a24e47 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -7,11 +7,13 @@ use axum::response::{Html, IntoResponse, Response}; #[derive(Template)] #[template(path = "idx.html")] -pub struct IdxTemplate { +pub struct IdxTemplate<'a> { pub toml: &'static str, + pub examples: &'a [&'static str], pub user: bool, pub star: bool, pub size: usize, + pub size_human: String, pub ttl: u64, } diff --git a/templates/idx.html b/templates/idx.html index debb081..b44e2e4 100644 --- a/templates/idx.html +++ b/templates/idx.html @@ -1,7 +1,5 @@ - + + @@ -48,7 +46,7 @@
-
{% if user %}{% endif %} diff --git a/templates/job.html b/templates/job.html index 922041a..36c2665 100644 --- a/templates/job.html +++ b/templates/job.html @@ -1,7 +1,5 @@ - + +