From 146602293e9f91f0c811d62070792a1ffdb61cb9 Mon Sep 17 00:00:00 2001 From: varex83 Date: Wed, 18 Oct 2023 17:15:49 +0300 Subject: [PATCH 1/4] update .gitmodules --- .gitmodules | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9a418033..2d2259c5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,9 @@ [submodule "api/cairo_compilers/v2.1.1"] path = api/cairo_compilers/v2.1.1 url = https://github.com/starkware-libs/cairo.git + branch = v2.1.1 [submodule "api/cairo_compilers/v2.2.0"] path = api/cairo_compilers/v2.2.0 - url = https://github.com/starkware-libs/cairo.git \ No newline at end of file + url = https://github.com/starkware-libs/cairo.git + branch = v2.2.0 \ No newline at end of file From d55e06e36b2a486d07ac981cbc08060f696ba0fa Mon Sep 17 00:00:00 2001 From: varex83 Date: Wed, 18 Oct 2023 20:34:28 +0300 Subject: [PATCH 2/4] refactor error handling --- api/Cargo.lock | 21 +++++ api/Cargo.toml | 1 + api/src/handlers/cairo_version.rs | 11 +-- api/src/handlers/cairo_versions.rs | 11 ++- api/src/handlers/compile_casm.rs | 139 ++++++++++++----------------- api/src/handlers/compile_sierra.rs | 110 +++++++++++------------ api/src/handlers/mod.rs | 9 +- api/src/handlers/save_code.rs | 41 ++++----- api/src/handlers/scarb_compile.rs | 77 ++++++++++------ api/src/main.rs | 1 + api/src/types.rs | 27 ++++++ api/src/worker.rs | 5 +- 12 files changed, 247 insertions(+), 206 deletions(-) create mode 100644 api/src/types.rs diff --git a/api/Cargo.lock b/api/Cargo.lock index 645e3337..2ba99cdc 100644 --- a/api/Cargo.lock +++ b/api/Cargo.lock @@ -23,6 +23,7 @@ dependencies = [ "rand", "rocket", "serde", + "thiserror", "tracing", "tracing-appender", "tracing-log", @@ -1222,6 +1223,26 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "thiserror" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "thread_local" version = "1.1.7" diff --git a/api/Cargo.toml b/api/Cargo.toml index deddc35f..9e2b1af4 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -20,3 +20,4 @@ crossbeam-queue = "0.3.8" crossbeam-skiplist = "0.1.1" fmt = "0.1.0" yansi = "0.5.1" +thiserror = "1.0.49" \ No newline at end of file diff --git a/api/src/handlers/cairo_version.rs b/api/src/handlers/cairo_version.rs index a662e37d..7965539d 100644 --- a/api/src/handlers/cairo_version.rs +++ b/api/src/handlers/cairo_version.rs @@ -1,5 +1,6 @@ use crate::handlers::process::{do_process_command, fetch_process_result}; use crate::handlers::types::{ApiCommand, ApiCommandResult}; +use crate::types::{ApiError, Result}; use crate::utils::lib::DEFAULT_CAIRO_DIR; use crate::worker::WorkerEngine; use rocket::State; @@ -11,7 +12,7 @@ use tracing::{error, info, instrument}; #[get("/cairo_version")] pub async fn cairo_version() -> String { info!("/cairo_version"); - do_cairo_version().unwrap_or_else(|e| e) + do_cairo_version().unwrap_or_else(|e| format!("Failed to get cairo version: {:?}", e)) } // Read the version from the cairo Cargo.toml file. @@ -36,7 +37,7 @@ pub async fn get_cairo_version_result(process_id: String, engine: &State Result { +pub fn do_cairo_version() -> Result { let mut version_caller = Command::new("cargo"); version_caller.current_dir(DEFAULT_CAIRO_DIR); match String::from_utf8( @@ -50,15 +51,15 @@ pub fn do_cairo_version() -> Result { .arg("--version") .stdout(Stdio::piped()) .spawn() - .map_err(|e| format!("Failed to get cairo version: {:?}", e))? + .map_err(|e| ApiError::FailedToExecuteCommand(e))? .wait_with_output() - .map_err(|e| format!("Failed to get cairo version: {:?}", e))? + .map_err(|e| ApiError::FailedToReadOutput(e))? .stdout, ) { Ok(version) => Ok(version), Err(e) => { error!("{:?}", e.to_string()); - Err(e.to_string()) + Err(ApiError::UTF8Error(e)) } } } diff --git a/api/src/handlers/cairo_versions.rs b/api/src/handlers/cairo_versions.rs index b9e421f1..708de92e 100644 --- a/api/src/handlers/cairo_versions.rs +++ b/api/src/handlers/cairo_versions.rs @@ -1,3 +1,4 @@ +use crate::types::{ApiError, Result}; use crate::utils::lib::CAIRO_COMPILERS_DIR; use rocket::tokio::fs::read_dir; use std::path::Path; @@ -6,14 +7,18 @@ use tracing::instrument; #[instrument] #[get("/cairo_versions")] pub async fn cairo_versions() -> String { - do_cairo_versions().await.unwrap_or_else(|e| e) + do_cairo_versions() + .await + .unwrap_or_else(|e| format!("Failed to get cairo versions: {:?}", e)) } /// Get cairo versions -pub async fn do_cairo_versions() -> Result { +pub async fn do_cairo_versions() -> Result { let path = Path::new(CAIRO_COMPILERS_DIR); - let mut dir = read_dir(path).await.unwrap(); + let mut dir = read_dir(path) + .await + .map_err(|e| ApiError::FailedToReadDir(e))?; let mut result = vec![]; while let Ok(Some(entry)) = dir.next_entry().await { diff --git a/api/src/handlers/compile_casm.rs b/api/src/handlers/compile_casm.rs index 980fc710..a9db7939 100644 --- a/api/src/handlers/compile_casm.rs +++ b/api/src/handlers/compile_casm.rs @@ -1,5 +1,6 @@ use crate::handlers::process::{do_process_command, fetch_process_result}; use crate::handlers::types::{ApiCommand, ApiCommandResult, CompileResponse}; +use crate::types::{ApiError, Result}; use crate::utils::lib::{get_file_ext, get_file_path, CAIRO_COMPILERS_DIR, CASM_ROOT}; use crate::worker::WorkerEngine; use rocket::fs::NamedFile; @@ -20,7 +21,7 @@ pub async fn compile_to_casm(version: String, remix_file_path: PathBuf) -> Json< .unwrap_or_else(|e| { Json(CompileResponse { file_content: "".to_string(), - message: e, + message: format!("Failed to compile to casm: {:?}", e), status: "CompilationFailed".to_string(), cairo_version: version, }) @@ -59,34 +60,22 @@ pub async fn compile_to_casm_result(process_id: String, engine: &State Result, String> { - let remix_file_path = match remix_file_path.to_str() { - Some(path) => path.to_string(), - None => { - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "File path not found".to_string(), - status: "FileNotFound".to_string(), - cairo_version: version, - })); - } - }; +) -> Result> { + let remix_file_path = remix_file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(); // check if the file has .sierra extension match get_file_ext(&remix_file_path) { ext if ext == "sierra" => { debug!("LOG: File extension is sierra"); } - _ => { + ext => { debug!("LOG: File extension not supported"); - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "File extension not supported".to_string(), - status: "FileExtensionNotSupported".to_string(), - cairo_version: version, - })); + return Err(ApiError::FileExtensionNotSupported(ext)); } } @@ -96,17 +85,12 @@ pub async fn do_compile_to_casm( let mut compile = Command::new("cargo"); - let path_to_cairo_compiler = Path::new(CAIRO_COMPILERS_DIR).join(&version); + let path_to_cairo_compiler = Path::new(CAIRO_COMPILERS_DIR).join(&cairo_version); if path_to_cairo_compiler.exists() { compile.current_dir(path_to_cairo_compiler); } else { - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "Cairo compiler not found".to_string(), - status: "CairoCompilerNotFound".to_string(), - cairo_version: version, - })); + return Err(ApiError::CairoVersionNotFound(cairo_version)); } let casm_path = Path::new(CASM_ROOT).join(&casm_remix_path); @@ -135,66 +119,55 @@ pub async fn do_compile_to_casm( .arg(&file_path) .arg(&casm_path) .stderr(Stdio::piped()) - .spawn(); - - if result.is_err() { - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "Failed to execute starknet-sierra-compile".to_string(), - status: "SierraCompilationFailed".to_string(), - cairo_version: version, - })); - } - - let result = result.unwrap(); + .spawn() + .map_err(|e| ApiError::FailedToExecuteCommand(e))?; debug!("LOG: ran command:{:?}", compile); - let output = result.wait_with_output(); - - if output.is_err() { - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "Failed to wait on child".to_string(), - status: "SierraCompilationFailed".to_string(), - cairo_version: version, - })); + let output = result + .wait_with_output() + .map_err(|e| ApiError::FailedToReadOutput(e))?; + + let file_content = fs::read_to_string( + NamedFile::open(&casm_path) + .await + .map_err(|e| ApiError::FailedToReadFile(e))? + .path() + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + ) + .await + .map_err(|e| ApiError::FailedToReadFile(e))?; + + let message = String::from_utf8(output.stderr) + .map_err(|e| ApiError::UTF8Error(e))? + .replace( + &file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + &remix_file_path, + ) + .replace( + &casm_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + &casm_remix_path, + ); + + let status = match output.status.code() { + Some(0) => "Success", + Some(_) => "SierraCompilationFailed", + None => "UnknownError", } - - let output = output.unwrap(); + .to_string(); Ok(Json(CompileResponse { - file_content: match NamedFile::open(&casm_path).await.ok() { - Some(file) => match file.path().to_str() { - Some(path) => match fs::read_to_string(path.to_string()).await { - Ok(casm) => casm.to_string(), - Err(e) => e.to_string(), - }, - None => "".to_string(), - }, - None => "".to_string(), - }, - message: String::from_utf8(output.stderr) - .map_err(|e| format!("Failed to read stderr: {:?}", e))? - .replace( - &file_path - .to_str() - .ok_or("Failed to parse string".to_string())? - .to_string(), - &remix_file_path, - ) - .replace( - &casm_path - .to_str() - .ok_or("Failed to parse string".to_string())? - .to_string(), - &casm_remix_path, - ), - status: match output.status.code() { - Some(0) => "Success".to_string(), - Some(_) => "SierraCompilationFailed".to_string(), - None => "UnknownError".to_string(), - }, - cairo_version: version, + file_content, + message, + status, + cairo_version, })) } diff --git a/api/src/handlers/compile_sierra.rs b/api/src/handlers/compile_sierra.rs index 5728aa1d..33698f2c 100644 --- a/api/src/handlers/compile_sierra.rs +++ b/api/src/handlers/compile_sierra.rs @@ -1,5 +1,6 @@ use crate::handlers::process::{do_process_command, fetch_process_result}; use crate::handlers::types::{ApiCommand, ApiCommandResult, CompileResponse}; +use crate::types::{ApiError, Result}; use crate::utils::lib::{get_file_ext, get_file_path, CAIRO_COMPILERS_DIR, SIERRA_ROOT}; use crate::worker::WorkerEngine; use rocket::fs::NamedFile; @@ -22,7 +23,7 @@ pub async fn compile_to_sierra(version: String, remix_file_path: PathBuf) -> Jso Ok(res) => res, Err(e) => Json(CompileResponse { file_content: "".to_string(), - message: e, + message: format!("Failed to compile to sierra: {:?}", e), status: "CompilationFailed".to_string(), cairo_version: version, }), @@ -61,34 +62,22 @@ pub async fn get_siera_compile_result(process_id: String, engine: &State Result, String> { - let remix_file_path = match remix_file_path.to_str() { - Some(path) => path.to_string(), - None => { - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "File path not found".to_string(), - status: "FileNotFound".to_string(), - cairo_version: version, - })); - } - }; +) -> Result> { + let remix_file_path = remix_file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(); // check if the file has .cairo extension match get_file_ext(&remix_file_path) { ext if ext == "cairo" => { debug!("LOG: File extension is cairo"); } - _ => { + ext => { debug!("LOG: File extension not supported"); - return Ok(Json(CompileResponse { - file_content: "".to_string(), - message: "File extension not supported".to_string(), - status: "FileExtensionNotSupported".to_string(), - cairo_version: version, - })); + return Err(ApiError::CairoVersionNotFound(ext)); } } @@ -98,11 +87,11 @@ pub async fn do_compile_to_sierra( let mut compile = Command::new("cargo"); - let path_to_cairo_compiler = Path::new(CAIRO_COMPILERS_DIR).join(&version); + let path_to_cairo_compiler = Path::new(CAIRO_COMPILERS_DIR).join(&cairo_version); if path_to_cairo_compiler.exists() { compile.current_dir(path_to_cairo_compiler); } else { - return Err(format!("Cairo compiler with version {} not found", version)); + return Err(ApiError::CairoVersionNotFound(cairo_version)); } // replace .cairo with @@ -135,46 +124,53 @@ pub async fn do_compile_to_sierra( .stderr(Stdio::piped()) .stdout(Stdio::piped()) .spawn() - .map_err(|e| format!("Failed to execute starknet-compile: {:?}", e))?; + .map_err(|e| ApiError::FailedToExecuteCommand(e))?; debug!("LOG: ran command:{:?}", compile); let output = result .wait_with_output() - .map_err(|e| format!("Failed to wait on child: {:?}", e))?; + .map_err(|e| ApiError::FailedToReadOutput(e))?; + + let file_content = fs::read_to_string( + NamedFile::open(&sierra_path) + .await + .map_err(|e| ApiError::FailedToReadFile(e))? + .path() + .to_str() + .ok_or(ApiError::FailedToParseString)?, + ) + .await + .map_err(|e| ApiError::FailedToReadFile(e))?; + + let message = String::from_utf8(output.stderr) + .map_err(|e| ApiError::UTF8Error(e))? + .replace( + &file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + &remix_file_path, + ) + .replace( + &sierra_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + &sierra_remix_path, + ); + + let status = match output.status.code() { + Some(0) => "Success", + Some(_) => "CompilationFailed", + None => "UnknownError", + } + .to_string(); Ok(Json(CompileResponse { - file_content: match NamedFile::open(&sierra_path).await.ok() { - Some(file) => match file.path().to_str() { - Some(path) => match fs::read_to_string(path.to_string()).await { - Ok(sierra) => sierra.to_string(), - Err(e) => e.to_string(), - }, - None => "".to_string(), - }, - None => "".to_string(), - }, - message: String::from_utf8(output.stderr) - .map_err(|e| format!("Failed to read stderr: {:?}", e))? - .replace( - &file_path - .to_str() - .ok_or(format!("Formation of filepath failed"))? - .to_string(), - &remix_file_path, - ) - .replace( - &sierra_path - .to_str() - .ok_or(format!("Formation of filepath failed"))? - .to_string(), - &sierra_remix_path, - ), - status: match output.status.code() { - Some(0) => "Success".to_string(), - Some(_) => "CompilationFailed".to_string(), - None => "UnknownError".to_string(), - }, - cairo_version: version, + file_content, + message, + status, + cairo_version, })) } diff --git a/api/src/handlers/mod.rs b/api/src/handlers/mod.rs index 5f4b76f0..c3b5db69 100644 --- a/api/src/handlers/mod.rs +++ b/api/src/handlers/mod.rs @@ -12,6 +12,7 @@ use crate::handlers::compile_casm::do_compile_to_casm; use crate::handlers::compile_sierra::do_compile_to_sierra; use crate::handlers::scarb_compile::do_scarb_compile; use crate::handlers::types::{ApiCommand, ApiCommandResult, FileContentMap}; +use crate::types::{ApiError, Result}; use rocket::serde::json::Json; use std::path::Path; use tracing::info; @@ -31,7 +32,7 @@ pub async fn who_is_this() -> &'static str { "Who are you?" } -pub async fn dispatch_command(command: ApiCommand) -> Result { +pub async fn dispatch_command(command: ApiCommand) -> Result { match command { ApiCommand::CairoVersion => match do_cairo_version() { Ok(result) => Ok(ApiCommandResult::CairoVersion(result)), @@ -63,13 +64,13 @@ pub async fn dispatch_command(command: ApiCommand) -> Result Result, String> { +fn get_files_recursive(base_path: &Path) -> Result> { let mut file_content_map_array: Vec = Vec::new(); if base_path.is_dir() { for entry in base_path .read_dir() - .map_err(|e| format!("Error while reading dir {:?}", e))? + .map_err(|e| ApiError::FailedToReadDir(e))? .flatten() { let path = entry.path(); @@ -78,7 +79,7 @@ fn get_files_recursive(base_path: &Path) -> Result, String> } else if let Ok(content) = std::fs::read_to_string(&path) { let file_name = path .file_name() - .ok_or(format!("Error while reading file name"))? + .ok_or(ApiError::FailedToReadFilename)? .to_string_lossy() .to_string(); let file_content = content; diff --git a/api/src/handlers/save_code.rs b/api/src/handlers/save_code.rs index a4a73f77..1bff6a20 100644 --- a/api/src/handlers/save_code.rs +++ b/api/src/handlers/save_code.rs @@ -1,3 +1,4 @@ +use crate::types::{ApiError, Result}; use crate::utils::lib::get_file_path; use rocket::data::ToByteUnit; use rocket::tokio::fs; @@ -8,18 +9,18 @@ use tracing::info; #[post("/save_code/", data = "")] pub async fn save_code(file: Data<'_>, remix_file_path: PathBuf) -> String { info!("/save_code/{:?}", remix_file_path); - do_save_code(file, remix_file_path).await + do_save_code(file, remix_file_path) + .await + .unwrap_or_else(|e| format!("Failed to save code: {:?}", e)) } /// Upload a data file /// -pub async fn do_save_code(file: Data<'_>, remix_file_path: PathBuf) -> String { - let remix_file_path = match remix_file_path.to_str() { - Some(path) => path.to_string(), - None => { - return "".to_string(); - } - }; +pub async fn do_save_code(file: Data<'_>, remix_file_path: PathBuf) -> Result { + let remix_file_path = remix_file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(); let file_path = get_file_path(&remix_file_path); @@ -39,20 +40,14 @@ pub async fn do_save_code(file: Data<'_>, remix_file_path: PathBuf) -> String { } // Modify to zip and unpack. - let saved_file = file.open(128_i32.gibibytes()).into_file(&file_path).await; + let _ = file + .open(128_i32.gibibytes()) + .into_file(&file_path) + .await + .map_err(|e| ApiError::FailedToSaveFile(e))?; - match saved_file { - Ok(_) => { - debug!("LOG: File saved successfully"); - match file_path.to_str() { - Some(path) => path.to_string(), - None => "".to_string(), - } - } - Err(e) => { - debug!("LOG: Error saving file: {:?}", e); - "".to_string() - // set the response with not ok code. - } - } + Ok(file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string()) } diff --git a/api/src/handlers/scarb_compile.rs b/api/src/handlers/scarb_compile.rs index 743d9950..84c5e7af 100644 --- a/api/src/handlers/scarb_compile.rs +++ b/api/src/handlers/scarb_compile.rs @@ -1,6 +1,7 @@ use crate::handlers::get_files_recursive; use crate::handlers::process::{do_process_command, fetch_process_result}; use crate::handlers::types::{ApiCommand, ApiCommandResult, ScarbCompileResponse}; +use crate::types::{ApiError, Result}; use crate::utils::lib::get_file_path; use crate::worker::WorkerEngine; use rocket::serde::json; @@ -14,7 +15,13 @@ use tracing::{info, instrument}; #[get("/compile-scarb/")] pub async fn scarb_compile(remix_file_path: PathBuf) -> Json { info!("/compile-scarb/{:?}", remix_file_path); - do_scarb_compile(remix_file_path).await.unwrap() + do_scarb_compile(remix_file_path).await.unwrap_or_else(|e| { + Json(ScarbCompileResponse { + file_content_map_array: vec![], + message: format!("Failed to compile to scarb: {:?}", e), + status: "CompilationFailed".to_string(), + }) + }) } #[instrument] @@ -29,26 +36,19 @@ pub async fn scarb_compile_async(remix_file_path: PathBuf, engine: &State) -> String { info!("/compile-scarb-result/{:?}", process_id); fetch_process_result(process_id, engine, |result| match result { - ApiCommandResult::ScarbCompile(scarb_result) => json::to_string(&scarb_result).unwrap(), + ApiCommandResult::ScarbCompile(scarb_result) => json::to_string(&scarb_result) + .unwrap_or_else(|e| format!("Failed to fetch result: {:?}", e)), _ => String::from("Result not available"), }) } /// Run Scarb to compile a project /// -pub async fn do_scarb_compile( - remix_file_path: PathBuf, -) -> Result, String> { - let remix_file_path = match remix_file_path.to_str() { - Some(path) => path.to_string(), - None => { - return Ok(Json(ScarbCompileResponse { - file_content_map_array: vec![], - message: "File path not found".to_string(), - status: "FileNotFound".to_string(), - })); - } - }; +pub async fn do_scarb_compile(remix_file_path: PathBuf) -> Result> { + let remix_file_path = remix_file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(); let file_path = get_file_path(&remix_file_path); @@ -60,26 +60,45 @@ pub async fn do_scarb_compile( .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .map_err(|e| format!("Failed to execute scarb build: {:?}", e))?; + .map_err(|e| ApiError::FailedToExecuteCommand(e))?; debug!("LOG: ran command:{:?}", compile); let output = result .wait_with_output() - .map_err(|e| format!("Failed to wait: {:?}", e))?; + .map_err(|e| ApiError::FailedToReadOutput(e))?; + + let file_content_map_array = get_files_recursive(&file_path.join("target/dev"))?; + + let message = String::from_utf8(output.stdout) + .map_err(|e| ApiError::UTF8Error(e))? + .replace( + &file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + &remix_file_path, + ) + + &String::from_utf8(output.stderr) + .map_err(|e| ApiError::UTF8Error(e))? + .replace( + &file_path + .to_str() + .ok_or(ApiError::FailedToParseString)? + .to_string(), + &remix_file_path, + ); + + let status = match output.status.code() { + Some(0) => "Success", + Some(_) => "SierraCompilationFailed", + None => "UnknownError", + } + .to_string(); Ok(Json(ScarbCompileResponse { - file_content_map_array: get_files_recursive(&file_path.join("target/dev"))?, - message: String::from_utf8(output.stdout) - .map_err(|e| format!("Failed to read stdout: {:?}", e))? - .replace(&file_path.to_str().unwrap().to_string(), &remix_file_path) - + &String::from_utf8(output.stderr) - .map_err(|e| format!("Failed to read stderr: {:?}", e))? - .replace(&file_path.to_str().unwrap().to_string(), &remix_file_path), - status: match output.status.code() { - Some(0) => "Success".to_string(), - Some(_) => "SierraCompilationFailed".to_string(), - None => "UnknownError".to_string(), - }, + file_content_map_array, + message, + status, })) } diff --git a/api/src/main.rs b/api/src/main.rs index d391dec4..af5a3f9b 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -4,6 +4,7 @@ extern crate rocket; pub mod cors; pub mod handlers; pub mod tracing_log; +pub mod types; pub mod utils; pub mod worker; diff --git a/api/src/types.rs b/api/src/types.rs new file mode 100644 index 00000000..531853ed --- /dev/null +++ b/api/src/types.rs @@ -0,0 +1,27 @@ +use std::io::Error as IoError; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum ApiError { + #[error("Failed to execute command: {0}")] + FailedToExecuteCommand(IoError), + #[error("Failed to read output: {0}")] + FailedToReadOutput(IoError), + #[error("UTF8 error: {0}")] + UTF8Error(#[from] std::string::FromUtf8Error), + #[error("Failed to read dir: {0}")] + FailedToReadDir(IoError), + #[error("Failed to read file: {0}")] + FailedToReadFile(IoError), + #[error("Failed to parse string")] + FailedToParseString, + #[error("File extension <{0}> not supported")] + FileExtensionNotSupported(String), + #[error("Cairo version {0} not found")] + CairoVersionNotFound(String), + #[error("Failed to save file: {0}")] + FailedToSaveFile(IoError), + #[error("Failed to read filename")] + FailedToReadFilename, +} diff --git a/api/src/worker.rs b/api/src/worker.rs index 2297810d..a433ea4c 100644 --- a/api/src/worker.rs +++ b/api/src/worker.rs @@ -1,5 +1,6 @@ use crate::handlers; use crate::handlers::types::{ApiCommand, ApiCommandResult}; +use crate::types::ApiError; use crossbeam_queue::ArrayQueue; use crossbeam_skiplist::SkipMap; use rocket::tokio; @@ -15,7 +16,7 @@ pub enum ProcessState { New, Running, Completed(ApiCommandResult), - Error(String), + Error(ApiError), } impl Display for ProcessState { @@ -24,7 +25,7 @@ impl Display for ProcessState { ProcessState::New => write!(f, "New"), ProcessState::Running => write!(f, "Running"), ProcessState::Completed(_) => write!(f, "Completed"), - ProcessState::Error(e) => write!(f, "Error({})", e), + ProcessState::Error(e) => write!(f, "Error({:?})", e), } } } From fd0f36de905411d037a5fe2e6f4feb7c3e4817b6 Mon Sep 17 00:00:00 2001 From: varex83 Date: Wed, 18 Oct 2023 21:26:38 +0300 Subject: [PATCH 3/4] implement map purging --- api/Cargo.lock | 146 +++++++++++++++++++++++++++++++++++++++++++ api/Cargo.toml | 3 +- api/src/utils/lib.rs | 2 + api/src/worker.rs | 114 ++++++++++++++++++++++++++++++++- 4 files changed, 261 insertions(+), 4 deletions(-) diff --git a/api/Cargo.lock b/api/Cargo.lock index 2ba99cdc..1ff394fa 100644 --- a/api/Cargo.lock +++ b/api/Cargo.lock @@ -11,10 +11,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "api" version = "0.1.0" dependencies = [ + "chrono", "crossbeam", "crossbeam-queue", "crossbeam-skiplist", @@ -98,6 +114,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + [[package]] name = "bytes" version = "1.4.0" @@ -116,6 +138,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + [[package]] name = "cookie" version = "0.17.0" @@ -127,6 +163,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "crossbeam" version = "0.8.2" @@ -538,6 +580,29 @@ dependencies = [ "want", ] +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -591,6 +656,15 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -718,6 +792,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -1511,6 +1594,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + [[package]] name = "winapi" version = "0.3.9" @@ -1542,6 +1679,15 @@ dependencies = [ "windows-targets 0.42.1", ] +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.42.0" diff --git a/api/Cargo.toml b/api/Cargo.toml index 9e2b1af4..a7625f3a 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -20,4 +20,5 @@ crossbeam-queue = "0.3.8" crossbeam-skiplist = "0.1.1" fmt = "0.1.0" yansi = "0.5.1" -thiserror = "1.0.49" \ No newline at end of file +thiserror = "1.0.49" +chrono = "0.4.31" \ No newline at end of file diff --git a/api/src/utils/lib.rs b/api/src/utils/lib.rs index 0364296d..495d1154 100644 --- a/api/src/utils/lib.rs +++ b/api/src/utils/lib.rs @@ -16,6 +16,8 @@ pub const TEMP_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/", "temp/"); pub const DEFAULT_CAIRO_VERSION: &str = "v2.2.0"; +pub const DURATION_TO_PURGE: u64 = 5; // 5 seconds + pub fn get_file_ext(file_path: &str) -> String { match file_path.split('.').last() { Some(ext) => ext.to_string(), diff --git a/api/src/worker.rs b/api/src/worker.rs index a433ea4c..1bb46888 100644 --- a/api/src/worker.rs +++ b/api/src/worker.rs @@ -1,14 +1,17 @@ use crate::handlers; use crate::handlers::types::{ApiCommand, ApiCommandResult}; use crate::types::ApiError; +use crate::utils::lib::DURATION_TO_PURGE; use crossbeam_queue::ArrayQueue; use crossbeam_skiplist::SkipMap; use rocket::tokio; +use rocket::tokio::sync::Mutex; use rocket::tokio::task::JoinHandle; use rocket::tokio::time; use rocket::tokio::time::sleep; use std::fmt::{Display, Formatter}; use std::sync::Arc; +use tracing::info; use uuid::Uuid; #[derive(Debug)] @@ -30,12 +33,18 @@ impl Display for ProcessState { } } +pub type ProcessStateMap = SkipMap; +pub type Timestamp = u64; + #[derive(Debug)] pub struct WorkerEngine { pub num_workers: u32, pub worker_threads: Vec>, pub arc_command_queue: Arc>, - pub arc_process_states: Arc>, + pub arc_process_states: Arc, + pub arc_timestamps_to_purge: Arc>, + pub is_supervisor_enabled: Arc>, + pub supervisor_thread: Arc>>, } impl WorkerEngine { @@ -52,11 +61,21 @@ impl WorkerEngine { // Create a collection of worker threads let worker_threads: Vec> = vec![]; + // Create a flag to enable/disable the supervisor thread + let is_supervisor_enabled = Arc::new(Mutex::new(true)); + + // Create a collection of timestamps to purge + let timestamps_to_purge: ArrayQueue<(Uuid, Timestamp)> = ArrayQueue::new(queue_capacity); + let arc_timestamps_to_purge = Arc::new(timestamps_to_purge); + WorkerEngine { num_workers, arc_command_queue, arc_process_states, worker_threads, + supervisor_thread: Arc::new(None), + arc_timestamps_to_purge, + is_supervisor_enabled, } } @@ -65,10 +84,86 @@ impl WorkerEngine { // add to collection let arc_clone = self.arc_command_queue.clone(); let arc_states = self.arc_process_states.clone(); + let arc_timestamps_to_purge = self.arc_timestamps_to_purge.clone(); self.worker_threads.push(tokio::spawn(async move { - WorkerEngine::worker(arc_clone, arc_states).await; + WorkerEngine::worker(arc_clone, arc_states, arc_timestamps_to_purge).await; })); } + + // start supervisor thread + { + let is_supervisor_enabled = self.is_supervisor_enabled.clone(); + let arc_process_states = self.arc_process_states.clone(); + let process_timestamps_to_purge = self.arc_timestamps_to_purge.clone(); + + self.supervisor_thread = Arc::new(Some(tokio::spawn(async move { + WorkerEngine::supervisor( + is_supervisor_enabled, + arc_process_states, + process_timestamps_to_purge, + ) + .await; + }))); + } + } + + pub async fn enable_supervisor_thread(&mut self) { + let is_supervisor_enabled = self.is_supervisor_enabled.clone(); + let arc_process_states = self.arc_process_states.clone(); + let process_timestamps_to_purge = self.arc_timestamps_to_purge.clone(); + let mut is_enabled = self.is_supervisor_enabled.lock().await; + *is_enabled = true; + self.supervisor_thread = Arc::new(Some(tokio::spawn(async move { + WorkerEngine::supervisor( + is_supervisor_enabled, + arc_process_states, + process_timestamps_to_purge, + ) + .await; + }))); + } + + pub async fn supervisor( + is_supervisor_enabled: Arc>, + arc_process_states: Arc, + process_timestamps_to_purge: Arc>, + ) { + loop { + let is_supervisor_enabled = is_supervisor_enabled.lock().await; + if !*is_supervisor_enabled { + break; + } + + let now = Self::timestamp(); + + while let Some((process_id, timestamp)) = process_timestamps_to_purge.pop() { + if timestamp < now { + arc_process_states.remove(&process_id); + + info!("Process {:?} removed from process states", process_id); + } else { + process_timestamps_to_purge + .push((process_id, timestamp)) + .unwrap(); + break; + } + } + + sleep(time::Duration::from_millis(2000)).await; + } + } + + pub async fn disable_supervisor_thread(&mut self) { + let mut is_enabled = self.is_supervisor_enabled.lock().await; + *is_enabled = false; + + if let Ok(supervisor_thread) = Arc::try_unwrap(self.supervisor_thread.clone()) { + if let Some(join_handle) = supervisor_thread { + let _ = join_handle.await; + } + } + + self.supervisor_thread = Arc::new(None); } pub fn enqueue_command(&self, command: ApiCommand) -> Result { @@ -88,7 +183,8 @@ impl WorkerEngine { // worker function pub async fn worker( arc_command_queue: Arc>, - arc_process_states: Arc>, + arc_process_states: Arc, + arc_timestamps_to_purge: Arc>, ) { info!("Starting worker thread..."); 'worker_loop: loop { @@ -109,9 +205,17 @@ impl WorkerEngine { Ok(result) => { arc_process_states .insert(process_id, ProcessState::Completed(result)); + + arc_timestamps_to_purge + .push((process_id, Self::timestamp() + DURATION_TO_PURGE)) + .unwrap(); } Err(e) => { arc_process_states.insert(process_id, ProcessState::Error(e)); + + arc_timestamps_to_purge + .push((process_id, Self::timestamp() + DURATION_TO_PURGE)) + .unwrap(); } } } @@ -125,4 +229,8 @@ impl WorkerEngine { } info!("Worker thread finished..."); } + + fn timestamp() -> u64 { + chrono::Utc::now().timestamp() as u64 + } } From 2b42e8c7b9773db1d59eb7b0bd199131d82422f9 Mon Sep 17 00:00:00 2001 From: varex83 Date: Wed, 18 Oct 2023 21:29:06 +0300 Subject: [PATCH 4/4] implement map purging --- api/src/utils/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/utils/lib.rs b/api/src/utils/lib.rs index 495d1154..e0f7426d 100644 --- a/api/src/utils/lib.rs +++ b/api/src/utils/lib.rs @@ -16,7 +16,7 @@ pub const TEMP_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/", "temp/"); pub const DEFAULT_CAIRO_VERSION: &str = "v2.2.0"; -pub const DURATION_TO_PURGE: u64 = 5; // 5 seconds +pub const DURATION_TO_PURGE: u64 = 60 * 5; // 5 minutes pub fn get_file_ext(file_path: &str) -> String { match file_path.split('.').last() {