From d813ff12d084ffcc257fcb1cab99fac53abb4a44 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Thu, 4 Apr 2024 04:59:11 +0000 Subject: [PATCH 1/2] fix graph execution issue --- Cargo.lock | 10 ++--- crates/cli/Cargo.toml | 6 +-- crates/common/Cargo.toml | 4 +- crates/common/src/cache.rs | 12 ++++-- crates/common/src/common.rs | 18 ++++----- crates/common/src/directory.rs | 6 +++ crates/common/src/file.rs | 23 ++++++++++- crates/common/src/git.rs | 14 +++++-- crates/common/src/http.rs | 6 +-- crates/core/Cargo.toml | 2 +- crates/core/src/deps.rs | 40 ++++++++++--------- crates/graphql/Cargo.toml | 6 +-- crates/graphql/src/schema/objects/pipeline.rs | 16 ++++++-- crates/server/Cargo.toml | 6 +-- fixtures/.gitignore | 2 + 15 files changed, 108 insertions(+), 63 deletions(-) create mode 100644 fixtures/.gitignore diff --git a/Cargo.lock b/Cargo.lock index 88855fd..0f944b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1872,7 +1872,7 @@ dependencies = [ [[package]] name = "fluentci-common" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "dirs 5.0.1", @@ -1886,7 +1886,7 @@ dependencies = [ [[package]] name = "fluentci-core" -version = "0.1.5" +version = "0.1.6" dependencies = [ "anyhow", "chrono", @@ -1903,7 +1903,7 @@ dependencies = [ [[package]] name = "fluentci-engine" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "clap", @@ -1942,7 +1942,7 @@ dependencies = [ [[package]] name = "fluentci-graphql" -version = "0.1.6" +version = "0.1.7" dependencies = [ "anyhow", "async-graphql", @@ -1969,7 +1969,7 @@ dependencies = [ [[package]] name = "fluentci-server" -version = "0.1.6" +version = "0.1.7" dependencies = [ "actix-cors", "actix-web", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index e3dab8a..5cec23a 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" name = "fluentci-engine" readme = "../../README.md" repository = "https://github.com/fluentci-io/fluentci-engine" -version = "0.2.1" +version = "0.2.2" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -16,9 +16,9 @@ version = "0.2.1" anyhow = "1.0.81" clap = "3.2.20" extism = "1.2.0" -fluentci-core = {path = "../core", version = "0.1.5"} +fluentci-core = {path = "../core", version = "0.1.6"} fluentci-ext = {path = "../ext", version = "0.1.4"} -fluentci-server = {path = "../server", version = "0.1.6"} +fluentci-server = {path = "../server", version = "0.1.7"} fluentci-shared = {path = "../shared", version = "0.1.1"} get-port = "4.0.0" md5 = "0.7.0" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 36a6a01..f4d3e4f 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -7,14 +7,14 @@ keywords = ["nix", "environment", "ci", "cd", "devops"] license = "MPL-2.0" name = "fluentci-common" repository = "https://github.com/fluentci-io/fluentci-engine" -version = "0.1.1" +version = "0.1.2" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "1.0.81" dirs = "5.0.1" -fluentci-core = {path = "../core", version = "0.1.5"} +fluentci-core = {path = "../core", version = "0.1.6"} fluentci-ext = {path = "../ext", version = "0.1.4"} fluentci-types = {path = "../types", version = "0.1.2"} regex = "1.10.4" diff --git a/crates/common/src/cache.rs b/crates/common/src/cache.rs index 066a97f..982230d 100644 --- a/crates/common/src/cache.rs +++ b/crates/common/src/cache.rs @@ -11,14 +11,14 @@ pub fn cache(graph: Arc>, key: &str) -> Result { graph.reset(); let cache = graph - .vertices + .volumes .iter() - .find(|v| v.label == "cache" && v.command == key); + .find(|v| v.label == "cache" && v.key == key); if let Some(cache) = cache { return Ok(Cache { id: cache.id.clone(), - key: cache.command.clone(), + key: cache.key.clone(), path: "".into(), }); } @@ -32,6 +32,12 @@ pub fn cache(graph: Arc>, key: &str) -> Result { Arc::new(Box::new(CacheExt::default())), )); + graph.execute(GraphCommand::AddVolume( + id.clone(), + "cache".into(), + key.into(), + )); + let cache = Cache { id, key: key.into(), diff --git a/crates/common/src/common.rs b/crates/common/src/common.rs index 6943851..31b2d1a 100644 --- a/crates/common/src/common.rs +++ b/crates/common/src/common.rs @@ -90,14 +90,14 @@ pub fn with_cache(graph: Arc>, cache_id: String, path: String) -> R graph.runner = Arc::new(Box::new(CacheExt::default())); graph.runner.setup()?; - if let Some(cache) = graph.vertices.iter().find(|v| v.id.clone() == cache_id) { + if let Some(cache) = graph.volumes.iter().find(|v| v.id.clone() == cache_id) { let id = Uuid::new_v4().to_string(); let dep_id = graph.vertices[graph.size() - 1].id.clone(); let deps = match graph.size() { 1 => vec![], _ => vec![dep_id], }; - let cache_key_path = format!("{}:{}", cache.command, path); + let cache_key_path = format!("{}:{}", cache.path, path); graph.execute(GraphCommand::AddVertex( id.clone(), "withCache".into(), @@ -121,17 +121,17 @@ pub fn with_cache(graph: Arc>, cache_id: String, path: String) -> R pub fn with_file(graph: Arc>, file_id: String, path: String) -> Result<(), Error> { let mut graph = graph.lock().unwrap(); let runner = graph.runner.clone(); - graph.runner = Arc::new(Box::new(CacheExt::default())); + graph.runner = Arc::new(Box::new(Runner::default())); graph.runner.setup()?; - if let Some(file) = graph.vertices.iter().find(|v| v.id.clone() == file_id) { + if let Some(file) = graph.volumes.iter().find(|v| v.id.clone() == file_id) { let id = Uuid::new_v4().to_string(); let dep_id = graph.vertices[graph.size() - 1].id.clone(); let deps = match graph.size() { 1 => vec![], _ => vec![dep_id], }; - let copy_file = format!("cp {} {}", file.command, path); + let copy_file = format!("cp {} {}", file.path, path); graph.execute(GraphCommand::AddVertex( id.clone(), "withFile".into(), @@ -226,12 +226,10 @@ pub fn zip(graph: Arc>, path: String) -> Result { path: format!("{}/{}", parent_dir, output_file), }; - graph.execute(GraphCommand::AddVertex( + graph.execute(GraphCommand::AddVolume( id, "file".into(), file.path.clone(), - vec![], - Arc::new(Box::new(Runner::default())), )); Ok(file) @@ -273,12 +271,10 @@ pub fn tar_czvf(graph: Arc>, path: String) -> Result { path: format!("{}/{}", parent_dir, output_file), }; - graph.execute(GraphCommand::AddVertex( + graph.execute(GraphCommand::AddVolume( id, "file".into(), file.path.clone(), - vec![], - Arc::new(Box::new(Runner::default())), )); Ok(file) diff --git a/crates/common/src/directory.rs b/crates/common/src/directory.rs index a6548a1..ffe8f11 100644 --- a/crates/common/src/directory.rs +++ b/crates/common/src/directory.rs @@ -43,6 +43,12 @@ pub fn directory(graph: Arc>, path: String, reset: bool) -> Result< Arc::new(Box::new(Runner::default())), )); + graph.execute(GraphCommand::AddVolume( + id.clone(), + "directory".into(), + path.clone(), + )); + let path = canonicalize(path).unwrap().to_str().unwrap().to_string(); let directory = Directory { id, path }; Ok(directory) diff --git a/crates/common/src/file.rs b/crates/common/src/file.rs index 20b7d83..6afe311 100644 --- a/crates/common/src/file.rs +++ b/crates/common/src/file.rs @@ -44,6 +44,12 @@ pub fn file(graph: Arc>, path: String, reset: bool) -> Result>) -> Result { graph.execute(GraphCommand::AddEdge(x, y)); graph.runner = Arc::new(Box::new(RunnerExt::default())); - let directory = Directory { - id, - path: graph.work_dir.clone(), - }; + let path = graph.work_dir.clone(); + + graph.execute(GraphCommand::AddVolume( + id.clone(), + "directory".into(), + path.clone(), + )); + + let directory = Directory { id, path }; + Ok(directory) } diff --git a/crates/common/src/http.rs b/crates/common/src/http.rs index 38a8e1b..e69998e 100644 --- a/crates/common/src/http.rs +++ b/crates/common/src/http.rs @@ -5,7 +5,7 @@ use std::{ use anyhow::Error; use fluentci_core::deps::{Graph, GraphCommand}; -use fluentci_ext::{http::Http as HttpExt, runner::Runner}; +use fluentci_ext::http::Http as HttpExt; use fluentci_types::file::File; use uuid::Uuid; @@ -43,12 +43,10 @@ pub fn http(graph: Arc>, url: String, reset: bool) -> Result, pub edges: Vec, + pub volumes: Vec, tx: Sender<(String, usize)>, pub runner: Arc>, pub work_dir: String, @@ -42,6 +52,7 @@ impl Graph { let work_dir = current_dir().unwrap().to_str().unwrap().to_string(); Graph { vertices: Vec::new(), + volumes: Vec::new(), edges: Vec::new(), tx, runner, @@ -51,23 +62,15 @@ impl Graph { pub fn execute(&mut self, command: GraphCommand) { match command { + GraphCommand::AddVolume(id, label, path) => { + self.volumes.push(Volume { + id, + label, + key: path.clone(), + path, + }); + } GraphCommand::AddVertex(id, label, command, needs, runner) => { - if label == "cache" - && self - .vertices - .iter() - .any(|v| v.command == command && v.label == "cache") - { - return; - } - if label == "file" - && self - .vertices - .iter() - .any(|v| v.command == command && v.label == "file") - { - return; - } if let Some(vertex) = self.vertices.iter_mut().find(|v| v.id == id) { vertex.needs.extend(needs); } else { @@ -107,9 +110,9 @@ impl Graph { "git-last-commit", "tree", "http", + "cache", "file", "directory", - "cache", "chmod", "withFile", ]; @@ -327,8 +330,7 @@ impl Graph { } pub fn reset(&mut self) { - let keep = vec!["cache", "file", "directory"]; - self.vertices.retain(|v| keep.contains(&v.label.as_str())); + self.vertices.clear(); self.edges.clear(); self.work_dir = current_dir().unwrap().to_str().unwrap().to_string(); } diff --git a/crates/graphql/Cargo.toml b/crates/graphql/Cargo.toml index 2bd9417..875fa08 100644 --- a/crates/graphql/Cargo.toml +++ b/crates/graphql/Cargo.toml @@ -7,7 +7,7 @@ keywords = ["nix", "environment", "ci", "cd", "devops"] license = "MPL-2.0" name = "fluentci-graphql" repository = "https://github.com/fluentci-io/fluentci-engine" -version = "0.1.6" +version = "0.1.7" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -16,8 +16,8 @@ anyhow = "1.0.80" async-graphql = "7.0.2" async-graphql-actix-web = "7.0.2" dirs = "5.0.1" -fluentci-common = {path = "../common", version = "0.1.1"} -fluentci-core = {path = "../core", version = "0.1.5"} +fluentci-common = {path = "../common", version = "0.1.2"} +fluentci-core = {path = "../core", version = "0.1.6"} fluentci-ext = {path = "../ext", version = "0.1.4"} fluentci-types = {path = "../types", version = "0.1.1"} regex = "1.10.3" diff --git a/crates/graphql/src/schema/objects/pipeline.rs b/crates/graphql/src/schema/objects/pipeline.rs index 2b39ae2..403f170 100644 --- a/crates/graphql/src/schema/objects/pipeline.rs +++ b/crates/graphql/src/schema/objects/pipeline.rs @@ -58,10 +58,11 @@ impl Pipeline { 1 => vec![], _ => vec![dep_id], }; + graph.execute(GraphCommand::AddVertex( id.clone(), "http".into(), - url, + url.clone(), deps, Arc::new(Box::new(HttpExt::default())), )); @@ -72,11 +73,20 @@ impl Pipeline { let y = graph.size() - 1; graph.execute(GraphCommand::AddEdge(x, y)); } + let filename = sha256::digest(url).to_string(); + let work_dir = graph.work_dir.clone(); let file = File { - id: ID(id), - path: "/file".into(), + id: ID(id.clone()), + path: format!("{}/{}", work_dir, filename), }; + + graph.execute(GraphCommand::AddVolume( + id, + "file".into(), + file.path.clone(), + )); + Ok(file) } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index fc2eb60..b33a4fc 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -7,7 +7,7 @@ keywords = ["nix", "environment", "ci", "cd", "devops"] license = "MPL-2.0" name = "fluentci-server" repository = "https://github.com/fluentci-io/fluentci-engine" -version = "0.1.6" +version = "0.1.7" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -17,9 +17,9 @@ actix-web = "4.5.1" anyhow = "1.0.81" async-graphql = "7.0.2" async-graphql-actix-web = "7.0.2" -fluentci-core = {path = "../core", version = "0.1.5"} +fluentci-core = {path = "../core", version = "0.1.6"} fluentci-ext = {path = "../ext", version = "0.1.4"} -fluentci-graphql = {path = "../graphql", version = "0.1.5"} +fluentci-graphql = {path = "../graphql", version = "0.1.6"} mime_guess = "2.0.4" owo-colors = "4.0.0" tokio = "1.36.0" diff --git a/fixtures/.gitignore b/fixtures/.gitignore new file mode 100644 index 0000000..baae5e6 --- /dev/null +++ b/fixtures/.gitignore @@ -0,0 +1,2 @@ +*.zip +*.tar.gz \ No newline at end of file From 67795768f4edb8e32b019196917c4e5599a4e493 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Thu, 4 Apr 2024 05:00:30 +0000 Subject: [PATCH 2/2] ci: enable DAGGER_CLOUD_TOKEN --- .github/workflows/ci.yml | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9c34d7..56047af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: env: PACKAGE_NAME: fluentci-engine WORK_DIR: ./fixtures - #DAGGER_CLOUD_TOKEN: ${{ secrets.DAGGER_CLOUD_TOKEN }} + DAGGER_CLOUD_TOKEN: ${{ secrets.DAGGER_CLOUD_TOKEN }} FLUENTCI_CACHE_CDN_ENDPOINT: https://cache.fluentci.io FLUENTCI_CACHE_S3_ENDPOINT: https://fe5b1e2ce9f94f4c0415ab94ce402012.r2.cloudflarestorage.com FLUENTCI_CACHE_S3_BUCKET: cache diff --git a/flake.nix b/flake.nix index e7e74b7..4555a56 100644 --- a/flake.nix +++ b/flake.nix @@ -40,7 +40,7 @@ inherit src; pname = "fluentci-engine"; - version = "0.1.0"; + version = "0.2.2"; cargoExtraArgs = "--package=fluentci-engine"; buildInputs = [