Merge pull request #16 from fluentci-io/fix/graph-exec

fix graph execution issue

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
         env:
           PACKAGE_NAME: fluentci-engine
           WORK_DIR: ./fixtures
-          #DAGGER_CLOUD_TOKEN: ${{ secrets.DAGGER_CLOUD_TOKEN }}
+          DAGGER_CLOUD_TOKEN: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           FLUENTCI_CACHE_CDN_ENDPOINT: https://cache.fluentci.io
           FLUENTCI_CACHE_S3_ENDPOINT: https://fe5b1e2ce9f94f4c0415ab94ce402012.r2.cloudflarestorage.com
           FLUENTCI_CACHE_S3_BUCKET: cache

crates/cli/Cargo.toml

@@ -8,17 +8,17 @@ license = "MPL-2.0"
 name = "fluentci-engine"
 readme = "../../README.md"
 repository = "https://github.com/fluentci-io/fluentci-engine"
-version = "0.2.1"
+version = "0.2.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 anyhow = "1.0.81"
 clap = "3.2.20"
 extism = "1.2.0"
-fluentci-core = {path = "../core", version = "0.1.5"}
+fluentci-core = {path = "../core", version = "0.1.6"}
 fluentci-ext = {path = "../ext", version = "0.1.4"}
-fluentci-server = {path = "../server", version = "0.1.6"}
+fluentci-server = {path = "../server", version = "0.1.7"}
 fluentci-shared = {path = "../shared", version = "0.1.1"}
 get-port = "4.0.0"
 md5 = "0.7.0"

crates/common/Cargo.toml

@@ -7,14 +7,14 @@ keywords = ["nix", "environment", "ci", "cd", "devops"]
 license = "MPL-2.0"
 name = "fluentci-common"
 repository = "https://github.com/fluentci-io/fluentci-engine"
-version = "0.1.1"
+version = "0.1.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 anyhow = "1.0.81"
 dirs = "5.0.1"
-fluentci-core = {path = "../core", version = "0.1.5"}
+fluentci-core = {path = "../core", version = "0.1.6"}
 fluentci-ext = {path = "../ext", version = "0.1.4"}
 fluentci-types = {path = "../types", version = "0.1.2"}
 regex = "1.10.4"

crates/common/src/cache.rs

@@ -11,14 +11,14 @@ pub fn cache(graph: Arc<Mutex<Graph>>, key: &str) -> Result<Cache, Error> {
     graph.reset();
 
     let cache = graph
-        .vertices
+        .volumes
         .iter()
-        .find(|v| v.label == "cache" && v.command == key);
+        .find(|v| v.label == "cache" && v.key == key);
 
     if let Some(cache) = cache {
         return Ok(Cache {
             id: cache.id.clone(),
-            key: cache.command.clone(),
+            key: cache.key.clone(),
             path: "".into(),
         });
     }
@@ -32,6 +32,12 @@ pub fn cache(graph: Arc<Mutex<Graph>>, key: &str) -> Result<Cache, Error> {
         Arc::new(Box::new(CacheExt::default())),
     ));
 
+    graph.execute(GraphCommand::AddVolume(
+        id.clone(),
+        "cache".into(),
+        key.into(),
+    ));
+
     let cache = Cache {
         id,
         key: key.into(),

crates/common/src/common.rs

@@ -90,14 +90,14 @@ pub fn with_cache(graph: Arc<Mutex<Graph>>, cache_id: String, path: String) -> R
     graph.runner = Arc::new(Box::new(CacheExt::default()));
     graph.runner.setup()?;
 
-    if let Some(cache) = graph.vertices.iter().find(|v| v.id.clone() == cache_id) {
+    if let Some(cache) = graph.volumes.iter().find(|v| v.id.clone() == cache_id) {
         let id = Uuid::new_v4().to_string();
         let dep_id = graph.vertices[graph.size() - 1].id.clone();
         let deps = match graph.size() {
             1 => vec![],
             _ => vec![dep_id],
         };
-        let cache_key_path = format!("{}:{}", cache.command, path);
+        let cache_key_path = format!("{}:{}", cache.path, path);
         graph.execute(GraphCommand::AddVertex(
             id.clone(),
             "withCache".into(),
@@ -121,17 +121,17 @@ pub fn with_cache(graph: Arc<Mutex<Graph>>, cache_id: String, path: String) -> R
 pub fn with_file(graph: Arc<Mutex<Graph>>, file_id: String, path: String) -> Result<(), Error> {
     let mut graph = graph.lock().unwrap();
     let runner = graph.runner.clone();
-    graph.runner = Arc::new(Box::new(CacheExt::default()));
+    graph.runner = Arc::new(Box::new(Runner::default()));
     graph.runner.setup()?;
 
-    if let Some(file) = graph.vertices.iter().find(|v| v.id.clone() == file_id) {
+    if let Some(file) = graph.volumes.iter().find(|v| v.id.clone() == file_id) {
         let id = Uuid::new_v4().to_string();
         let dep_id = graph.vertices[graph.size() - 1].id.clone();
         let deps = match graph.size() {
             1 => vec![],
             _ => vec![dep_id],
         };
-        let copy_file = format!("cp {} {}", file.command, path);
+        let copy_file = format!("cp {} {}", file.path, path);
         graph.execute(GraphCommand::AddVertex(
             id.clone(),
             "withFile".into(),
@@ -226,12 +226,10 @@ pub fn zip(graph: Arc<Mutex<Graph>>, path: String) -> Result<File, Error> {
         path: format!("{}/{}", parent_dir, output_file),
     };
 
-    graph.execute(GraphCommand::AddVertex(
+    graph.execute(GraphCommand::AddVolume(
         id,
         "file".into(),
         file.path.clone(),
-        vec![],
-        Arc::new(Box::new(Runner::default())),
     ));
 
     Ok(file)
@@ -273,12 +271,10 @@ pub fn tar_czvf(graph: Arc<Mutex<Graph>>, path: String) -> Result<File, Error> {
         path: format!("{}/{}", parent_dir, output_file),
     };
 
-    graph.execute(GraphCommand::AddVertex(
+    graph.execute(GraphCommand::AddVolume(
         id,
         "file".into(),
         file.path.clone(),
-        vec![],
-        Arc::new(Box::new(Runner::default())),
     ));
 
     Ok(file)

crates/common/src/directory.rs

@@ -43,6 +43,12 @@ pub fn directory(graph: Arc<Mutex<Graph>>, path: String, reset: bool) -> Result<
         Arc::new(Box::new(Runner::default())),
     ));
 
+    graph.execute(GraphCommand::AddVolume(
+        id.clone(),
+        "directory".into(),
+        path.clone(),
+    ));
+
     let path = canonicalize(path).unwrap().to_str().unwrap().to_string();
     let directory = Directory { id, path };
     Ok(directory)

crates/common/src/file.rs

@@ -44,6 +44,12 @@ pub fn file(graph: Arc<Mutex<Graph>>, path: String, reset: bool) -> Result<File,
         Arc::new(Box::new(Runner::default())),
     ));
 
+    graph.execute(GraphCommand::AddVolume(
+        id.clone(),
+        "file".into(),
+        path.clone(),
+    ));
+
     let file = File { id, path };
     Ok(file)
 }
@@ -133,9 +139,16 @@ pub fn tar_xzvf(
     let parent_dir = parent_dir[..parent_dir.len() - 1].join("/");
 
     let dir = Directory {
-        id,
+        id: id.clone(),
         path: format!("{}/{}", parent_dir, output_dir),
     };
+
+    graph.execute(GraphCommand::AddVolume(
+        id,
+        "directory".into(),
+        dir.path.clone(),
+    ));
+
     Ok(dir)
 }
 
@@ -176,10 +189,16 @@ pub fn unzip(
     let parent_dir = parent_dir[..parent_dir.len() - 1].join("/");
 
     let dir = Directory {
-        id,
+        id: id.clone(),
         path: format!("{}/{}", parent_dir, output_dir),
     };
 
+    graph.execute(GraphCommand::AddVolume(
+        id,
+        "directory".into(),
+        dir.path.clone(),
+    ));
+
     Ok(dir)
 }
 

crates/common/src/git.rs

@@ -131,9 +131,15 @@ pub fn tree(graph: Arc<Mutex<Graph>>) -> Result<Directory, Error> {
     graph.execute(GraphCommand::AddEdge(x, y));
     graph.runner = Arc::new(Box::new(RunnerExt::default()));
 
-    let directory = Directory {
-        id,
-        path: graph.work_dir.clone(),
-    };
+    let path = graph.work_dir.clone();
+
+    graph.execute(GraphCommand::AddVolume(
+        id.clone(),
+        "directory".into(),
+        path.clone(),
+    ));
+
+    let directory = Directory { id, path };
+
     Ok(directory)
 }

crates/common/src/http.rs

@@ -5,7 +5,7 @@ use std::{
 
 use anyhow::Error;
 use fluentci_core::deps::{Graph, GraphCommand};
-use fluentci_ext::{http::Http as HttpExt, runner::Runner};
+use fluentci_ext::http::Http as HttpExt;
 use fluentci_types::file::File;
 use uuid::Uuid;
 
@@ -43,12 +43,10 @@ pub fn http(graph: Arc<Mutex<Graph>>, url: String, reset: bool) -> Result<File,
         path: format!("{}/{}", work_dir, filename),
     };
 
-    graph.execute(GraphCommand::AddVertex(
+    graph.execute(GraphCommand::AddVolume(
         id,
         "file".into(),
         file.path.clone(),
-        vec![],
-        Arc::new(Box::new(Runner::default())),
     ));
 
     Ok(file)

crates/core/Cargo.toml

@@ -7,7 +7,7 @@ keywords = ["nix", "environment", "ci", "cd", "devops"]
 license = "MPL-2.0"
 name = "fluentci-core"
 repository = "https://github.com/fluentci-io/fluentci-engine"
-version = "0.1.5"
+version = "0.1.6"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

crates/core/src/deps.rs

@@ -16,7 +16,16 @@ use fluentci_types::Output;
 use super::edge::Edge;
 use super::vertex::{Runnable, Vertex};
 
+#[derive(Default, Debug, Clone)]
+pub struct Volume {
+    pub id: String,
+    pub label: String,
+    pub path: String,
+    pub key: String,
+}
+
 pub enum GraphCommand {
+    AddVolume(String, String, String),
     AddVertex(
         String,
         String,
@@ -32,6 +41,7 @@ pub enum GraphCommand {
 pub struct Graph {
     pub vertices: Vec<Vertex>,
     pub edges: Vec<Edge>,
+    pub volumes: Vec<Volume>,
     tx: Sender<(String, usize)>,
     pub runner: Arc<Box<dyn Extension + Send + Sync>>,
     pub work_dir: String,
@@ -42,6 +52,7 @@ impl Graph {
         let work_dir = current_dir().unwrap().to_str().unwrap().to_string();
         Graph {
             vertices: Vec::new(),
+            volumes: Vec::new(),
             edges: Vec::new(),
             tx,
             runner,
@@ -51,23 +62,15 @@ impl Graph {
 
     pub fn execute(&mut self, command: GraphCommand) {
         match command {
+            GraphCommand::AddVolume(id, label, path) => {
+                self.volumes.push(Volume {
+                    id,
+                    label,
+                    key: path.clone(),
+                    path,
+                });
+            }
             GraphCommand::AddVertex(id, label, command, needs, runner) => {
-                if label == "cache"
-                    && self
-                        .vertices
-                        .iter()
-                        .any(|v| v.command == command && v.label == "cache")
-                {
-                    return;
-                }
-                if label == "file"
-                    && self
-                        .vertices
-                        .iter()
-                        .any(|v| v.command == command && v.label == "file")
-                {
-                    return;
-                }
                 if let Some(vertex) = self.vertices.iter_mut().find(|v| v.id == id) {
                     vertex.needs.extend(needs);
                 } else {
@@ -107,9 +110,9 @@ impl Graph {
             "git-last-commit",
             "tree",
             "http",
+            "cache",
             "file",
             "directory",
-            "cache",
             "chmod",
             "withFile",
         ];
@@ -327,8 +330,7 @@ impl Graph {
     }
 
     pub fn reset(&mut self) {
-        let keep = vec!["cache", "file", "directory"];
-        self.vertices.retain(|v| keep.contains(&v.label.as_str()));
+        self.vertices.clear();
         self.edges.clear();
         self.work_dir = current_dir().unwrap().to_str().unwrap().to_string();
     }