diff --git a/DESCRIPTION b/DESCRIPTION index 68bd35a..a195c8c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,7 +25,7 @@ Suggests: rmarkdown, testthat, visNetwork -URL: https://rpahl.github.io/pipeflow/ +URL: https://rpahl.github.io/pipeflow, https://github.com/rpahl/pipeflow BugReports: https://github.com/rpahl/pipeflow/issues VignetteBuilder: knitr Config/testthat/edition: 3 diff --git a/_pkgdown.yml b/_pkgdown.yml index 5572d10..5f52493 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -2,92 +2,91 @@ url: https://github.com/rpahl/pipeflow template: - params: - bootstrap: 3 - bootswatch: united + bootstrap: 5 + bootswatch: united authors: Roman Pahl: href: https://rpahl.github.io/r-some-blog/about.html -reference: - - title: "Pipeline" - desc: "A pipeline consists of a sequence of analysis steps." - contents: - - Pipeline - - pipelineAliases - - subtitle: "Logging" - contents: - - set_log_layout - -articles: - - title: "Overview" - contents: - - get-started - - modify-pipeline - - combine-pipelines - - collect-output - - split-and-combine - - self-modify-pipeline - - home: links: - text: Visit my R blog href: https://rpahl.github.io/r-some-blog/ +reference: +- subtitle: "Pipeline" + desc: A pipeline is created by instantiating an object of the Pipeline + class. Follow the Pipeline link below for a detailed documenation of + all member functions provided by a Pipeline object. + contents: + - Pipeline +- subtitle: "Alias functions" + desc: There is one alias function for each member function of a + Pipeline object. For the function documentation see the respective + member function under the Pipeline class link above. + contents: + - pipelineAliases +- subtitle: "Logging" + contents: + - set_log_layout + navbar: - title: "pipeflow" - left: - - icon: fa-home fa-lg + structure: + left: [home, reference, articles, myPackages] + right: [search, github, stackoverflow, bluesky, blog] + components: + home: + icon: fa-home fa-lg href: index.html aria-label: home - - text: "Get started" - href: articles/get-started.html - - text: "Reference" - href: reference/index.html - - text: "Articles" + articles: + text: "Articles" menu: - - title: "Overview" - - text: "Overview" - href: articles/index.html - - - text: "---" - - text: "Get started" - - text: "Get started with pipeflow" - href: articles/get-started.html - - text: "Modifying existing pipelines" - href: articles/modify-pipeline.html - - text: "Combining pipelines" - href: articles/combine-pipelines.html - - text: "Collecting pipeline output" - href: articles/collect-output.html + - text: "Overview" + href: articles/index.html + - text: "---" + - text: "Get started" + - text: "Get started with pipeflow" + href: articles/get-started.html + - text: "Modifying existing pipelines" + href: articles/modify-pipeline.html + - text: "Combining pipelines" + href: articles/combine-pipelines.html + - text: "Collecting pipeline output" + href: articles/collect-output.html + - text: ---- + - text: "Advanced" + - text: "How to use pipelines with split data sets" + href: articles/split-and-combine.html + - text: "How pipelines can modify themselves at runtime" + href: articles/self-modify-pipeline.html - - text: "---" - - text: "Advanced" - - text: "How to use pipelines with split data sets" - href: articles/split-and-combine.html - - text: "How pipelines can modify themselves at runtime" - href: articles/self-modify-pipeline.html - - text: "My packages" + myPackages: + text: "My Packages" menu: - - text: "container" - href: https://rpahl.github.io/container/ - - text: "GroupSeq" - href: https://rpahl.github.io/GroupSeq/ - - text: "pipeflow" - href: https://rpahl.github.io/pipeflow/ - right: - - icon: fab fa-github + - text: "container" + href: https://rpahl.github.io/container/ + - text: "GroupSeq" + href: https://rpahl.github.io/GroupSeq/ + - text: "pipeflow" + href: https://rpahl.github.io/pipeflow/ + + + github: + icon: fab fa-github href: https://github.com/rpahl/pipeflow/ aria-label: github - - icon: fab fa-stack-overflow + stackoverflow: + icon: fab fa-stack-overflow href: https://stackoverflow.com/users/8120617/rpahl aria-label: stackoverflow - - icon: fab fa-bluesky + bluesky: + icon: fab fa-bluesky href: https://bsky.app/profile/rpahl.bsky.social aria-label: bluesky - - icon: fas fa-blog + blog: + icon: fas fa-blog href: https://rpahl.github.io/r-some-blog/ aria-label: blog diff --git a/docs/404.html b/docs/404.html index 653b81e..143c861 100644 --- a/docs/404.html +++ b/docs/404.html @@ -4,7 +4,7 @@ - + Page not found (404) • pipeflow @@ -12,180 +12,97 @@ - - - - - - + + + + + - - - - -
-
-
-
-

Page not found (404)

Content not found. Please use links in the navbar. -
- - - +
- -
- diff --git a/docs/articles/collect-output.html b/docs/articles/collect-output.html index a663001..3194e88 100644 --- a/docs/articles/collect-output.html +++ b/docs/articles/collect-output.html @@ -4,7 +4,7 @@ - + Collecting output • pipeflow @@ -12,155 +12,86 @@ - - - - - - + + + + + + - - + + Skip to contents -
-
-
-
-

Collecting output

+
+
+

Collecting output

Source: vignettes/collect-output.Rmd - - +
collect-output.Rmd
@@ -180,7 +111,7 @@

Flag output steps
-library(pipeflow)
+library(pipeflow)
 library(ggplot2)
 
 pip <- pipe_new(
@@ -267,15 +198,15 @@ 
Flag output steps
 pip$run()
-# INFO  [2024-12-01 17:13:42.301] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:13:42.329] Step 1/6 data
-# INFO  [2024-12-01 17:13:42.338] Step 2/6 data_prep
-# INFO  [2024-12-01 17:13:42.361] Step 3/6 data_summary
-# INFO  [2024-12-01 17:13:42.363] Step 4/6 model_fit
-# INFO  [2024-12-01 17:13:42.367] Step 5/6 model_summary
-# INFO  [2024-12-01 17:13:42.369] Step 6/6 model_plot
-# INFO  [2024-12-01 17:13:42.380] Finished execution of steps.
-# INFO  [2024-12-01 17:13:42.381] Done.
+# INFO  [2024-12-01 21:41:31.888] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:31.916] Step 1/6 data
+# INFO  [2024-12-01 21:41:31.925] Step 2/6 data_prep
+# INFO  [2024-12-01 21:41:31.948] Step 3/6 data_summary
+# INFO  [2024-12-01 21:41:31.950] Step 4/6 model_fit
+# INFO  [2024-12-01 21:41:31.954] Step 5/6 model_summary
+# INFO  [2024-12-01 21:41:31.956] Step 6/6 model_plot
+# INFO  [2024-12-01 21:41:31.968] Finished execution of steps.
+# INFO  [2024-12-01 21:41:31.968] Done.
 
 out <- pip$collect_out()
 
@@ -379,15 +310,15 @@ 
Grouping output steps

-
- - - +
-
- diff --git a/docs/articles/combine-pipelines.html b/docs/articles/combine-pipelines.html index e5a4391..8c7b9dc 100644 --- a/docs/articles/combine-pipelines.html +++ b/docs/articles/combine-pipelines.html @@ -4,7 +4,7 @@ - + Combining pipelines • pipeflow @@ -12,156 +12,86 @@ - - - - - - + + + + + + - - + + Skip to contents -
-
+
-
-
-

Combining pipelines

+
+

Combining pipelines

Source: vignettes/combine-pipelines.Rmd - - +
combine-pipelines.Rmd
@@ -182,7 +112,7 @@

Define two pipelines
-library(pipeflow)
+library(pipeflow)
 library(ggplot2)
 
 pip1 <- pipe_new(
@@ -314,7 +244,8 @@ 
Combine pipelines{"x":{"nodes":{"id":[1,2,3,4,5,6],"label":["data","data_prep","standardize","data.modeling","fit","plot"],"group":["data","data_prep","standardize","data.modeling","fit","plot"],"shape":["database","database","box","database","box","box"],"color":["lightblue","lightblue","lightblue","lightblue","orange","orange"],"title":["
<\/p>","
<\/p>","
<\/p>","
<\/p>","
<\/p>","
<\/p>"]},"edges":{"from":[1,2,3,4,4,5],"to":[2,3,4,5,6,6],"arrows":["to","to","to","to","to","to"]},"nodesToDataframe":true,"edgesToDataframe":true,"options":{"width":"100%","height":"100%","nodes":{"shape":"dot"},"manipulation":{"enabled":false},"layout":{"hierarchical":{"enabled":true,"direction":"LR","sortMethod":"directed"}}},"groups":["data","data_prep","standardize","data.modeling","fit","plot"],"width":null,"height":100,"idselection":{"enabled":false},"byselection":{"enabled":false},"main":null,"submain":null,"footer":null,"background":"rgba(0, 0, 0, 0)"},"evals":[],"jsHooks":[]}

 
Relative indexing
 

-
Since the name of the last step might not always be known1, the
+
Since the name of the last step might not always be known1, the
 pipeflow package also provides a relative position indexing
 mechanism, which allows to rewrite the above command as follows:

 
+# <bytecode: 0x000001c01370b3f8>
+# <environment: 0x000001c00f2b05a8>

@@ -370,15 +301,15 @@

Run combined pipelineLet’s now run the combined pipeline and inspect the plot.

 pip$run()
-# INFO  [2024-12-01 17:13:47.390] Start run of 'preprocessing.modeling' pipeline:
-# INFO  [2024-12-01 17:13:47.462] Step 1/6 data
-# INFO  [2024-12-01 17:13:47.470] Step 2/6 data_prep
-# INFO  [2024-12-01 17:13:47.486] Step 3/6 standardize
-# INFO  [2024-12-01 17:13:47.489] Step 4/6 data.modeling
-# INFO  [2024-12-01 17:13:47.491] Step 5/6 fit
-# INFO  [2024-12-01 17:13:47.495] Step 6/6 plot
-# INFO  [2024-12-01 17:13:47.509] Finished execution of steps.
-# INFO  [2024-12-01 17:13:47.509] Done.
+# INFO [2024-12-01 21:41:36.693] Start run of 'preprocessing.modeling' pipeline: +# INFO [2024-12-01 21:41:36.763] Step 1/6 data +# INFO [2024-12-01 21:41:36.772] Step 2/6 data_prep +# INFO [2024-12-01 21:41:36.788] Step 3/6 standardize +# INFO [2024-12-01 21:41:36.791] Step 4/6 data.modeling +# INFO [2024-12-01 21:41:36.793] Step 5/6 fit +# INFO [2024-12-01 21:41:36.797] Step 6/6 plot +# INFO [2024-12-01 21:41:36.810] Finished execution of steps. +# INFO [2024-12-01 21:41:36.811] Done.

 pip$run()
-# INFO  [2024-12-01 17:13:48.062] Start run of 'preprocessing.modeling' pipeline:
-# INFO  [2024-12-01 17:13:48.064] Step 1/6 data - skip 'done' step
-# INFO  [2024-12-01 17:13:48.065] Step 2/6 data_prep - skip 'done' step
-# INFO  [2024-12-01 17:13:48.066] Step 3/6 standardize - skip 'done' step
-# INFO  [2024-12-01 17:13:48.066] Step 4/6 data.modeling - skip 'done' step
-# INFO  [2024-12-01 17:13:48.067] Step 5/6 fit
-# INFO  [2024-12-01 17:13:48.072] Step 6/6 plot
-# INFO  [2024-12-01 17:13:48.091] Finished execution of steps.
-# INFO  [2024-12-01 17:13:48.092] Done.
+# INFO [2024-12-01 21:41:37.360] Start run of 'preprocessing.modeling' pipeline: +# INFO [2024-12-01 21:41:37.361] Step 1/6 data - skip 'done' step +# INFO [2024-12-01 21:41:37.362] Step 2/6 data_prep - skip 'done' step +# INFO [2024-12-01 21:41:37.362] Step 3/6 standardize - skip 'done' step +# INFO [2024-12-01 21:41:37.363] Step 4/6 data.modeling - skip 'done' step +# INFO [2024-12-01 21:41:37.364] Step 5/6 fit +# INFO [2024-12-01 21:41:37.368] Step 6/6 plot +# INFO [2024-12-01 21:41:37.385] Finished execution of steps. +# INFO [2024-12-01 21:41:37.386] Done.
 -
-
-
    -

  1. A typical example would be appending several pipelines -in a programmatic context.↩︎

    2. -
-
-
- - + - - diff --git a/docs/articles/get-started.html b/docs/articles/get-started.html index e05757e..a713267 100644 --- a/docs/articles/get-started.html +++ b/docs/articles/get-started.html @@ -4,7 +4,7 @@ - + Get started with pipeflow • pipeflow @@ -12,156 +12,86 @@ - - - - - - + + + + + + - - + + Skip to contents -
-
+
-
-
-

Get started with pipeflow

+
+

Get started with pipeflow

Source: vignettes/get-started.Rmd - - +
get-started.Rmd
@@ -198,7 +128,7 @@

Define the pipeline
-library(pipeflow)
+library(pipeflow)
 library(ggplot2)
 
 pip <- Pipeline$new("my-pipeline", data = airquality)

@@ -302,13 +232,13 @@

Run the pipeline and inspect th the pipeline, which will produce the following output:

 pip$run()
-# INFO  [2024-12-01 17:13:52.506] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:13:52.532] Step 1/4 data
-# INFO  [2024-12-01 17:13:52.542] Step 2/4 data_prep
-# INFO  [2024-12-01 17:13:52.565] Step 3/4 model_fit
-# INFO  [2024-12-01 17:13:52.569] Step 4/4 model_plot
-# INFO  [2024-12-01 17:13:52.581] Finished execution of steps.
-# INFO  [2024-12-01 17:13:52.582] Done.
+# INFO [2024-12-01 21:41:41.560] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:41.586] Step 1/4 data +# INFO [2024-12-01 21:41:41.597] Step 2/4 data_prep +# INFO [2024-12-01 21:41:41.620] Step 3/4 model_fit +# INFO [2024-12-01 21:41:41.623] Step 4/4 model_plot +# INFO [2024-12-01 21:41:41.635] Finished execution of steps. +# INFO [2024-12-01 21:41:41.636] Done.

Let’s inspect the pipeline again.

The outdated steps were re-run as expected and the output was updated accordingly.

 
 pip$run()$get_out("model_plot")
-# INFO  [2024-12-01 17:13:53.884] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:13:53.885] Step 1/4 data - skip 'done' step
-# INFO  [2024-12-01 17:13:53.886] Step 2/4 data_prep - skip 'done' step
-# INFO  [2024-12-01 17:13:53.887] Step 3/4 model_fit - skip 'done' step
-# INFO  [2024-12-01 17:13:53.888] Step 4/4 model_plot
-# INFO  [2024-12-01 17:13:53.896] Finished execution of steps.
-# INFO  [2024-12-01 17:13:53.896] Done.
+# INFO [2024-12-01 21:41:42.919] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:42.920] Step 1/4 data - skip 'done' step +# INFO [2024-12-01 21:41:42.921] Step 2/4 data_prep - skip 'done' step +# INFO [2024-12-01 21:41:42.922] Step 3/4 model_fit - skip 'done' step +# INFO [2024-12-01 21:41:42.923] Step 4/4 model_plot +# INFO [2024-12-01 21:41:42.931] Finished execution of steps. +# INFO [2024-12-01 21:41:42.931] Done.

model-plot

If instead we were to change the input data, since all steps depend on it, we expect all steps to be rerun.

@@ -439,45 +369,38 @@

Change parameters of the pipeline# 4: model_plot model_fit,data_prep <gg[11]> FALSE model_plot Outdated

 pip$run()$get_out("model_plot")
-# INFO  [2024-12-01 17:13:54.169] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:13:54.170] Step 1/4 data
-# INFO  [2024-12-01 17:13:54.172] Step 2/4 data_prep
-# INFO  [2024-12-01 17:13:54.177] Step 3/4 model_fit
-# INFO  [2024-12-01 17:13:54.180] Step 4/4 model_plot
-# INFO  [2024-12-01 17:13:54.185] Finished execution of steps.
-# INFO  [2024-12-01 17:13:54.186] Done.
+# INFO [2024-12-01 21:41:43.202] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:43.203] Step 1/4 data +# INFO [2024-12-01 21:41:43.206] Step 2/4 data_prep +# INFO [2024-12-01 21:41:43.211] Step 3/4 model_fit +# INFO [2024-12-01 21:41:43.214] Step 4/4 model_plot +# INFO [2024-12-01 21:41:43.219] Finished execution of steps. +# INFO [2024-12-01 21:41:43.220] Done.

model-plot

Next, let’s see how to modify the pipeline.

- - - - + - - diff --git a/docs/articles/index.html b/docs/articles/index.html index 1e90188..cc812aa 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -1,157 +1,87 @@ -Articles • pipeflow - - -
-
-
-
-

Articles

+
    • +
    • +
    • +
    • +
    • +
+ + +
+
+
+
+

Articles

-

Overview

-

+

All vignettes

+
-
Get started with pipeflow
+
Collecting output
+

Shows how to collect and group pipeline output.

+
Combining pipelines
+

Shows how to combine different pipelines to a single pipeline.

+
Get started with pipeflow

Start here if this is your first time using pipeflow.

Modifying existing pipelines

Shows how to insert, replace, and remove steps in a pipeline.

-
Combining pipelines
-

Shows how to combine different pipelines to a single pipeline.

-
Collecting output
-

Shows how to collect and group pipeline output.

-
Advanced: How to use pipeflow with split data sets
-

Shows how you easily apply pipelines at multiple data sets and then combine the output at the end of the pipeline.

Advanced: How pipelines can modify themselves at runtime

Shows how you can setup pipelines to modify themselves at runtime, which, for example, allows for changing pipeline parameters based on intermediate results or even dynamically modify the pipeline’s own structure during a pipeline run.

+
Advanced: How to use pipeflow with split data sets
+

Shows how you easily apply pipelines at multiple data sets and then combine the output at the end of the pipeline.

-
-
+
-
diff --git a/docs/articles/modify-pipeline.html b/docs/articles/modify-pipeline.html index 1705b2b..50a4244 100644 --- a/docs/articles/modify-pipeline.html +++ b/docs/articles/modify-pipeline.html @@ -4,7 +4,7 @@ - + Modifying existing pipelines • pipeflow @@ -12,155 +12,86 @@ - - - - - - + + + + + + - - + + Skip to contents -
-
-
-
-

Modifying existing pipelines

+
+
+

Modifying existing pipelines

Source: vignettes/modify-pipeline.Rmd - - +
modify-pipeline.Rmd
@@ -282,14 +213,14 @@

Replace existing steps
 pip$set_params(list(xVar = "Solar.R", yVar = "Wind"))
 pip$run()
-# INFO  [2024-12-01 17:13:58.886] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:13:58.887] Step 1/5 data - skip 'done' step
-# INFO  [2024-12-01 17:13:58.888] Step 2/5 data_prep - skip 'done' step
-# INFO  [2024-12-01 17:13:58.888] Step 3/5 standardize
-# INFO  [2024-12-01 17:13:58.891] Step 4/5 model_fit
-# INFO  [2024-12-01 17:13:58.894] Step 5/5 model_plot
-# INFO  [2024-12-01 17:13:58.899] Finished execution of steps.
-# INFO  [2024-12-01 17:13:58.900] Done.

+# INFO [2024-12-01 21:41:47.485] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:47.486] Step 1/5 data - skip 'done' step +# INFO [2024-12-01 21:41:47.487] Step 2/5 data_prep - skip 'done' step +# INFO [2024-12-01 21:41:47.488] Step 3/5 standardize +# INFO [2024-12-01 21:41:47.490] Step 4/5 model_fit +# INFO [2024-12-01 21:41:47.493] Step 5/5 model_plot +# INFO [2024-12-01 21:41:47.498] Finished execution of steps. +# INFO [2024-12-01 21:41:47.498] Done.
 pip$get_out("model_fit") |> coefficients()
 #  (Intercept)      Solar.R 
@@ -348,33 +279,26 @@ 
Removing stepsCombining
 pipelines vignette.
-
- - - +
-
- diff --git a/docs/articles/self-modify-pipeline.html b/docs/articles/self-modify-pipeline.html index 94b3055..03d5e2d 100644 --- a/docs/articles/self-modify-pipeline.html +++ b/docs/articles/self-modify-pipeline.html @@ -4,7 +4,7 @@ - + Advanced: How pipelines can modify themselves at runtime • pipeflow @@ -12,156 +12,86 @@ - - - - - - + + + + + + - - + + Skip to contents -
-
-
-
-

Advanced: How pipelines can modify themselves at -runtime

+
+
+

Advanced: How pipelines can modify themselves at runtime

Source: vignettes/self-modify-pipeline.Rmd - - +
self-modify-pipeline.Rmd
@@ -178,7 +108,7 @@

Changing pipeline parameters at residuals for normality using the Shapiro-Wilk test, and plots the residuals.

-library(pipeflow)
+library(pipeflow)
 library(ggplot2)
 
 pip <- pipe_new("my-pipeline") |>
@@ -232,13 +162,13 @@ 
Changing pipeline parameters at
 pip$set_data(airquality)
 pip$set_params(list(xVar = "Ozone", yVar = "Temp"))
 pip$run()$collect_out()
-# INFO  [2024-12-01 17:14:02.986] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:03.016] Step 1/4 data
-# INFO  [2024-12-01 17:14:03.030] Step 2/4 fit
-# INFO  [2024-12-01 17:14:03.048] Step 3/4 residual_shapiro_p_value
-# INFO  [2024-12-01 17:14:03.050] Step 4/4 plot
-# INFO  [2024-12-01 17:14:03.065] Finished execution of steps.
-# INFO  [2024-12-01 17:14:03.066] Done.
+# INFO  [2024-12-01 21:41:51.403] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:51.430] Step 1/4 data
+# INFO  [2024-12-01 21:41:51.444] Step 2/4 fit
+# INFO  [2024-12-01 21:41:51.462] Step 3/4 residual_shapiro_p_value
+# INFO  [2024-12-01 21:41:51.464] Step 4/4 plot
+# INFO  [2024-12-01 21:41:51.481] Finished execution of steps.
+# INFO  [2024-12-01 21:41:51.482] Done.
 # $residual_shapiro_p_value
 # [1] 0.00022598
 # 
@@ -257,13 +187,13 @@ 
Changing pipeline parameters at
     pip$set_params(list(pointColor = "red"))
     pip$run()$collect_out()
 }
-# INFO  [2024-12-01 17:14:03.468] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:03.469] Step 1/4 data - skip 'done' step
-# INFO  [2024-12-01 17:14:03.470] Step 2/4 fit - skip 'done' step
-# INFO  [2024-12-01 17:14:03.471] Step 3/4 residual_shapiro_p_value - skip 'done' step
-# INFO  [2024-12-01 17:14:03.472] Step 4/4 plot
-# INFO  [2024-12-01 17:14:03.486] Finished execution of steps.
-# INFO  [2024-12-01 17:14:03.487] Done.
+# INFO  [2024-12-01 21:41:51.876] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:51.877] Step 1/4 data - skip 'done' step
+# INFO  [2024-12-01 21:41:51.878] Step 2/4 fit - skip 'done' step
+# INFO  [2024-12-01 21:41:51.879] Step 3/4 residual_shapiro_p_value - skip 'done' step
+# INFO  [2024-12-01 21:41:51.880] Step 4/4 plot
+# INFO  [2024-12-01 21:41:51.893] Finished execution of steps.
+# INFO  [2024-12-01 21:41:51.894] Done.
 # $residual_shapiro_p_value
 # [1] 0.00022598
 # 
@@ -301,13 +231,13 @@ 
Changing pipeline parameters at
 pip$set_data(airquality)
 pip$set_params(list(xVar = "Ozone", yVar = "Temp", .self = pip))
 pip$run()$collect_out()
-# INFO  [2024-12-01 17:14:03.788] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:03.789] Step 1/4 data
-# INFO  [2024-12-01 17:14:03.791] Step 2/4 fit
-# INFO  [2024-12-01 17:14:03.796] Step 3/4 residual_shapiro_p_value
-# INFO  [2024-12-01 17:14:03.800] Step 4/4 plot
-# INFO  [2024-12-01 17:14:03.807] Finished execution of steps.
-# INFO  [2024-12-01 17:14:03.808] Done.
+# INFO  [2024-12-01 21:41:52.189] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:52.190] Step 1/4 data
+# INFO  [2024-12-01 21:41:52.192] Step 2/4 fit
+# INFO  [2024-12-01 21:41:52.196] Step 3/4 residual_shapiro_p_value
+# INFO  [2024-12-01 21:41:52.200] Step 4/4 plot
+# INFO  [2024-12-01 21:41:52.207] Finished execution of steps.
+# INFO  [2024-12-01 21:41:52.207] Done.
 # $residual_shapiro_p_value
 # [1] 0.00022598
 # 
@@ -351,13 +281,13 @@ 
Changing the pipeline struct
 respectively.

 
 pip$set_data(1)$run()
-# INFO  [2024-12-01 17:14:04.064] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:04.065] Step 1/4 data
-# INFO  [2024-12-01 17:14:04.067] Step 2/4 f1
-# INFO  [2024-12-01 17:14:04.070] Step 3/4 f2
-# INFO  [2024-12-01 17:14:04.072] Step 4/4 f3
-# INFO  [2024-12-01 17:14:04.073] Finished execution of steps.
-# INFO  [2024-12-01 17:14:04.074] Done.
+# INFO  [2024-12-01 21:41:52.449] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:52.450] Step 1/4 data
+# INFO  [2024-12-01 21:41:52.452] Step 2/4 f1
+# INFO  [2024-12-01 21:41:52.454] Step 3/4 f2
+# INFO  [2024-12-01 21:41:52.456] Step 4/4 f3
+# INFO  [2024-12-01 21:41:52.457] Finished execution of steps.
+# INFO  [2024-12-01 21:41:52.458] Done.
 pip
 #      step depends    out keepOut  group  state
 #    <char>  <list> <list>  <lgcl> <char> <char>
@@ -410,13 +340,13 @@ 
Modify a step
 pip$set_params(list(.self = pip))
 pip$set_data(1)$run()
-# INFO  [2024-12-01 17:14:04.194] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:04.195] Step 1/4 data
-# INFO  [2024-12-01 17:14:04.197] Step 2/4 f1
-# INFO  [2024-12-01 17:14:04.199] Step 3/4 f2
-# INFO  [2024-12-01 17:14:04.201] Step 4/4 f3
-# INFO  [2024-12-01 17:14:04.203] Finished execution of steps.
-# INFO  [2024-12-01 17:14:04.203] Done.
+# INFO  [2024-12-01 21:41:52.575] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:52.576] Step 1/4 data
+# INFO  [2024-12-01 21:41:52.578] Step 2/4 f1
+# INFO  [2024-12-01 21:41:52.580] Step 3/4 f2
+# INFO  [2024-12-01 21:41:52.582] Step 4/4 f3
+# INFO  [2024-12-01 21:41:52.584] Finished execution of steps.
+# INFO  [2024-12-01 21:41:52.584] Done.
 pip
 #      step depends    out keepOut  group  state
 #    <char>  <list> <list>  <lgcl> <char> <char>
@@ -427,13 +357,13 @@ 
Modify a step
 pip$set_data(10)$run()
-# INFO  [2024-12-01 17:14:04.262] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:04.264] Step 1/4 data
-# INFO  [2024-12-01 17:14:04.266] Step 2/4 f1
-# INFO  [2024-12-01 17:14:04.269] Step 3/4 f2
-# INFO  [2024-12-01 17:14:04.275] Step 4/4 f3
-# INFO  [2024-12-01 17:14:04.276] Finished execution of steps.
-# INFO  [2024-12-01 17:14:04.276] Done.
+# INFO  [2024-12-01 21:41:52.643] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:52.644] Step 1/4 data
+# INFO  [2024-12-01 21:41:52.646] Step 2/4 f1
+# INFO  [2024-12-01 21:41:52.649] Step 3/4 f2
+# INFO  [2024-12-01 21:41:52.654] Step 4/4 f3
+# INFO  [2024-12-01 21:41:52.656] Finished execution of steps.
+# INFO  [2024-12-01 21:41:52.656] Done.
 pip
 #      step depends    out keepOut  group  state
 #    <char>  <list> <list>  <lgcl> <char> <char>
@@ -522,19 +452,19 @@ 
Insert and remove steps
 pip$set_params(list(.self = pip))
 pip$set_data(10)$run()
-# INFO  [2024-12-01 17:14:04.453] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:04.454] Step 1/4 data
-# INFO  [2024-12-01 17:14:04.457] Step 2/4 f1
-# INFO  [2024-12-01 17:14:04.460] Step 3/4 f2
-# INFO  [2024-12-01 17:14:04.475] Abort pipeline execution and restart on new.
-# INFO  [2024-12-01 17:14:04.475] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:04.476] Step 1/5 data - skip 'done' step
-# INFO  [2024-12-01 17:14:04.477] Step 2/5 f1 - skip 'done' step
-# INFO  [2024-12-01 17:14:04.478] Step 3/5 f2a
-# INFO  [2024-12-01 17:14:04.484] Step 4/5 f2b
-# INFO  [2024-12-01 17:14:04.487] Step 5/5 f3
-# INFO  [2024-12-01 17:14:04.489] Finished execution of steps.
-# INFO  [2024-12-01 17:14:04.489] Done.

+# INFO  [2024-12-01 21:41:52.829] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:52.830] Step 1/4 data
+# INFO  [2024-12-01 21:41:52.832] Step 2/4 f1
+# INFO  [2024-12-01 21:41:52.834] Step 3/4 f2
+# INFO  [2024-12-01 21:41:52.848] Abort pipeline execution and restart on new.
+# INFO  [2024-12-01 21:41:52.848] Start run of 'my-pipeline' pipeline:
+# INFO  [2024-12-01 21:41:52.849] Step 1/5 data - skip 'done' step
+# INFO  [2024-12-01 21:41:52.850] Step 2/5 f1 - skip 'done' step
+# INFO  [2024-12-01 21:41:52.851] Step 3/5 f2a
+# INFO  [2024-12-01 21:41:52.856] Step 4/5 f2b
+# INFO  [2024-12-01 21:41:52.858] Step 5/5 f3
+# INFO  [2024-12-01 21:41:52.859] Finished execution of steps.
+# INFO  [2024-12-01 21:41:52.860] Done.

The log output shows the abort and re-run of the pipeline. Let’s see the final structure and step outputs.

@@ -552,33 +482,26 @@ 
Insert and remove steps

-
- - - +
-
- diff --git a/docs/articles/split-and-combine.html b/docs/articles/split-and-combine.html index 60487c6..855d968 100644 --- a/docs/articles/split-and-combine.html +++ b/docs/articles/split-and-combine.html @@ -4,7 +4,7 @@ - + Advanced: How to use pipeflow with split data sets • pipeflow @@ -12,158 +12,87 @@ - - - - - - + + + + + + - - + + Skip to contents -
-
+
-
-
-

Advanced: How to use pipeflow with split data -sets

+
+

Advanced: How to use pipeflow with split data sets

Source: vignettes/split-and-combine.Rmd - - +
split-and-combine.Rmd
@@ -184,7 +113,7 @@

Define pipelineLet’s first define our pipeline, which, to keep matters simple, just fits a linear model and outputs the model coefficients.

-library(pipeflow)
+library(pipeflow)
 
 pip <- pipe_new(
         "my-pipeline"
@@ -239,12 +168,12 @@ 
Define pipelinepip$set_data(iris)
 pip$set_params(list(xVar = "Sepal.Length", yVar = "Sepal.Width"))
 pip$run()
-# INFO  [2024-12-01 17:14:08.139] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:08.164] Step 1/3 data
-# INFO  [2024-12-01 17:14:08.175] Step 2/3 fit
-# INFO  [2024-12-01 17:14:08.194] Step 3/3 coefs
-# INFO  [2024-12-01 17:14:08.195] Finished execution of steps.
-# INFO  [2024-12-01 17:14:08.196] Done.
+# INFO [2024-12-01 21:41:55.998] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:56.024] Step 1/3 data +# INFO [2024-12-01 21:41:56.035] Step 2/3 fit +# INFO [2024-12-01 21:41:56.055] Step 3/3 coefs +# INFO [2024-12-01 21:41:56.056] Finished execution of steps. +# INFO [2024-12-01 21:41:56.056] Done.

 pip$collect_out()
 # $coefs
@@ -266,24 +195,24 @@ 
Split data set}
 
 results <- lapply(split(iris, iris$Species), FUN = run_pipeline)
-# INFO  [2024-12-01 17:14:08.332] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:08.333] Step 1/3 data
-# INFO  [2024-12-01 17:14:08.335] Step 2/3 fit
-# INFO  [2024-12-01 17:14:08.339] Step 3/3 coefs
-# INFO  [2024-12-01 17:14:08.343] Finished execution of steps.
-# INFO  [2024-12-01 17:14:08.344] Done.
-# INFO  [2024-12-01 17:14:08.360] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:08.361] Step 1/3 data
-# INFO  [2024-12-01 17:14:08.363] Step 2/3 fit
-# INFO  [2024-12-01 17:14:08.366] Step 3/3 coefs
-# INFO  [2024-12-01 17:14:08.367] Finished execution of steps.
-# INFO  [2024-12-01 17:14:08.367] Done.
-# INFO  [2024-12-01 17:14:08.370] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:08.371] Step 1/3 data
-# INFO  [2024-12-01 17:14:08.373] Step 2/3 fit
-# INFO  [2024-12-01 17:14:08.375] Step 3/3 coefs
-# INFO  [2024-12-01 17:14:08.376] Finished execution of steps.
-# INFO  [2024-12-01 17:14:08.377] Done.
+# INFO [2024-12-01 21:41:56.192] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:56.193] Step 1/3 data +# INFO [2024-12-01 21:41:56.195] Step 2/3 fit +# INFO [2024-12-01 21:41:56.199] Step 3/3 coefs +# INFO [2024-12-01 21:41:56.203] Finished execution of steps. +# INFO [2024-12-01 21:41:56.204] Done. +# INFO [2024-12-01 21:41:56.222] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:56.223] Step 1/3 data +# INFO [2024-12-01 21:41:56.225] Step 2/3 fit +# INFO [2024-12-01 21:41:56.227] Step 3/3 coefs +# INFO [2024-12-01 21:41:56.228] Finished execution of steps. +# INFO [2024-12-01 21:41:56.229] Done. +# INFO [2024-12-01 21:41:56.232] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:56.233] Step 1/3 data +# INFO [2024-12-01 21:41:56.235] Step 2/3 fit +# INFO [2024-12-01 21:41:56.238] Step 3/3 coefs +# INFO [2024-12-01 21:41:56.239] Finished execution of steps. +# INFO [2024-12-01 21:41:56.239] Done.
 pip$collect_out()
 # $setosa
@@ -486,19 +415,19 @@ 
Combine output
 pip$set_params(list(xVar = "Sepal.Length", yVar = "Sepal.Width"))
 pip$run()
-# INFO  [2024-12-01 17:14:09.360] Start run of 'my-pipeline' pipeline:
-# INFO  [2024-12-01 17:14:09.361] Step 1/10 data.setosa
-# INFO  [2024-12-01 17:14:09.364] Step 2/10 fit.setosa
-# INFO  [2024-12-01 17:14:09.367] Step 3/10 coefs.setosa
-# INFO  [2024-12-01 17:14:09.368] Step 4/10 data.versicolor
-# INFO  [2024-12-01 17:14:09.371] Step 5/10 fit.versicolor
-# INFO  [2024-12-01 17:14:09.374] Step 6/10 coefs.versicolor
-# INFO  [2024-12-01 17:14:09.376] Step 7/10 data.virginica
-# INFO  [2024-12-01 17:14:09.378] Step 8/10 fit.virginica
-# INFO  [2024-12-01 17:14:09.381] Step 9/10 coefs.virginica
-# INFO  [2024-12-01 17:14:09.383] Step 10/10 combine_coefs
-# INFO  [2024-12-01 17:14:09.386] Finished execution of steps.
-# INFO  [2024-12-01 17:14:09.386] Done.
+# INFO [2024-12-01 21:41:57.260] Start run of 'my-pipeline' pipeline: +# INFO [2024-12-01 21:41:57.263] Step 1/10 data.setosa +# INFO [2024-12-01 21:41:57.265] Step 2/10 fit.setosa +# INFO [2024-12-01 21:41:57.268] Step 3/10 coefs.setosa +# INFO [2024-12-01 21:41:57.270] Step 4/10 data.versicolor +# INFO [2024-12-01 21:41:57.272] Step 5/10 fit.versicolor +# INFO [2024-12-01 21:41:57.275] Step 6/10 coefs.versicolor +# INFO [2024-12-01 21:41:57.277] Step 7/10 data.virginica +# INFO [2024-12-01 21:41:57.279] Step 8/10 fit.virginica +# INFO [2024-12-01 21:41:57.282] Step 9/10 coefs.virginica +# INFO [2024-12-01 21:41:57.284] Step 10/10 combine_coefs +# INFO [2024-12-01 21:41:57.286] Finished execution of steps. +# INFO [2024-12-01 21:41:57.286] Done.
 pip$collect_out()
 # $combine_coefs
@@ -507,33 +436,26 @@ 
Combine output# coefs.versicolor   0.8721460    0.3197193
 # coefs.virginica    1.4463054    0.2318905
-
- - - + - - diff --git a/docs/authors.html b/docs/authors.html index e64ee8d..530bbb1 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -1,169 +1,97 @@ -Authors and Citation • pipeflow - - -
-
+ + + -
-
-
-
-

Authors and Citation

-
+
+
+
+
+

Authors and Citation

+
+
+

Authors

-
-
-

Citation

- Source: DESCRIPTION -
-
+
+

Citation

+

Source: DESCRIPTION

-

Pahl R (2024). +

Pahl R (2024). pipeflow: Implement Data Analysis Workflows with Pipelines. -R package version 0.1.1, https://rpahl.github.io/pipeflow/. +R package version 0.1.1, https://github.com/rpahl/pipeflow, https://rpahl.github.io/pipeflow.

- 
@Manual{,
+      
@Manual{,
   title = {pipeflow: Implement Data Analysis Workflows with Pipelines},
   author = {Roman Pahl},
   year = {2024},
-  note = {R package version 0.1.1},
-  url = {https://rpahl.github.io/pipeflow/},
+  note = {R package version 0.1.1, https://github.com/rpahl/pipeflow},
+  url = {https://rpahl.github.io/pipeflow},
 }

+
-
- -
- +
-
diff --git a/docs/index.html b/docs/index.html index 7279a3e..c8f6893 100644 --- a/docs/index.html +++ b/docs/index.html @@ -4,7 +4,7 @@ - + Implement Data Analysis Workflows with Pipelines • pipeflow @@ -12,152 +12,80 @@ - - - - - - + + + + + + - - - - -
-
-
+
+
+
+
-

pipeflow logo -

+
+

pipeflow +

+

pipeflow is an intuitive yet powerful framework for building and running data processing pipelines. Whether you’re working in scientific computing, machine learning (AI), or statistical reporting, pipeflow adapts seamlessly to any scenario—from small exploratory analyses to complex production workflows.

Since pipeflow pipelines are essentially sequences of R functions, they’re easy to learn and can be used interactively by typical R users. Advanced features like dynamic branching and self-modifying pipelines provide additional flexibility for more complex workflows.

@@ -216,7 +144,7 @@

InstallationUsage

+library(pipeflow)

Getting Started @@ -239,10 +167,7 @@

Advanced topics

-
- - -
- diff --git a/docs/pkgdown.js b/docs/pkgdown.js index 6f0eee4..1a99c65 100644 --- a/docs/pkgdown.js +++ b/docs/pkgdown.js @@ -2,83 +2,43 @@ (function($) { $(function() { - $('.navbar-fixed-top').headroom(); + $('nav.navbar').headroom(); - $('body').css('padding-top', $('.navbar').height() + 10); - $(window).resize(function(){ - $('body').css('padding-top', $('.navbar').height() + 10); + Toc.init({ + $nav: $("#toc"), + $scope: $("main h2, main h3, main h4, main h5, main h6") }); - $('[data-toggle="tooltip"]').tooltip(); - - var cur_path = paths(location.pathname); - var links = $("#navbar ul li a"); - var max_length = -1; - var pos = -1; - for (var i = 0; i < links.length; i++) { - if (links[i].getAttribute("href") === "#") - continue; - // Ignore external links - if (links[i].host !== location.host) - continue; - - var nav_path = paths(links[i].pathname); - - var length = prefix_length(nav_path, cur_path); - if (length > max_length) { - max_length = length; - pos = i; - } - } - - // Add class to parent
  • , and enclosing
  • if in dropdown - if (pos >= 0) { - var menu_anchor = $(links[pos]); - menu_anchor.parent().addClass("active"); - menu_anchor.closest("li.dropdown").addClass("active"); - } - }); - - function paths(pathname) { - var pieces = pathname.split("/"); - pieces.shift(); // always starts with / - - var end = pieces[pieces.length - 1]; - if (end === "index.html" || end === "") - pieces.pop(); - return(pieces); - } - - // Returns -1 if not found - function prefix_length(needle, haystack) { - if (needle.length > haystack.length) - return(-1); - - // Special case for length-0 haystack, since for loop won't run - if (haystack.length === 0) { - return(needle.length === 0 ? 0 : -1); + if ($('#toc').length) { + $('body').scrollspy({ + target: '#toc', + offset: $("nav.navbar").outerHeight() + 1 + }); } - for (var i = 0; i < haystack.length; i++) { - if (needle[i] != haystack[i]) - return(i); - } + // Activate popovers + $('[data-bs-toggle="popover"]').popover({ + container: 'body', + html: true, + trigger: 'focus', + placement: "top", + sanitize: false, + }); - return(haystack.length); - } + $('[data-bs-toggle="tooltip"]').tooltip(); /* Clipboard --------------------------*/ function changeTooltipMessage(element, msg) { - var tooltipOriginalTitle=element.getAttribute('data-original-title'); - element.setAttribute('data-original-title', msg); + var tooltipOriginalTitle=element.getAttribute('data-bs-original-title'); + element.setAttribute('data-bs-original-title', msg); $(element).tooltip('show'); - element.setAttribute('data-original-title', tooltipOriginalTitle); + element.setAttribute('data-bs-original-title', tooltipOriginalTitle); } if(ClipboardJS.isSupported()) { $(document).ready(function() { - var copyButton = ""; + var copyButton = ""; $("div.sourceCode").addClass("hasCopyButton"); @@ -89,20 +49,114 @@ $('.btn-copy-ex').tooltip({container: 'body'}); // Initialize clipboard: - var clipboardBtnCopies = new ClipboardJS('[data-clipboard-copy]', { + var clipboard = new ClipboardJS('[data-clipboard-copy]', { text: function(trigger) { return trigger.parentNode.textContent.replace(/\n#>[^\n]*/g, ""); } }); - clipboardBtnCopies.on('success', function(e) { + clipboard.on('success', function(e) { changeTooltipMessage(e.trigger, 'Copied!'); e.clearSelection(); }); - clipboardBtnCopies.on('error', function() { + clipboard.on('error', function(e) { changeTooltipMessage(e.trigger,'Press Ctrl+C or Command+C to copy'); }); + }); } + + /* Search marking --------------------------*/ + var url = new URL(window.location.href); + var toMark = url.searchParams.get("q"); + var mark = new Mark("main#main"); + if (toMark) { + mark.mark(toMark, { + accuracy: { + value: "complementary", + limiters: [",", ".", ":", "/"], + } + }); + } + + /* Search --------------------------*/ + /* Adapted from https://github.com/rstudio/bookdown/blob/2d692ba4b61f1e466c92e78fd712b0ab08c11d31/inst/resources/bs4_book/bs4_book.js#L25 */ + // Initialise search index on focus + var fuse; + $("#search-input").focus(async function(e) { + if (fuse) { + return; + } + + $(e.target).addClass("loading"); + var response = await fetch($("#search-input").data("search-index")); + var data = await response.json(); + + var options = { + keys: ["what", "text", "code"], + ignoreLocation: true, + threshold: 0.1, + includeMatches: true, + includeScore: true, + }; + fuse = new Fuse(data, options); + + $(e.target).removeClass("loading"); + }); + + // Use algolia autocomplete + var options = { + autoselect: true, + debug: true, + hint: false, + minLength: 2, + }; + var q; +async function searchFuse(query, callback) { + await fuse; + + var items; + if (!fuse) { + items = []; + } else { + q = query; + var results = fuse.search(query, { limit: 20 }); + items = results + .filter((x) => x.score <= 0.75) + .map((x) => x.item); + if (items.length === 0) { + items = [{dir:"Sorry 😿",previous_headings:"",title:"No results found.",what:"No results found.",path:window.location.href}]; + } + } + callback(items); +} + $("#search-input").autocomplete(options, [ + { + name: "content", + source: searchFuse, + templates: { + suggestion: (s) => { + if (s.title == s.what) { + return `${s.dir} >
    ${s.title}
    `; + } else if (s.previous_headings == "") { + return `${s.dir} >
    ${s.title}
    > ${s.what}`; + } else { + return `${s.dir} >
    ${s.title}
    > ${s.previous_headings} > ${s.what}`; + } + }, + }, + }, + ]).on('autocomplete:selected', function(event, s) { + window.location.href = s.path + "?q=" + q + "#" + s.id; + }); + }); })(window.jQuery || window.$) + +document.addEventListener('keydown', function(event) { + // Check if the pressed key is '/' + if (event.key === '/') { + event.preventDefault(); // Prevent any default action associated with the '/' key + document.getElementById('search-input').focus(); // Set focus to the search input + } +}); diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 8471c1a..2ad7bfb 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -8,7 +8,7 @@ articles: modify-pipeline: modify-pipeline.html self-modify-pipeline: self-modify-pipeline.html split-and-combine: split-and-combine.html -last_built: 2024-12-01T16:13Z +last_built: 2024-12-01T20:41Z urls: reference: https://github.com/rpahl/pipeflow/reference article: https://github.com/rpahl/pipeflow/articles diff --git a/docs/reference/Pipeline.html b/docs/reference/Pipeline.html index b7a7331..0d6a960 100644 --- a/docs/reference/Pipeline.html +++ b/docs/reference/Pipeline.html @@ -1,5 +1,5 @@ -Pipeline Class — Pipeline • pipeflowPipeline Class — Pipeline • pipeflow - - -
    -
    -
    -
    -

    Pipeline Class

    - Source: R/pipeflow_pipeline.R - +
      • +
      • +
      • +
      • +
      • +
    + + +
    +
    +
    +
    +

    Pipeline Class

    + Source: R/pipeflow_pipeline.R +
    Pipeline.Rd
    -
    +

    This class implements an analysis pipeline. A pipeline consists of a sequence of analysis steps, which can be added one by one. Each added step may or may not depend on one or more previous steps. The pipeline @@ -144,12 +84,12 @@

    Pipeline Class

    -
    -

    Author

    +
    +

    Author

    Roman Pahl

    -
    -

    Public fields

    +
    +

    Public fields

    name

    string name of the pipeline

    @@ -160,8 +100,8 @@

    Public fields

    -
    -

    Methods

    +
    +

    Methods

    Public methods

    @@ -1821,8 +1761,8 @@

    Arguments -

    Examples

    +
    +

    Examples

    
 ## ------------------------------------------------
 ## Method `Pipeline$new`
@@ -1862,11 +1802,11 @@ 
    Examples
    
 p <- Pipeline$new("myPipe", data = c(1, 2, NA, 3, 4))
 p$add("calc_mean", mean, params = list(x = ~data, na.rm = TRUE))
 p$run()$get_out("calc_mean")
-#> INFO  [2024-12-01 17:13:31.766] Start run of 'myPipe' pipeline:
-#> INFO  [2024-12-01 17:13:31.791] Step 1/2 data
-#> INFO  [2024-12-01 17:13:31.799] Step 2/2 calc_mean
-#> INFO  [2024-12-01 17:13:31.818] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:31.819] Done.
+#> INFO  [2024-12-01 21:41:21.908] Start run of 'myPipe' pipeline:
+#> INFO  [2024-12-01 21:41:21.933] Step 1/2 data
+#> INFO  [2024-12-01 21:41:21.942] Step 2/2 calc_mean
+#> INFO  [2024-12-01 21:41:21.961] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:21.962] Done.
 #> [1] 2.5
 
 p <- Pipeline$new("myPipe", data = 1:10)
@@ -1883,21 +1823,21 @@ 
    Examples
    
 #> 2:     s1 <function[1]> function <list[1]>    data [NULL]   FALSE     s1
 #>      description                time  state
 #>           <char>              <POSc> <char>
-#> 1:               2024-12-01 17:13:31    New
-#> 2: multiply by 2 2024-12-01 17:13:31    New
+#> 1:               2024-12-01 21:41:21    New
+#> 2: multiply by 2 2024-12-01 21:41:21    New
 
 p <- Pipeline$new("myPipe", data = data.frame(x = 1:5, y = 1:5))
 p$add("prep_x", \(data = ~data) data$x, group = "prep")
 p$add("prep_y", \(data = ~data) (data$y)^2, group = "prep")
 p$add("sum", \(x = ~prep_x, y = ~prep_y) x + y)
 p$run()$collect_out(all = TRUE)
-#> INFO  [2024-12-01 17:13:31.832] Start run of 'myPipe' pipeline:
-#> INFO  [2024-12-01 17:13:31.833] Step 1/4 data
-#> INFO  [2024-12-01 17:13:31.836] Step 2/4 prep_x
-#> INFO  [2024-12-01 17:13:31.838] Step 3/4 prep_y
-#> INFO  [2024-12-01 17:13:31.840] Step 4/4 sum
-#> INFO  [2024-12-01 17:13:31.841] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:31.841] Done.
+#> INFO  [2024-12-01 21:41:21.975] Start run of 'myPipe' pipeline:
+#> INFO  [2024-12-01 21:41:21.976] Step 1/4 data
+#> INFO  [2024-12-01 21:41:21.979] Step 2/4 prep_x
+#> INFO  [2024-12-01 21:41:21.981] Step 3/4 prep_y
+#> INFO  [2024-12-01 21:41:21.983] Step 4/4 sum
+#> INFO  [2024-12-01 21:41:21.984] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:21.985] Done.
 #> $data
 #>   x y
 #> 1 1 1
@@ -1977,12 +1917,12 @@ 
    Examples
    
 p$add("step1", \(x = ~data) x + 2)
 p$add("step2", \(x = ~step1) x + 2, keepOut = TRUE)
 p$run()
-#> INFO  [2024-12-01 17:13:31.977] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:31.978] Step 1/3 data
-#> INFO  [2024-12-01 17:13:31.980] Step 2/3 step1
-#> INFO  [2024-12-01 17:13:31.982] Step 3/3 step2
-#> INFO  [2024-12-01 17:13:31.983] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:31.983] Done.
+#> INFO  [2024-12-01 21:41:22.118] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.119] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.121] Step 2/3 step1
+#> INFO  [2024-12-01 21:41:22.123] Step 3/3 step2
+#> INFO  [2024-12-01 21:41:22.124] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.125] Done.
 p$collect_out()
 #> $step2
 #> [1] 5 6
@@ -2007,14 +1947,14 @@ 
    Examples
    
 #> 4:  step3    data [NULL]   FALSE   mult    New
 #> 5:  step4    data [NULL]   FALSE   mult    New
 p$run()
-#> INFO  [2024-12-01 17:13:32.014] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.015] Step 1/5 data
-#> INFO  [2024-12-01 17:13:32.018] Step 2/5 step1
-#> INFO  [2024-12-01 17:13:32.019] Step 3/5 step2
-#> INFO  [2024-12-01 17:13:32.021] Step 4/5 step3
-#> INFO  [2024-12-01 17:13:32.023] Step 5/5 step4
-#> INFO  [2024-12-01 17:13:32.024] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.024] Done.
+#> INFO  [2024-12-01 21:41:22.154] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.155] Step 1/5 data
+#> INFO  [2024-12-01 21:41:22.157] Step 2/5 step1
+#> INFO  [2024-12-01 21:41:22.159] Step 3/5 step2
+#> INFO  [2024-12-01 21:41:22.160] Step 4/5 step3
+#> INFO  [2024-12-01 21:41:22.162] Step 5/5 step4
+#> INFO  [2024-12-01 21:41:22.163] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.164] Done.
 p$collect_out(all = TRUE) |> str()
 #> List of 3
 #>  $ data: int [1:2] 1 2
@@ -2170,12 +2110,12 @@ 
    Examples
    
 p$add("add1", \(x = ~data) x + 1)
 p$add("add2", \(x = ~data, y = ~add1) x + y)
 p$run()
-#> INFO  [2024-12-01 17:13:32.148] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.149] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.151] Step 2/3 add1
-#> INFO  [2024-12-01 17:13:32.153] Step 3/3 add2
-#> INFO  [2024-12-01 17:13:32.155] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.155] Done.
+#> INFO  [2024-12-01 21:41:22.273] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.274] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.276] Step 2/3 add1
+#> INFO  [2024-12-01 21:41:22.278] Step 3/3 add2
+#> INFO  [2024-12-01 21:41:22.279] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.280] Done.
 p$get_out("add1")
 #> [1] 2 3
 p$get_out("add2")
@@ -2302,12 +2242,12 @@ 
    Examples
    
 p$add("add1", \(data = ~data, x = 1) x + data)
 p$add("add2", \(x = 1, y = 2, z = ~add1) x + y + z)
 p$run()
-#> INFO  [2024-12-01 17:13:32.197] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.198] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.200] Step 2/3 add1
-#> INFO  [2024-12-01 17:13:32.201] Step 3/3 add2
-#> INFO  [2024-12-01 17:13:32.203] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.203] Done.
+#> INFO  [2024-12-01 21:41:22.322] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.323] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.325] Step 2/3 add1
+#> INFO  [2024-12-01 21:41:22.326] Step 3/3 add2
+#> INFO  [2024-12-01 21:41:22.328] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.328] Done.
 add1 <- p$get_step("add1")
 print(add1)
 #>      step           fun funcName    params depends    out keepOut  group
@@ -2315,12 +2255,12 @@ 
    Examples
    
 #> 1:   add1 <function[1]> function <list[2]>    data    2,3   FALSE   add1
 #>    description                time  state
 #>         <char>              <POSc> <char>
-#> 1:             2024-12-01 17:13:32   Done
+#> 1:             2024-12-01 21:41:22   Done
 add1[["params"]]
 #> [[1]]
 #> [[1]]$data
 #> ~data
-#> <environment: 0x0000024601187450>
+#> <environment: 0x000002328176fc38>
 #> 
 #> [[1]]$x
 #> [1] 1
@@ -2417,12 +2357,12 @@ 
    Examples
    
 p$add("add1", \(x = 1, data = ~data) x + data)
 p$add("add2", \(x = 1, data = ~data) x + data)
 p$run()
-#> INFO  [2024-12-01 17:13:32.246] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.247] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.249] Step 2/3 add1
-#> INFO  [2024-12-01 17:13:32.250] Step 3/3 add2
-#> INFO  [2024-12-01 17:13:32.251] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.252] Done.
+#> INFO  [2024-12-01 21:41:22.370] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.371] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.373] Step 2/3 add1
+#> INFO  [2024-12-01 21:41:22.375] Step 3/3 add2
+#> INFO  [2024-12-01 21:41:22.376] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.376] Done.
 p$get_out("add1")
 #> [1] 2
 p$get_out("add2")
@@ -2433,12 +2373,12 @@ 
    Examples
    
 p$set_params(list(x = 3))
 #> skipping setting parameters x at locked step 'add1'
 p$run()
-#> INFO  [2024-12-01 17:13:32.272] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.273] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.275] Step 2/3 add1 - skip 'locked' step
-#> INFO  [2024-12-01 17:13:32.276] Step 3/3 add2
-#> INFO  [2024-12-01 17:13:32.277] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.278] Done.
+#> INFO  [2024-12-01 21:41:22.396] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.397] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.399] Step 2/3 add1 - skip 'locked' step
+#> INFO  [2024-12-01 21:41:22.400] Step 3/3 add2
+#> INFO  [2024-12-01 21:41:22.401] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.402] Done.
 p$get_out("add1")
 #> [1] 2
 p$get_out("add2")
@@ -2568,25 +2508,25 @@ 
    Examples
    
 p$add("add2", \(x = ~data, y = 2) x + y)
 p$add("mult", \(x = 1, y = 2) x * y, keepOut = TRUE)
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:32.335] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.336] Step 1/4 data
-#> INFO  [2024-12-01 17:13:32.338] Step 2/4 add1
-#> INFO  [2024-12-01 17:13:32.340] Step 3/4 add2
-#> INFO  [2024-12-01 17:13:32.341] Step 4/4 mult
-#> INFO  [2024-12-01 17:13:32.342] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.343] Done.
+#> INFO  [2024-12-01 21:41:22.462] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.463] Step 1/4 data
+#> INFO  [2024-12-01 21:41:22.465] Step 2/4 add1
+#> INFO  [2024-12-01 21:41:22.466] Step 3/4 add2
+#> INFO  [2024-12-01 21:41:22.468] Step 4/4 mult
+#> INFO  [2024-12-01 21:41:22.469] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.469] Done.
 #> $mult
 #> [1] 2
 #> 
 p$replace_step("mult", \(x = ~add1, y = ~add2) x * y, keepOut = TRUE)
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:32.346] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.347] Step 1/4 data - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.348] Step 2/4 add1 - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.349] Step 3/4 add2 - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.350] Step 4/4 mult
-#> INFO  [2024-12-01 17:13:32.351] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.351] Done.
+#> INFO  [2024-12-01 21:41:22.472] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.473] Step 1/4 data - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.474] Step 2/4 add1 - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.475] Step 3/4 add2 - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.476] Step 4/4 mult
+#> INFO  [2024-12-01 21:41:22.477] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.477] Done.
 #> $mult
 #> [1] 6
 #> 
@@ -2601,12 +2541,12 @@ 
    Examples
    
 p$add("f1", \(x = 1) x)
 p$add("f2", \(y = 1) y)
 p$run()
-#> INFO  [2024-12-01 17:13:32.367] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.368] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.369] Step 2/3 f1
-#> INFO  [2024-12-01 17:13:32.371] Step 3/3 f2
-#> INFO  [2024-12-01 17:13:32.372] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.372] Done.
+#> INFO  [2024-12-01 21:41:22.493] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.494] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.495] Step 2/3 f1
+#> INFO  [2024-12-01 21:41:22.496] Step 3/3 f2
+#> INFO  [2024-12-01 21:41:22.498] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.498] Done.
 p
 #>      step depends    out keepOut  group  state
 #>    <char>  <list> <list>  <lgcl> <char> <char>
@@ -2631,13 +2571,13 @@ 
    Examples
    
 p$add("add2", \(x = ~add1, z = 2) x + z)
 p$add("final", \(x = ~add1, y = ~add2) x * y, keepOut = TRUE)
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:32.382] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.383] Step 1/4 data
-#> INFO  [2024-12-01 17:13:32.385] Step 2/4 add1
-#> INFO  [2024-12-01 17:13:32.388] Step 3/4 add2
-#> INFO  [2024-12-01 17:13:32.389] Step 4/4 final
-#> INFO  [2024-12-01 17:13:32.391] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.391] Done.
+#> INFO  [2024-12-01 21:41:22.508] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.509] Step 1/4 data
+#> INFO  [2024-12-01 21:41:22.511] Step 2/4 add1
+#> INFO  [2024-12-01 21:41:22.513] Step 3/4 add2
+#> INFO  [2024-12-01 21:41:22.515] Step 4/4 final
+#> INFO  [2024-12-01 21:41:22.516] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.517] Done.
 #> $final
 #> [1] 8
 #> 
@@ -2650,25 +2590,25 @@ 
    Examples
    
 #> 3:   add2      add1      4   FALSE   add2 Outdated
 #> 4:  final add1,add2      8    TRUE  final Outdated
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:32.398] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.399] Step 1/4 data - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.400] Step 2/4 add1 - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.401] Step 3/4 add2
-#> INFO  [2024-12-01 17:13:32.402] Step 4/4 final
-#> INFO  [2024-12-01 17:13:32.404] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.404] Done.
+#> INFO  [2024-12-01 21:41:22.524] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.525] Step 1/4 data - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.526] Step 2/4 add1 - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.526] Step 3/4 add2
+#> INFO  [2024-12-01 21:41:22.528] Step 4/4 final
+#> INFO  [2024-12-01 21:41:22.529] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.530] Done.
 #> $final
 #> [1] 12
 #> 
 p$run(cleanUnkept = TRUE)  # clean up temporary results
-#> INFO  [2024-12-01 17:13:32.406] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.407] Step 1/4 data - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.407] Step 2/4 add1 - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.408] Step 3/4 add2 - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.409] Step 4/4 final - skip 'done' step
-#> INFO  [2024-12-01 17:13:32.410] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.410] Clean temporary results.
-#> INFO  [2024-12-01 17:13:32.411] Done.
+#> INFO  [2024-12-01 21:41:22.531] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.532] Step 1/4 data - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.533] Step 2/4 add1 - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.534] Step 3/4 add2 - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.535] Step 4/4 final - skip 'done' step
+#> INFO  [2024-12-01 21:41:22.535] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.536] Clean temporary results.
+#> INFO  [2024-12-01 21:41:22.536] Done.
 p
 #>      step   depends    out keepOut  group    state
 #>    <char>    <list> <list>  <lgcl> <char>   <char>
@@ -2687,17 +2627,17 @@ 
    Examples
    
     }
 )
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:32.417] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.418] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.420] Step 2/3 add1
-#> INFO  [2024-12-01 17:13:32.422] Step 3/3 new_pipe
-#> INFO  [2024-12-01 17:13:32.425] Abort pipeline execution and restart on new.
-#> INFO  [2024-12-01 17:13:32.426] Start run of 'new_pipe' pipeline:
-#> INFO  [2024-12-01 17:13:32.426] Step 1/3 data
-#> INFO  [2024-12-01 17:13:32.428] Step 2/3 add1
-#> INFO  [2024-12-01 17:13:32.430] Step 3/3 add2
-#> INFO  [2024-12-01 17:13:32.431] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:32.432] Done.
+#> INFO  [2024-12-01 21:41:22.543] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.544] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.546] Step 2/3 add1
+#> INFO  [2024-12-01 21:41:22.547] Step 3/3 new_pipe
+#> INFO  [2024-12-01 21:41:22.550] Abort pipeline execution and restart on new.
+#> INFO  [2024-12-01 21:41:22.551] Start run of 'new_pipe' pipeline:
+#> INFO  [2024-12-01 21:41:22.552] Step 1/3 data
+#> INFO  [2024-12-01 21:41:22.554] Step 2/3 add1
+#> INFO  [2024-12-01 21:41:22.556] Step 3/3 add2
+#> INFO  [2024-12-01 21:41:22.557] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:22.557] Done.
 #> $add2
 #> [1] 5
 #> 
@@ -2723,28 +2663,28 @@ 
    Examples
    
 p$add("add2", \(x = ~add1, z = 2) x + z)
 p$add("mult", \(x = ~add1, y = ~add2) x * y)
 p$run_step("add2")
-#> INFO  [2024-12-01 17:13:35.507] Start step run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.508] Step 1/3 data (upstream)
-#> INFO  [2024-12-01 17:13:35.510] Step 2/3 add1 (upstream)
-#> INFO  [2024-12-01 17:13:35.512] Step 3/3 add2
-#> INFO  [2024-12-01 17:13:35.514] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.514] Done.
+#> INFO  [2024-12-01 21:41:25.633] Start step run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.634] Step 1/3 data (upstream)
+#> INFO  [2024-12-01 21:41:25.636] Step 2/3 add1 (upstream)
+#> INFO  [2024-12-01 21:41:25.638] Step 3/3 add2
+#> INFO  [2024-12-01 21:41:25.640] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.640] Done.
 p$run_step("add2", downstream = TRUE)
-#> INFO  [2024-12-01 17:13:35.528] Start step run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.529] Step 1/4 data (upstream)
-#> INFO  [2024-12-01 17:13:35.532] Step 2/4 add1 (upstream)
-#> INFO  [2024-12-01 17:13:35.534] Step 3/4 add2
-#> INFO  [2024-12-01 17:13:35.535] Step 4/4 mult (downstream)
-#> INFO  [2024-12-01 17:13:35.536] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.537] Done.
+#> INFO  [2024-12-01 21:41:25.655] Start step run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.655] Step 1/4 data (upstream)
+#> INFO  [2024-12-01 21:41:25.658] Step 2/4 add1 (upstream)
+#> INFO  [2024-12-01 21:41:25.660] Step 3/4 add2
+#> INFO  [2024-12-01 21:41:25.664] Step 4/4 mult (downstream)
+#> INFO  [2024-12-01 21:41:25.665] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.666] Done.
 p$run_step("mult", upstream = TRUE)
-#> INFO  [2024-12-01 17:13:35.538] Start step run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.539] Step 1/4 data (upstream)
-#> INFO  [2024-12-01 17:13:35.541] Step 2/4 add1 (upstream)
-#> INFO  [2024-12-01 17:13:35.543] Step 3/4 add2 (upstream)
-#> INFO  [2024-12-01 17:13:35.545] Step 4/4 mult
-#> INFO  [2024-12-01 17:13:35.546] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.546] Done.
+#> INFO  [2024-12-01 21:41:25.667] Start step run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.668] Step 1/4 data (upstream)
+#> INFO  [2024-12-01 21:41:25.670] Step 2/4 add1 (upstream)
+#> INFO  [2024-12-01 21:41:25.672] Step 3/4 add2 (upstream)
+#> INFO  [2024-12-01 21:41:25.674] Step 4/4 mult
+#> INFO  [2024-12-01 21:41:25.675] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.675] Done.
 
 ## ------------------------------------------------
 ## Method `Pipeline$set_data`
@@ -2753,21 +2693,21 @@ 
    Examples
    
 p <- Pipeline$new("pipe", data = 1)
 p$add("add1", \(x = ~data, y = 1) x + y, keepOut = TRUE)
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:35.550] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.551] Step 1/2 data
-#> INFO  [2024-12-01 17:13:35.553] Step 2/2 add1
-#> INFO  [2024-12-01 17:13:35.554] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.555] Done.
+#> INFO  [2024-12-01 21:41:25.679] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.680] Step 1/2 data
+#> INFO  [2024-12-01 21:41:25.682] Step 2/2 add1
+#> INFO  [2024-12-01 21:41:25.683] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.684] Done.
 #> $add1
 #> [1] 2
 #> 
 p$set_data(3)
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:35.557] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.558] Step 1/2 data
-#> INFO  [2024-12-01 17:13:35.560] Step 2/2 add1
-#> INFO  [2024-12-01 17:13:35.561] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.562] Done.
+#> INFO  [2024-12-01 21:41:25.687] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.688] Step 1/2 data
+#> INFO  [2024-12-01 21:41:25.690] Step 2/2 add1
+#> INFO  [2024-12-01 21:41:25.691] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.692] Done.
 #> $add1
 #> [1] 4
 #> 
@@ -2795,18 +2735,18 @@ 
    Examples
    
 #> 8: add1.c        data.c [NULL]    TRUE      c Outdated
 #> 9: mult.c data.c,add1.c [NULL]    TRUE      c Outdated
 p3$run()$collect_out() |> str()
-#> INFO  [2024-12-01 17:13:35.584] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.585] Step 1/9 data.a
-#> INFO  [2024-12-01 17:13:35.587] Step 2/9 add1.a
-#> INFO  [2024-12-01 17:13:35.589] Step 3/9 mult.a
-#> INFO  [2024-12-01 17:13:35.591] Step 4/9 data.b
-#> INFO  [2024-12-01 17:13:35.593] Step 5/9 add1.b
-#> INFO  [2024-12-01 17:13:35.594] Step 6/9 mult.b
-#> INFO  [2024-12-01 17:13:35.596] Step 7/9 data.c
-#> INFO  [2024-12-01 17:13:35.598] Step 8/9 add1.c
-#> INFO  [2024-12-01 17:13:35.600] Step 9/9 mult.c
-#> INFO  [2024-12-01 17:13:35.601] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.602] Done.
+#> INFO  [2024-12-01 21:41:25.714] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.715] Step 1/9 data.a
+#> INFO  [2024-12-01 21:41:25.717] Step 2/9 add1.a
+#> INFO  [2024-12-01 21:41:25.719] Step 3/9 mult.a
+#> INFO  [2024-12-01 21:41:25.721] Step 4/9 data.b
+#> INFO  [2024-12-01 21:41:25.723] Step 5/9 add1.b
+#> INFO  [2024-12-01 21:41:25.725] Step 6/9 mult.b
+#> INFO  [2024-12-01 21:41:25.726] Step 7/9 data.c
+#> INFO  [2024-12-01 21:41:25.728] Step 8/9 add1.c
+#> INFO  [2024-12-01 21:41:25.730] Step 9/9 mult.c
+#> INFO  [2024-12-01 21:41:25.731] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.732] Done.
 #> List of 3
 #>  $ a:List of 2
 #>   ..$ add1.a: num 2
@@ -2836,18 +2776,18 @@ 
    Examples
    
 #> 8: add1.c        data.c [NULL]    TRUE add1.c Outdated
 #> 9: mult.c data.c,add1.c [NULL]    TRUE mult.c Outdated
 p3$run()$collect_out() |> str()
-#> INFO  [2024-12-01 17:13:35.627] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.628] Step 1/9 data.a
-#> INFO  [2024-12-01 17:13:35.631] Step 2/9 add1.a
-#> INFO  [2024-12-01 17:13:35.632] Step 3/9 mult.a
-#> INFO  [2024-12-01 17:13:35.637] Step 4/9 data.b
-#> INFO  [2024-12-01 17:13:35.639] Step 5/9 add1.b
-#> INFO  [2024-12-01 17:13:35.641] Step 6/9 mult.b
-#> INFO  [2024-12-01 17:13:35.642] Step 7/9 data.c
-#> INFO  [2024-12-01 17:13:35.644] Step 8/9 add1.c
-#> INFO  [2024-12-01 17:13:35.646] Step 9/9 mult.c
-#> INFO  [2024-12-01 17:13:35.647] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.648] Done.
+#> INFO  [2024-12-01 21:41:25.757] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.758] Step 1/9 data.a
+#> INFO  [2024-12-01 21:41:25.760] Step 2/9 add1.a
+#> INFO  [2024-12-01 21:41:25.762] Step 3/9 mult.a
+#> INFO  [2024-12-01 21:41:25.764] Step 4/9 data.b
+#> INFO  [2024-12-01 21:41:25.766] Step 5/9 add1.b
+#> INFO  [2024-12-01 21:41:25.767] Step 6/9 mult.b
+#> INFO  [2024-12-01 21:41:25.769] Step 7/9 data.c
+#> INFO  [2024-12-01 21:41:25.771] Step 8/9 add1.c
+#> INFO  [2024-12-01 21:41:25.773] Step 9/9 mult.c
+#> INFO  [2024-12-01 21:41:25.774] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.775] Done.
 #> List of 6
 #>  $ add1.a: num 2
 #>  $ mult.a: num 2
@@ -2865,13 +2805,13 @@ 
    Examples
    
 p$add("add2", \(x = ~data, y = 2) x + y)
 p$add("mult", \(x = ~add1, y = ~add2) x * y)
 p$run()$collect_out()
-#> INFO  [2024-12-01 17:13:35.656] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:35.657] Step 1/4 data
-#> INFO  [2024-12-01 17:13:35.660] Step 2/4 add1
-#> INFO  [2024-12-01 17:13:35.661] Step 3/4 add2
-#> INFO  [2024-12-01 17:13:35.663] Step 4/4 mult
-#> INFO  [2024-12-01 17:13:35.664] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:35.665] Done.
+#> INFO  [2024-12-01 21:41:25.783] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:25.784] Step 1/4 data
+#> INFO  [2024-12-01 21:41:25.786] Step 2/4 add1
+#> INFO  [2024-12-01 21:41:25.788] Step 3/4 add2
+#> INFO  [2024-12-01 21:41:25.790] Step 4/4 mult
+#> INFO  [2024-12-01 21:41:25.791] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:25.792] Done.
 #> $add1
 #> [1] 2
 #> 
@@ -3050,23 +2990,19 @@ 
    Examples
    
 #>
    -

    - -
    +
    -
    diff --git a/docs/reference/index.html b/docs/reference/index.html index f8e9131..0bdbf62 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -1,164 +1,123 @@ -Package index • pipeflow - - -
    -
    -
    -
    -

    Reference

    +
      • +
      • +
      • +
      • +
      • +
    + + +
    +
    +
    +
    +

    Package index

    - - - - - - - - -
    -

    Pipeline

    -

    A pipeline consists of a sequence of analysis steps.

    -
    -

    Pipeline

    -

    Pipeline Class

    -

    pipe_add() pipe_append() pipe_append_to_step_names() pipe_clone() pipe_collect_out() pipe_discard_steps() pipe_get_data() pipe_get_depends() pipe_get_depends_down() pipe_get_depends_up() pipe_get_graph() pipe_get_out() pipe_get_params() pipe_get_params_at_step() pipe_get_params_unique() pipe_get_params_unique_json() pipe_get_step() pipe_get_step_names() pipe_get_step_number() pipe_has_step() pipe_insert_after() pipe_insert_before() pipe_length() pipe_lock_step() pipe_new() pipe_print() pipe_pop_step() pipe_pop_steps_after() pipe_pop_steps_from() pipe_remove_step() pipe_rename_step() pipe_replace_step() pipe_reset() pipe_run() pipe_run_step() pipe_set_data() pipe_set_data_split() pipe_set_keep_out() pipe_set_params() pipe_set_params_at_step() pipe_split() pipe_unlock_step()

    -

    Pipeline alias functions

    -

    Logging

    -

    -
    -

    set_log_layout()

    -

    Set pipeflow log layout

    - - -
    +
    +

    Pipeline

    +

    A pipeline is created by instantiating an object of the Pipeline class. Follow the Pipeline link below for a detailed documenation of all member functions provided by a Pipeline object.

    -
    + + +
    +
    +

    Site built with pkgdown 2.1.1.

    +
    +
    diff --git a/docs/reference/pipelineAliases.html b/docs/reference/pipelineAliases.html index 9e580d1..7dfbf8b 100644 --- a/docs/reference/pipelineAliases.html +++ b/docs/reference/pipelineAliases.html @@ -1,135 +1,69 @@ -Pipeline alias functions — pipelineAliases • pipeflow - - -
    -
    - - - -
    -
    -
    -

    Pipeline alias functions

    - Source: R/pipeflow_aliases.R - +
    +
    +
    +
    +

    Pipeline alias functions

    + Source: R/pipeflow_aliases.R +
    pipelineAliases.Rd
    -
    +

    Alias functions, one for each member function of a Pipeline object.

    -
    +
    +

    Usage

    pipe_add(pip, ...)
 
 pipe_append(pip, ...)
@@ -215,8 +149,8 @@ 
    Pipeline alias functions
    
 pipe_unlock_step(pip, ...)
    -
    -

    Arguments

    +
    +

    Arguments

    pip
    @@ -227,28 +161,24 @@

    Arguments

    Arguments passed to the respective pipeline method

    -
    -

    Value

    +
    +

    Value

    The result of the respective pipeline method

    -
    - -
    +
    -
    diff --git a/docs/reference/set_log_layout.html b/docs/reference/set_log_layout.html index 5fc26e1..6efc545 100644 --- a/docs/reference/set_log_layout.html +++ b/docs/reference/set_log_layout.html @@ -1,159 +1,92 @@ -Set pipeflow log layout — set_log_layout • pipeflow - - -
    -
    -
    -
    -

    Set pipeflow log layout

    - Source: R/pipeflow_log.R - +
      • +
      • +
      • +
      • +
      • +
    + + +
    +
    +
    +
    +

    Set pipeflow log layout

    + Source: R/pipeflow_log.R +
    set_log_layout.Rd
    -
    +

    Set pipeflow log layout

    -
    +
    +

    Usage

    set_log_layout(layout)
    -
    -

    Arguments

    +
    +

    Arguments

    layout

    Layout name

    -
    -

    Value

    +
    +

    Value

    invisibly returns logger object

    -
    -

    Examples

    +
    +

    Examples

    p <- Pipeline$new("pipe", data = 1:2)
 p$add("add1", \(data = ~data, x = 1) x + data)
 p$run()
-#> INFO  [2024-12-01 17:13:39.067] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:39.068] Step 1/2 data
-#> INFO  [2024-12-01 17:13:39.070] Step 2/2 add1
-#> INFO  [2024-12-01 17:13:39.071] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:39.072] Done.
+#> INFO  [2024-12-01 21:41:29.126] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:29.127] Step 1/2 data
+#> INFO  [2024-12-01 21:41:29.129] Step 2/2 add1
+#> INFO  [2024-12-01 21:41:29.130] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:29.131] Done.
 
 lg <- set_log_layout("json")
 print(lg)
@@ -163,38 +96,34 @@ 
    Examples
    
 #>   [[1]]: <AppenderConsole> [all] -> console
 
 p$run()
-#> {"application":"pipeflow","level":"info","time":"2024-12-01T17:13:39+1:00","message":"Start run of 'pipe' pipeline:"}
-#> {"application":"pipeflow","level":"info","time":"2024-12-01T17:13:39+1:00","message":"Step 1/2 data - skip 'done' step"}
-#> {"application":"pipeflow","level":"info","time":"2024-12-01T17:13:39+1:00","message":"Step 2/2 add1 - skip 'done' step"}
-#> {"application":"pipeflow","level":"info","time":"2024-12-01T17:13:39+1:00","message":"Finished execution of steps."}
-#> {"application":"pipeflow","level":"info","time":"2024-12-01T17:13:39+1:00","message":"Done."}
+#> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Start run of 'pipe' pipeline:"}
+#> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Step 1/2 data - skip 'done' step"}
+#> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Step 2/2 add1 - skip 'done' step"}
+#> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Finished execution of steps."}
+#> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Done."}
 
 set_log_layout("text")
 p$run()
-#> INFO  [2024-12-01 17:13:39.104] Start run of 'pipe' pipeline:
-#> INFO  [2024-12-01 17:13:39.105] Step 1/2 data - skip 'done' step
-#> INFO  [2024-12-01 17:13:39.106] Step 2/2 add1 - skip 'done' step
-#> INFO  [2024-12-01 17:13:39.107] Finished execution of steps.
-#> INFO  [2024-12-01 17:13:39.107] Done.
+#> INFO  [2024-12-01 21:41:29.169] Start run of 'pipe' pipeline:
+#> INFO  [2024-12-01 21:41:29.170] Step 1/2 data - skip 'done' step
+#> INFO  [2024-12-01 21:41:29.171] Step 2/2 add1 - skip 'done' step
+#> INFO  [2024-12-01 21:41:29.171] Finished execution of steps.
+#> INFO  [2024-12-01 21:41:29.172] Done.
    -
    - -
    +
    -
    diff --git a/docs/search.json b/docs/search.json new file mode 100644 index 0000000..3608847 --- /dev/null +++ b/docs/search.json @@ -0,0 +1 @@ +[{"path":"https://github.com/rpahl/pipeflow/articles/collect-output.html","id":"flag-output-steps","dir":"Articles","previous_headings":"","what":"Flag output steps","title":"Collecting output","text":"Output steps flagged settting keepOut argument TRUE adding step pipeline. following example, want keep output steps data_summary, model_summary, model_plot. Looking pipeline, see steps data_summary, model-summary, model_plot flagged accordingly (see column keepOut). Graphically, steps flagged keepOut = TRUE displayed circle shape “normal” steps shown rectangle boxes. Now let’s run collect output flagged steps using collect_out method, returns list output flagged steps. expected, output list contains output flagged steps.","code":"library(pipeflow) library(ggplot2) pip <- pipe_new( \"my-pipeline\", data = airquality ) |> pipe_add( \"data_prep\", function(data = ~data) { replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9) } ) |> pipe_add( \"data_summary\", function( data = ~data_prep, xVar = \"Temp.Celsius\", yVar = \"Ozone\" ) { data[, c(xVar, yVar)] }, keepOut = TRUE # <- keep this ) |> pipe_add( \"model_fit\", function( data = ~data_prep, xVar = \"Temp.Celsius\", yVar = \"Ozone\" ) { lm(paste(yVar, \"~\", xVar), data = data) } ) |> pipe_add( \"model_summary\", function( fit = ~model_fit ) { summary(fit) }, keepOut = TRUE # <- keep this ) |> pipe_add( \"model_plot\", function( model = ~model_fit, data = ~data_prep, xVar = \"Temp.Celsius\", yVar = \"Ozone\", title = \"Linear model fit\" ) { coeffs <- coefficients(model) ggplot(data) + geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) + geom_abline(intercept = coeffs[1], slope = coeffs[2]) + labs(title = title) }, keepOut = TRUE # <- keep this ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: data_summary data_prep [NULL] TRUE data_summary New # 4: model_fit data_prep [NULL] FALSE model_fit New # 5: model_summary model_fit [NULL] TRUE model_summary New # 6: model_plot model_fit,data_prep [NULL] TRUE model_plot New pip$run() # INFO [2024-12-01 21:41:31.888] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:31.916] Step 1/6 data # INFO [2024-12-01 21:41:31.925] Step 2/6 data_prep # INFO [2024-12-01 21:41:31.948] Step 3/6 data_summary # INFO [2024-12-01 21:41:31.950] Step 4/6 model_fit # INFO [2024-12-01 21:41:31.954] Step 5/6 model_summary # INFO [2024-12-01 21:41:31.956] Step 6/6 model_plot # INFO [2024-12-01 21:41:31.968] Finished execution of steps. # INFO [2024-12-01 21:41:31.968] Done. out <- pip$collect_out() names(out) # [1] \"data_summary\" \"model_summary\" \"model_plot\" str(out, max.level = 1) # List of 3 # $ data_summary :'data.frame': 153 obs. of 2 variables: # $ model_summary:List of 12 # ..- attr(*, \"class\")= chr \"summary.lm\" # $ model_plot :List of 11 # ..- attr(*, \"class\")= chr [1:2] \"gg\" \"ggplot\""},{"path":"https://github.com/rpahl/pipeflow/articles/collect-output.html","id":"grouping-output-steps","dir":"Articles","previous_headings":"","what":"Grouping output steps","title":"Collecting output","text":"Often certain output steps related grouped together. can achieved conveniently setting group argument adding step pipeline. Let’s illustrate slightly modifying previous example. Looking pipeline, defined groups shown group column. see, default, group identical step name, , step represents trivial case one-sized group. , run pipeline collect output. can see, output related modelling grouped one sublist named Model.","code":"pip <- Pipeline$new(\"my-pipeline\", data = airquality) |> pipe_add( \"data_prep\", function(data = ~data) { replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9) } ) |> pipe_add( \"used_data\", function( data = ~data_prep, xVar = \"Temp.Celsius\", yVar = \"Ozone\" ) { data[, c(xVar, yVar)] }, keepOut = TRUE, group = \"Data\" # <- define 'Data' group here ) |> pipe_add( \"model_fit\", function( data = ~data_prep, xVar = \"Temp.Celsius\", yVar = \"Ozone\" ) { lm(paste(yVar, \"~\", xVar), data = data) } ) |> pipe_add( \"model_summary\", function( fit = ~model_fit ) { summary(fit) }, keepOut = TRUE, group = \"Model\" # <- define 'Model' group here ) |> pipe_add( \"model_plot\", function( model = ~model_fit, data = ~data_prep, xVar = \"Temp.Celsius\", yVar = \"Ozone\", title = \"Linear model fit\" ) { coeffs <- coefficients(model) ggplot(data) + geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) + geom_abline(intercept = coeffs[1], slope = coeffs[2]) + labs(title = title) }, keepOut = TRUE, group = \"Model\" # <- define 'Model' group here ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: used_data data_prep [NULL] TRUE Data New # 4: model_fit data_prep [NULL] FALSE model_fit New # 5: model_summary model_fit [NULL] TRUE Model New # 6: model_plot model_fit,data_prep [NULL] TRUE Model New pip$run() # INFO [2024-12-01 21:41:32.198] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:32.199] Step 1/6 data # INFO [2024-12-01 21:41:32.202] Step 2/6 data_prep # INFO [2024-12-01 21:41:32.205] Step 3/6 used_data # INFO [2024-12-01 21:41:32.206] Step 4/6 model_fit # INFO [2024-12-01 21:41:32.229] Step 5/6 model_summary # INFO [2024-12-01 21:41:32.231] Step 6/6 model_plot # INFO [2024-12-01 21:41:32.236] Finished execution of steps. # INFO [2024-12-01 21:41:32.237] Done. out <- pip$collect_out() names(out) # [1] \"Data\" \"Model\" str(out, max.level = 2) # List of 2 # $ Data :'data.frame': 153 obs. of 2 variables: # ..$ Temp.Celsius: num [1:153] 19.4 22.2 23.3 16.7 13.3 ... # ..$ Ozone : int [1:153] 41 36 12 18 NA 28 23 19 8 NA ... # $ Model:List of 2 # ..$ model_summary:List of 12 # .. ..- attr(*, \"class\")= chr \"summary.lm\" # ..$ model_plot :List of 11 # .. ..- attr(*, \"class\")= chr [1:2] \"gg\" \"ggplot\""},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"define-two-pipelines","dir":"Articles","previous_headings":"","what":"Define two pipelines","title":"Combining pipelines","text":"Let’s define one pipeline used data_preprocessing one modeling. Data preprocessing pipeline: Modeling pipeline: Graphically, two pipelines look follows:","code":"library(pipeflow) library(ggplot2) pip1 <- pipe_new( \"preprocessing\", data = airquality ) |> pipe_add( \"data_prep\", function(data = ~data) { replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9) } ) |> pipe_add( \"standardize\", function( data = ~data_prep, yVar = \"Ozone\" ) { data[, yVar] <- scale(data[, yVar]) data } ) pip1 # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: standardize data_prep [NULL] FALSE standardize New pip2 <- pipe_new( \"modeling\", data = airquality ) |> pipe_add( \"fit\", function( data = ~data, xVar = \"Temp\", yVar = \"Ozone\" ) { lm(paste(yVar, \"~\", xVar), data = data) } ) |> pipe_add( \"plot\", function( model = ~fit, data = ~data, xVar = \"Temp\", yVar = \"Ozone\", title = \"Linear model fit\" ) { coeffs <- coefficients(model) ggplot(data) + geom_point(aes(.data[[xVar]], .data[[yVar]])) + geom_abline(intercept = coeffs[1], slope = coeffs[2]) + labs(title = title) } ) pip2 # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: fit data [NULL] FALSE fit New # 3: plot fit,data [NULL] FALSE plot New"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"combine-pipelines","dir":"Articles","previous_headings":"","what":"Combine pipelines","title":"Combining pipelines","text":"Next combine two pipelines. can using append function. First , note data step second pipeline appended name second pipeline. particular, first step second pipeline renamed data data.modeling (line 4 step column) likewise dependencies second pipeline updated (see lines 5-6 depends column). , appending two pipelines, pipeflow ensures step names remain unique resulting combined pipeline therefore automatically renames duplicated step names necessary. Now, can also seen graphical representation pipeline, two pipelines yet connected. make actual use combined pipeline, therefore want use output first pipeline input second pipeline, , want use output standardize step data parameter input data.modeling step. One way achieve use replace function described earlier vignette modify pipeline, example:","code":"pip <- pip1$append(pip2) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: standardize data_prep [NULL] FALSE standardize New # 4: data.modeling [NULL] FALSE data New # 5: fit data.modeling [NULL] FALSE fit New # 6: plot fit,data.modeling [NULL] FALSE plot New pip$replace_step(\"data.modeling\", function(data = ~standardize) data) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: standardize data_prep [NULL] FALSE standardize New # 4: data.modeling standardize [NULL] FALSE data.modeling New # 5: fit data.modeling [NULL] FALSE fit Outdated # 6: plot fit,data.modeling [NULL] FALSE plot Outdated"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"relative-indexing","dir":"Articles","previous_headings":"Combine pipelines","what":"Relative indexing","title":"Combining pipelines","text":"Since name last step might always known1, pipeflow package also provides relative position indexing mechanism, allows rewrite command follows: Generally speaking, relative indexing mechanism allows refer steps positioned current step. index ~-1 can interpreted “go one step back”, ~-2 “go two steps back”, . Since scenario connecting two pipelines common avoid replacement steps manually, append function actually provides argument outAsIn allows appending “connecting” pipelines one go: inspect data.modeling step, see “hood” original step indeed replaced output last step first pipeline using relative indexing mechanism manually .","code":"pip$replace_step(\"data.modeling\", function(data = ~-1) data) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: standardize data_prep [NULL] FALSE standardize New # 4: data.modeling standardize [NULL] FALSE data.modeling New # 5: fit data.modeling [NULL] FALSE fit Outdated # 6: plot fit,data.modeling [NULL] FALSE plot Outdated pip <- pip1$append(pip2, outAsIn = TRUE) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: standardize data_prep [NULL] FALSE standardize New # 4: data.modeling standardize [NULL] FALSE data.modeling New # 5: fit data.modeling [NULL] FALSE fit Outdated # 6: plot fit,data.modeling [NULL] FALSE plot Outdated pip$get_step(\"data.modeling\")[[\"fun\"]] # [[1]] # function (data = ~-1) # data # # "},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"run-combined-pipeline","dir":"Articles","previous_headings":"","what":"Run combined pipeline","title":"Combining pipelines","text":"Let’s now run combined pipeline inspect plot. can see, plot shows linear model fit using standardized data. can now go ahead example change x-variable model rerun pipeline. creating pipelines, usually lot steps calculating intermediate results steps contain final results interested . example, interested final plot output. real-world scenario, pipeline contain many steps interest us. see conveniently tag, collect possibly group output final steps, see next vignette Collecting output.","code":"pip$run() # INFO [2024-12-01 21:41:36.693] Start run of 'preprocessing.modeling' pipeline: # INFO [2024-12-01 21:41:36.763] Step 1/6 data # INFO [2024-12-01 21:41:36.772] Step 2/6 data_prep # INFO [2024-12-01 21:41:36.788] Step 3/6 standardize # INFO [2024-12-01 21:41:36.791] Step 4/6 data.modeling # INFO [2024-12-01 21:41:36.793] Step 5/6 fit # INFO [2024-12-01 21:41:36.797] Step 6/6 plot # INFO [2024-12-01 21:41:36.810] Finished execution of steps. # INFO [2024-12-01 21:41:36.811] Done. pip$get_out(\"plot\") # Warning: Removed 37 rows containing missing values or values outside the scale range # (`geom_point()`). pip$set_params(list(xVar = \"Temp.Celsius\")) pip$run() # INFO [2024-12-01 21:41:37.360] Start run of 'preprocessing.modeling' pipeline: # INFO [2024-12-01 21:41:37.361] Step 1/6 data - skip 'done' step # INFO [2024-12-01 21:41:37.362] Step 2/6 data_prep - skip 'done' step # INFO [2024-12-01 21:41:37.362] Step 3/6 standardize - skip 'done' step # INFO [2024-12-01 21:41:37.363] Step 4/6 data.modeling - skip 'done' step # INFO [2024-12-01 21:41:37.364] Step 5/6 fit # INFO [2024-12-01 21:41:37.368] Step 6/6 plot # INFO [2024-12-01 21:41:37.385] Finished execution of steps. # INFO [2024-12-01 21:41:37.386] Done. pip$get_out(\"plot\") # Warning: Removed 37 rows containing missing values or values outside the scale range # (`geom_point()`)."},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"a-simple-example-to-get-started","dir":"Articles","previous_headings":"","what":"A simple example to get started","title":"Get started with pipeflow","text":"example, ’ll use base R’s airquality dataset. goal create analysis pipeline performs following steps: add new data column Temp.Celsius containing temperature degrees Celsius fit linear model data plot data model fit. following, ’ll show define run pipeline, inspect output specifics steps, finally re-run pipeline different parameter settings, one selling points using pipeline.","code":"head(airquality) # Ozone Solar.R Wind Temp Month Day # 1 41 190 7.4 67 5 1 # 2 36 118 8.0 72 5 2 # 3 12 149 12.6 74 5 3 # 4 18 313 11.5 62 5 4 # 5 NA NA 14.3 56 5 5 # 6 28 NA 14.9 66 5 6"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"define-the-pipeline","dir":"Articles","previous_headings":"A simple example to get started","what":"Define the pipeline","title":"Get started with pipeflow","text":"easier understanding, go step step. First, create new pipeline name “-pipeline” airquality dataset input data. Let’s view pipeline ’s table form initialization. , step represented one row table step column lists name step state column shows current state step. now, don’t worry columns - explained later. can see, data always first step pipeline initially state New, means pipeline run yet. Next, add step called data_prep, consists function takes output data step first argument, adds new column returns modified data output. refer output earlier pipeline step, always use tilde (~) operator. , let’s inspect pipeline. second step called data_prep added depends data step can seen column depends, list dependencies step. Next, add step called model_fit, fits linear model data. function takes output data_prep defines parameter xVar, used specify variable used predictor linear model. Lastly, add step called model_plot, plots data linear model fit. function uses output model_fit data_prep step. also defines xVar parameter parameter title, used title plot. completes pipeline, now looks follows: last line, see model_plot step depends model_fit data_prep step. visualize pipeline, can use get_graph() method, returns list arguments can passed visNetwork function visNetwork package.","code":"library(pipeflow) library(ggplot2) pip <- Pipeline$new(\"my-pipeline\", data = airquality) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New pip$add( \"data_prep\", function(data = ~data) { replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9) } ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New pip$add( \"model_fit\", function( data = ~data_prep, xVar = \"Temp.Celsius\" ) { lm(paste(\"Ozone ~\", xVar), data = data) } ) pip$add( \"model_plot\", function( model = ~model_fit, data = ~data_prep, xVar = \"Temp.Celsius\", title = \"Linear model fit\" ) { coeffs <- coefficients(model) ggplot(data) + geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) + geom_abline(intercept = coeffs[1], slope = coeffs[2]) + labs(title = title) } ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data [NULL] FALSE data_prep New # 3: model_fit data_prep [NULL] FALSE model_fit New # 4: model_plot model_fit,data_prep [NULL] FALSE model_plot New library(visNetwork) do.call(visNetwork, args = pip$get_graph()) |> visHierarchicalLayout(direction = \"LR\")"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"run-the-pipeline-and-inspect-the-output","dir":"Articles","previous_headings":"A simple example to get started","what":"Run the pipeline and inspect the output","title":"Get started with pipeflow","text":"run pipeline, simply call run() method pipeline, produce following output: Let’s inspect pipeline . can see state steps changed New Done, graphically represented color change blue green. addition, output added column. can inspect output specific step calling get_out() method pipeline.","code":"pip$run() # INFO [2024-12-01 21:41:41.560] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:41.586] Step 1/4 data # INFO [2024-12-01 21:41:41.597] Step 2/4 data_prep # INFO [2024-12-01 21:41:41.620] Step 3/4 model_fit # INFO [2024-12-01 21:41:41.623] Step 4/4 model_plot # INFO [2024-12-01 21:41:41.635] Finished execution of steps. # INFO [2024-12-01 21:41:41.636] Done. pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: model_fit data_prep FALSE model_fit Done # 4: model_plot model_fit,data_prep FALSE model_plot Done pip$get_out(\"model_fit\") # # Call: # lm(formula = paste(\"Ozone ~\", xVar), data = data) # # Coefficients: # (Intercept) Temp.Celsius # -69.277 4.372 pip$get_out(\"model_plot\")"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"change-parameters-of-the-pipeline","dir":"Articles","previous_headings":"A simple example to get started","what":"Change parameters of the pipeline","title":"Get started with pipeflow","text":"parameter pipeline bound specific step (.e. defined tilde (~) operator) can changed calling set_params() method pipeline. Let’s first list unbound parameters: Now let’s assume want change xVar entire pipeline \"Solar.R\". can simply follows: Changing parameters pipeline least effect steps depend changed parameter. can see model_fit model_plot steps now state Outdated (graphically indicated orange color), means need rerun. can calling run() method pipeline . outdated steps re-run expected output updated accordingly. just change title plot, obviously model_plot step needs rerun. instead change input data, since steps depend , expect steps rerun. Next, let’s see modify pipeline.","code":"pip$get_params() # $model_fit # $model_fit$xVar # [1] \"Temp.Celsius\" # # # $model_plot # $model_plot$xVar # [1] \"Temp.Celsius\" # # $model_plot$title # [1] \"Linear model fit\" pip$set_params(list(xVar = \"Solar.R\")) pip$get_params() # $model_fit # $model_fit$xVar # [1] \"Solar.R\" # # # $model_plot # $model_plot$xVar # [1] \"Solar.R\" # # $model_plot$title # [1] \"Linear model fit\" pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: model_fit data_prep FALSE model_fit Outdated # 4: model_plot model_fit,data_prep FALSE model_plot Outdated pip$run() # INFO [2024-12-01 21:41:42.547] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:42.549] Step 1/4 data - skip 'done' step # INFO [2024-12-01 21:41:42.550] Step 2/4 data_prep - skip 'done' step # INFO [2024-12-01 21:41:42.551] Step 3/4 model_fit # INFO [2024-12-01 21:41:42.555] Step 4/4 model_plot # INFO [2024-12-01 21:41:42.564] Finished execution of steps. # INFO [2024-12-01 21:41:42.565] Done. pip$get_out(\"model_plot\") pip$set_params(list(title = \"Some new title\")) pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: model_fit data_prep FALSE model_fit Done # 4: model_plot model_fit,data_prep FALSE model_plot Outdated pip$run()$get_out(\"model_plot\") # INFO [2024-12-01 21:41:42.919] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:42.920] Step 1/4 data - skip 'done' step # INFO [2024-12-01 21:41:42.921] Step 2/4 data_prep - skip 'done' step # INFO [2024-12-01 21:41:42.922] Step 3/4 model_fit - skip 'done' step # INFO [2024-12-01 21:41:42.923] Step 4/4 model_plot # INFO [2024-12-01 21:41:42.931] Finished execution of steps. # INFO [2024-12-01 21:41:42.931] Done. pip$set_data(airquality[1:10, ]) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: data_prep data FALSE data_prep Outdated # 3: model_fit data_prep FALSE model_fit Outdated # 4: model_plot model_fit,data_prep FALSE model_plot Outdated pip$run()$get_out(\"model_plot\") # INFO [2024-12-01 21:41:43.202] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:43.203] Step 1/4 data # INFO [2024-12-01 21:41:43.206] Step 2/4 data_prep # INFO [2024-12-01 21:41:43.211] Step 3/4 model_fit # INFO [2024-12-01 21:41:43.214] Step 4/4 model_plot # INFO [2024-12-01 21:41:43.219] Finished execution of steps. # INFO [2024-12-01 21:41:43.220] Done."},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"existing-pipeline","dir":"Articles","previous_headings":"","what":"Existing pipeline","title":"Modifying existing pipelines","text":"Let’s start left Get started pipeflow vignette, , following pipeline following set data","code":"pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: model_fit data_prep FALSE model_fit Done # 4: model_plot model_fit,data_prep FALSE model_plot Done pip$get_data() |> head(3) # Ozone Solar.R Wind Temp Month Day # 1 41 190 7.4 67 5 1 # 2 36 118 8.0 72 5 2 # 3 12 149 12.6 74 5 3"},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"insert-new-step","dir":"Articles","previous_headings":"","what":"Insert new step","title":"Modifying existing pipelines","text":"Let’s say want insert new step data_prep step standardizes y-variable. can see, standardize step now part pipeline, far used step.","code":"pip$insert_after( afterStep = \"data_prep\", step = \"standardize\", function( data = ~`data_prep`, yVar = \"Ozone\" ) { data[, yVar] <- scale(data[, yVar]) data } ) pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: standardize data_prep [NULL] FALSE standardize New # 4: model_fit data_prep FALSE model_fit Done # 5: model_plot model_fit,data_prep FALSE model_plot Done library(visNetwork) do.call(visNetwork, args = pip$get_graph()) |> visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\")"},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"replace-existing-steps","dir":"Articles","previous_headings":"","what":"Replace existing steps","title":"Modifying existing pipelines","text":"Let’s revisit function definition model_fit step use standardized data, need change data dependency refers standardize step. Also instead fixed y-variable model, want pass paramter. model_plot step needs updated similar way. updated pipeline now looks follows. see model_fit model_plot steps now use standardized data. Let’s re-run pipeline inspect output.","code":"pip$get_step(\"model_fit\")[[\"fun\"]] # [[1]] # function (data = ~data_prep, xVar = \"Temp.Celsius\") # { # lm(paste(\"Ozone ~\", xVar), data = data) # } pip$replace_step( \"model_fit\", function( data = ~standardize, # <- changed data reference xVar = \"Temp.Celsius\", yVar = \"Ozone\" # <- new y-variable ) { lm(paste(yVar, \"~\", xVar), data = data) } ) pip$replace_step( \"model_plot\", function( model = ~model_fit, data = ~standardize, # <- changed data reference xVar = \"Temp.Celsius\", yVar = \"Ozone\", # <- new y-variable title = \"Linear model fit\" ) { coeffs <- coefficients(model) ggplot(data) + geom_point(aes(.data[[xVar]], .data[[yVar]])) + geom_abline(intercept = coeffs[1], slope = coeffs[2]) + labs(title = title) } ) pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: standardize data_prep [NULL] FALSE standardize New # 4: model_fit standardize [NULL] FALSE model_fit New # 5: model_plot model_fit,standardize [NULL] FALSE model_plot New pip$set_params(list(xVar = \"Solar.R\", yVar = \"Wind\")) pip$run() # INFO [2024-12-01 21:41:47.485] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:47.486] Step 1/5 data - skip 'done' step # INFO [2024-12-01 21:41:47.487] Step 2/5 data_prep - skip 'done' step # INFO [2024-12-01 21:41:47.488] Step 3/5 standardize # INFO [2024-12-01 21:41:47.490] Step 4/5 model_fit # INFO [2024-12-01 21:41:47.493] Step 5/5 model_plot # INFO [2024-12-01 21:41:47.498] Finished execution of steps. # INFO [2024-12-01 21:41:47.498] Done. pip$get_out(\"model_fit\") |> coefficients() # (Intercept) Solar.R # 0.979672739 -0.006625601 pip$get_out(\"model_plot\") # Warning: Removed 2 rows containing missing values or values outside the scale range # (`geom_point()`)."},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"removing-steps","dir":"Articles","previous_headings":"","what":"Removing steps","title":"Modifying existing pipelines","text":"Let’s see pipeline . trying remove step, pipeflow default checks step used step. , raise error. enforce removing step together downstream dependencies, can use recursive argument. Naturally, last step never downstream dependencies, can removed without issues. another way just remove last step. Replacing steps pipeline shown vignette allow re-use existing pipelines adapt programmatically new requirements. Another way re-using pipelines combine , shown Combining pipelines vignette.","code":"pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done # 3: standardize data_prep FALSE standardize Done # 4: model_fit standardize FALSE model_fit Done # 5: model_plot model_fit,standardize FALSE model_plot Done try(pip$remove_step(\"standardize\")) # Error in pip$remove_step(\"standardize\") : # cannot remove step 'standardize' because the following steps depend on it: 'model_fit', 'model_plot' pip$remove_step(\"standardize\", recursive = TRUE) # Removing step 'standardize' and its downstream dependencies: 'model_fit', 'model_plot' pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: data_prep data FALSE data_prep Done pip$pop_step() # [1] \"data_prep\" pip # step depends out keepOut group state # # 1: data FALSE data Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"changing-pipeline-parameters-at-runtime","dir":"Articles","previous_headings":"","what":"Changing pipeline parameters at runtime","title":"Advanced: How pipelines can modify themselves at runtime","text":"Let’s first define pipeline, fits linear model, checks ’s residuals normality using Shapiro-Wilk test, plots residuals. pipeline looks like can run like Now let’s imagine, want change color points plot depending Shapiro-Wilk test result. obvious way change plot step passing test result plot step function change color . However, interested another way keep plot function unchanged. example, run pipeline second time follows: mentioned another vignette, solution ideal, requires run additional code outside pipeline framework. solve issue, therefore basically set parameter within pipeline execution. , make pipeline aware , can done passing pipeline object parameter. Let’s update residual_shapiro_p_value step example. Now just make sure set .self parameter. simple “trick” opens wide range possibilities pipeline modifications runtime. show next section, limited changing parameters can also used modify pipeline structure .","code":"library(pipeflow) library(ggplot2) pip <- pipe_new(\"my-pipeline\") |> pipe_add( \"fit\", function(data = ~data, xVar = \"x\", yVar = \"y\") { lm(paste(yVar, \"~\", xVar), data = data) } ) |> pipe_add( \"residual_shapiro_p_value\", function(fit = ~fit) { residuals <- residuals(fit) p <- shapiro.test(residuals)$p.value p }, keepOut = TRUE ) |> pipe_add( \"plot\", function(fit = ~fit, pointColor = \"black\") { data <- data.frame( fitted = predict(fit), residuals = residuals(fit) ) ggplot(data, aes(x = fitted, y = residuals)) + geom_point(shape = 21, color = pointColor) + geom_hline(yintercept = 0, linetype = \"dashed\") + theme_minimal() }, keepOut = TRUE ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: fit data [NULL] FALSE fit New # 3: residual_shapiro_p_value fit [NULL] TRUE residual_shapiro_p_value New # 4: plot fit [NULL] TRUE plot New pip$set_data(airquality) pip$set_params(list(xVar = \"Ozone\", yVar = \"Temp\")) pip$run()$collect_out() # INFO [2024-12-01 21:41:51.403] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:51.430] Step 1/4 data # INFO [2024-12-01 21:41:51.444] Step 2/4 fit # INFO [2024-12-01 21:41:51.462] Step 3/4 residual_shapiro_p_value # INFO [2024-12-01 21:41:51.464] Step 4/4 plot # INFO [2024-12-01 21:41:51.481] Finished execution of steps. # INFO [2024-12-01 21:41:51.482] Done. # $residual_shapiro_p_value # [1] 0.00022598 # # $plot if (pip$get_out(\"residual_shapiro_p_value\") < 0.05) { pip$set_params(list(pointColor = \"red\")) pip$run()$collect_out() } # INFO [2024-12-01 21:41:51.876] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:51.877] Step 1/4 data - skip 'done' step # INFO [2024-12-01 21:41:51.878] Step 2/4 fit - skip 'done' step # INFO [2024-12-01 21:41:51.879] Step 3/4 residual_shapiro_p_value - skip 'done' step # INFO [2024-12-01 21:41:51.880] Step 4/4 plot # INFO [2024-12-01 21:41:51.893] Finished execution of steps. # INFO [2024-12-01 21:41:51.894] Done. # $residual_shapiro_p_value # [1] 0.00022598 # # $plot pip$replace_step( \"residual_shapiro_p_value\", function( fit = ~fit, .self = NULL ) { residuals <- residuals(fit) p <- shapiro.test(residuals)$p.value if (!is.null(.self) && p < 0.05) { .self$set_params(list(pointColor = \"red\")) } p }, keepOut = TRUE ) pip$set_data(airquality) pip$set_params(list(xVar = \"Ozone\", yVar = \"Temp\", .self = pip)) pip$run()$collect_out() # INFO [2024-12-01 21:41:52.189] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:52.190] Step 1/4 data # INFO [2024-12-01 21:41:52.192] Step 2/4 fit # INFO [2024-12-01 21:41:52.196] Step 3/4 residual_shapiro_p_value # INFO [2024-12-01 21:41:52.200] Step 4/4 plot # INFO [2024-12-01 21:41:52.207] Finished execution of steps. # INFO [2024-12-01 21:41:52.207] Done. # $residual_shapiro_p_value # [1] 0.00022598 # # $plot"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"changing-the-pipeline-structure-at-runtime","dir":"Articles","previous_headings":"","what":"Changing the pipeline structure at runtime","title":"Advanced: How pipelines can modify themselves at runtime","text":"Subsequently, pipeline steps comprised basic functions order keep examples simple. focus pipeline structure can modified runtime. pipeline just adds 1, 2, 3 input data, respectively. column table shows output step. Now let’s modify last step pipeline input greater 10, pipeline replace last step new step now instead f2 references f1 subtracts 3 input.","code":"pip <- pipe_new(\"my-pipeline\") |> pipe_add( \"f1\", function(x = ~data) { x + 1 } ) |> pipe_add( \"f2\", function(x = ~f1) { x + 2 } ) |> pipe_add( \"f3\", function(x = ~f2) { x + 3 } ) pip$set_data(1)$run() # INFO [2024-12-01 21:41:52.449] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:52.450] Step 1/4 data # INFO [2024-12-01 21:41:52.452] Step 2/4 f1 # INFO [2024-12-01 21:41:52.454] Step 3/4 f2 # INFO [2024-12-01 21:41:52.456] Step 4/4 f3 # INFO [2024-12-01 21:41:52.457] Finished execution of steps. # INFO [2024-12-01 21:41:52.458] Done. pip # step depends out keepOut group state # # 1: data 1 FALSE data Done # 2: f1 data 2 FALSE f1 Done # 3: f2 f1 4 FALSE f2 Done # 4: f3 f2 7 FALSE f3 Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"modify-a-step","dir":"Articles","previous_headings":"Changing the pipeline structure at runtime","what":"Modify a step","title":"Advanced: How pipelines can modify themselves at runtime","text":"run pipeline , nothing change. Now let’s try input 10. see output pipeline dependencies last step changed.","code":"pip <- pipe_new(\"my-pipeline\") |> pipe_add( \"f1\", function(x = ~data) { x + 1 } ) |> pipe_add( \"f2\", function(x = ~f1, .self = NULL) { if (x > 10 && !is.null(.self)) { .self$replace_step( \"f3\", function(x = ~f1) { x - 3 } ) } x + 2 } ) |> pipe_add( \"f3\", function(x = ~f2) { x + 3 } ) pip$set_params(list(.self = pip)) pip$set_data(1)$run() # INFO [2024-12-01 21:41:52.575] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:52.576] Step 1/4 data # INFO [2024-12-01 21:41:52.578] Step 2/4 f1 # INFO [2024-12-01 21:41:52.580] Step 3/4 f2 # INFO [2024-12-01 21:41:52.582] Step 4/4 f3 # INFO [2024-12-01 21:41:52.584] Finished execution of steps. # INFO [2024-12-01 21:41:52.584] Done. pip # step depends out keepOut group state # # 1: data 1 FALSE data Done # 2: f1 data 2 FALSE f1 Done # 3: f2 f1 4 FALSE f2 Done # 4: f3 f2 7 FALSE f3 Done pip$set_data(10)$run() # INFO [2024-12-01 21:41:52.643] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:52.644] Step 1/4 data # INFO [2024-12-01 21:41:52.646] Step 2/4 f1 # INFO [2024-12-01 21:41:52.649] Step 3/4 f2 # INFO [2024-12-01 21:41:52.654] Step 4/4 f3 # INFO [2024-12-01 21:41:52.656] Finished execution of steps. # INFO [2024-12-01 21:41:52.656] Done. pip # step depends out keepOut group state # # 1: data 10 FALSE data Done # 2: f1 data 11 FALSE f1 Done # 3: f2 f1 13 FALSE f2 Done # 4: f3 f1 8 FALSE f3 Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"insert-and-remove-steps","dir":"Articles","previous_headings":"Changing the pipeline structure at runtime","what":"Insert and remove steps","title":"Advanced: How pipelines can modify themselves at runtime","text":"last example, instead just replacing, go bit insert remove steps. pipeline definition follows: Basically, input greater 10, insert two new steps f1, remove f2, replace f3 new step adds 30 input. Also note return pipeline object case. important, pipeline’s run function argument recursive, default set TRUE means step returns pipeline, run current pipeline aborted returned pipeline re-run. Let’s see pipeline structure running . now let’s run input 10. log output shows abort re-run pipeline. Let’s see final structure step outputs. final structure expected new steps inserted old step removed. mentioned , just simple example show possibilities. leave user come sensible complex use cases.","code":"pip <- pipe_new( \"my-pipeline\" ) |> pipe_add( \"f1\", function(x = ~data) { x + 1 } ) |> pipe_add( \"f2\", function(x = ~f1, .self = NULL) { if (x > 10 && !is.null(.self)) { .self$insert_after( afterStep = \"f1\", step = \"f2a\", function(x = ~f1) { x + 21 } ) .self$insert_after( afterStep = \"f2a\", step = \"f2b\", function(x = ~f2a) { x + 22 } ) .self$replace_step( \"f3\", function(x = ~f2b) { x + 30 } ) .self$remove_step(\"f2\") return(.self) } x + 2 } ) |> pipe_add( \"f3\", function(x = ~f2) { x + 3 } ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: f1 data [NULL] FALSE f1 New # 3: f2 f1 [NULL] FALSE f2 New # 4: f3 f2 [NULL] FALSE f3 New pip$set_params(list(.self = pip)) pip$set_data(10)$run() # INFO [2024-12-01 21:41:52.829] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:52.830] Step 1/4 data # INFO [2024-12-01 21:41:52.832] Step 2/4 f1 # INFO [2024-12-01 21:41:52.834] Step 3/4 f2 # INFO [2024-12-01 21:41:52.848] Abort pipeline execution and restart on new. # INFO [2024-12-01 21:41:52.848] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:52.849] Step 1/5 data - skip 'done' step # INFO [2024-12-01 21:41:52.850] Step 2/5 f1 - skip 'done' step # INFO [2024-12-01 21:41:52.851] Step 3/5 f2a # INFO [2024-12-01 21:41:52.856] Step 4/5 f2b # INFO [2024-12-01 21:41:52.858] Step 5/5 f3 # INFO [2024-12-01 21:41:52.859] Finished execution of steps. # INFO [2024-12-01 21:41:52.860] Done. pip # step depends out keepOut group state # # 1: data 10 FALSE data Done # 2: f1 data 11 FALSE f1 Done # 3: f2a f1 32 FALSE f2a Done # 4: f2b f2a 54 FALSE f2b Done # 5: f3 f2b 84 FALSE f3 Done"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"motivation","dir":"Articles","previous_headings":"","what":"Motivation","title":"Advanced: How to use pipeflow with split data sets","text":"common scenario split data set subsets apply analysis part. context pipelines, means like apply pipeline multiple times data subset. additon, may want combine parts individual output. see, pipeflow provides built-function handle scenario.","code":""},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"define-pipeline","dir":"Articles","previous_headings":"","what":"Define pipeline","title":"Advanced: How to use pipeflow with split data sets","text":"Let’s first define pipeline, , keep matters simple, just fits linear model outputs model coefficients. pipeline looks like : graphically: use iris data set working example. First, apply pipeline whole data set.","code":"library(pipeflow) pip <- pipe_new( \"my-pipeline\" ) |> pipe_add( \"fit\", function( data = ~data, xVar = \"x\", yVar = \"y\" ) { lm(paste(yVar, \"~\", xVar), data = data) } ) |> pipe_add( \"coefs\", function( fit = ~fit ) { coefficients(fit) }, keepOut = TRUE ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: fit data [NULL] FALSE fit New # 3: coefs fit [NULL] TRUE coefs New library(visNetwork) do.call(visNetwork, args = c(pip$get_graph(), list(height = 100))) |> visHierarchicalLayout(direction = \"LR\") head(iris) # Sepal.Length Sepal.Width Petal.Length Petal.Width Species # 1 5.1 3.5 1.4 0.2 setosa # 2 4.9 3.0 1.4 0.2 setosa # 3 4.7 3.2 1.3 0.2 setosa # 4 4.6 3.1 1.5 0.2 setosa # 5 5.0 3.6 1.4 0.2 setosa # 6 5.4 3.9 1.7 0.4 setosa pip$set_data(iris) pip$set_params(list(xVar = \"Sepal.Length\", yVar = \"Sepal.Width\")) pip$run() # INFO [2024-12-01 21:41:55.998] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:56.024] Step 1/3 data # INFO [2024-12-01 21:41:56.035] Step 2/3 fit # INFO [2024-12-01 21:41:56.055] Step 3/3 coefs # INFO [2024-12-01 21:41:56.056] Finished execution of steps. # INFO [2024-12-01 21:41:56.056] Done. pip$collect_out() # $coefs # (Intercept) Sepal.Length # 3.4189468 -0.0618848"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"split-data-set","dir":"Articles","previous_headings":"","what":"Split data set","title":"Advanced: How to use pipeflow with split data sets","text":"Next, want apply pipeline species separately. One way use R’s split function. can split Species column run pipeline subset. example: Unfortunately, approach create additional code run outside pipeline framework. Let’s now see handle scenario within pipeline framework. reminder, pipeline looks like : deal split data sets, use built-function set_data_split. function actually transforms pipeline: can see, pipeline now replicated data subset. Note set_data_split accepts list data frames, just output split. Now let’s run pipeline. can see, output now obtained without need write additional code outside pipeline framework. Finally, side note another built-function named split, can used split pipeline independent parts. works pipeline, working example, naturally split pipeline parts defined data split. function especially useful want separate parts pipeline code order run parallel.","code":"run_pipeline <- function(data) { pip$set_data(data) pip$run() pip$collect_out() } results <- lapply(split(iris, iris$Species), FUN = run_pipeline) # INFO [2024-12-01 21:41:56.192] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:56.193] Step 1/3 data # INFO [2024-12-01 21:41:56.195] Step 2/3 fit # INFO [2024-12-01 21:41:56.199] Step 3/3 coefs # INFO [2024-12-01 21:41:56.203] Finished execution of steps. # INFO [2024-12-01 21:41:56.204] Done. # INFO [2024-12-01 21:41:56.222] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:56.223] Step 1/3 data # INFO [2024-12-01 21:41:56.225] Step 2/3 fit # INFO [2024-12-01 21:41:56.227] Step 3/3 coefs # INFO [2024-12-01 21:41:56.228] Finished execution of steps. # INFO [2024-12-01 21:41:56.229] Done. # INFO [2024-12-01 21:41:56.232] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:56.233] Step 1/3 data # INFO [2024-12-01 21:41:56.235] Step 2/3 fit # INFO [2024-12-01 21:41:56.238] Step 3/3 coefs # INFO [2024-12-01 21:41:56.239] Finished execution of steps. # INFO [2024-12-01 21:41:56.239] Done. results # $setosa # $setosa$coefs # (Intercept) Sepal.Length # -0.5694327 0.7985283 # # # $versicolor # $versicolor$coefs # (Intercept) Sepal.Length # 0.8721460 0.3197193 # # # $virginica # $virginica$coefs # (Intercept) Sepal.Length # 1.4463054 0.2318905 pip # step depends out keepOut group state # # 1: data FALSE data Done # 2: fit data FALSE fit Done # 3: coefs fit 1.4463054,0.2318905 TRUE coefs Done splitData <- split(iris, iris$Species) pip$set_data_split(splitData) pip # step depends out keepOut group state # # 1: data.setosa [NULL] FALSE setosa New # 2: fit.setosa data.setosa FALSE setosa Outdated # 3: coefs.setosa fit.setosa 1.4463054,0.2318905 TRUE setosa Outdated # 4: data.versicolor [NULL] FALSE versicolor New # 5: fit.versicolor data.versicolor FALSE versicolor Outdated # 6: coefs.versicolor fit.versicolor 1.4463054,0.2318905 TRUE versicolor Outdated # 7: data.virginica [NULL] FALSE virginica New # 8: fit.virginica data.virginica FALSE virginica Outdated # 9: coefs.virginica fit.virginica 1.4463054,0.2318905 TRUE virginica Outdated do.call(visNetwork, args = pip$get_graph()) |> visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\") pip$run() # INFO [2024-12-01 21:41:56.710] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:56.711] Step 1/9 data.setosa # INFO [2024-12-01 21:41:56.713] Step 2/9 fit.setosa # INFO [2024-12-01 21:41:56.716] Step 3/9 coefs.setosa # INFO [2024-12-01 21:41:56.717] Step 4/9 data.versicolor # INFO [2024-12-01 21:41:56.719] Step 5/9 fit.versicolor # INFO [2024-12-01 21:41:56.722] Step 6/9 coefs.versicolor # INFO [2024-12-01 21:41:56.723] Step 7/9 data.virginica # INFO [2024-12-01 21:41:56.725] Step 8/9 fit.virginica # INFO [2024-12-01 21:41:56.728] Step 9/9 coefs.virginica # INFO [2024-12-01 21:41:56.729] Finished execution of steps. # INFO [2024-12-01 21:41:56.730] Done. pip$collect_out() # $setosa # (Intercept) Sepal.Length # -0.5694327 0.7985283 # # $versicolor # (Intercept) Sepal.Length # 0.8721460 0.3197193 # # $virginica # (Intercept) Sepal.Length # 1.4463054 0.2318905 pip$split() # [[1]] # step depends out keepOut group state # # 1: data.setosa FALSE setosa Done # 2: fit.setosa data.setosa FALSE setosa Done # 3: coefs.setosa fit.setosa -0.5694327, 0.7985283 TRUE setosa Done # # [[2]] # step depends out keepOut group state # # 1: data.versicolor FALSE versicolor Done # 2: fit.versicolor data.versicolor FALSE versicolor Done # 3: coefs.versicolor fit.versicolor 0.8721460,0.3197193 TRUE versicolor Done # # [[3]] # step depends out keepOut group state # # 1: data.virginica FALSE virginica Done # 2: fit.virginica data.virginica FALSE virginica Done # 3: coefs.virginica fit.virginica 1.4463054,0.2318905 TRUE virginica Done"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"combine-output","dir":"Articles","previous_headings":"","what":"Combine output","title":"Advanced: How to use pipeflow with split data sets","text":"cases, may want (re-)combine output pipeline parts. example, may want combine coefficients linear models one table. Let’s matter extend pipeline one step end. pipeline looks now. Next want apply set_data_split function, basically need make sure pipeline split except last step combines everything. achieve using toStep parameter, basically tells pipeline split certain step. see last step replicated data subset now contains list dependencies, namely: Graphically becomes even clear: Finally, let’s see plays run pipeline.","code":"pip <- pipe_new( \"my-pipeline\" ) |> pipe_add( \"fit\", function( data = ~data, xVar = \"x\", yVar = \"y\" ) { lm(paste(yVar, \"~\", xVar), data = data) } ) |> pipe_add( \"coefs\", function( fit = ~fit ) { coefficients(fit) } ) |> pipe_add( \"combine_coefs\", function( coefs = ~coefs ) { coefs |> do.call(rbind, args = _) |> as.data.frame() }, keepOut = TRUE ) pip # step depends out keepOut group state # # 1: data [NULL] FALSE data New # 2: fit data [NULL] FALSE fit New # 3: coefs fit [NULL] FALSE coefs New # 4: combine_coefs coefs [NULL] TRUE combine_coefs New pip$set_data_split(split(iris, iris$Species), toStep = \"coefs\") pip # step depends out keepOut group state # # 1: data.setosa [NULL] FALSE setosa New # 2: fit.setosa data.setosa [NULL] FALSE setosa Outdated # 3: coefs.setosa fit.setosa [NULL] FALSE setosa Outdated # 4: data.versicolor [NULL] FALSE versicolor New # 5: fit.versicolor data.versicolor [NULL] FALSE versicolor Outdated # 6: coefs.versicolor fit.versicolor [NULL] FALSE versicolor Outdated # 7: data.virginica [NULL] FALSE virginica New # 8: fit.virginica data.virginica [NULL] FALSE virginica Outdated # 9: coefs.virginica fit.virginica [NULL] FALSE virginica Outdated # 10: combine_coefs [NULL] TRUE combine_coefs New pip$get_depends()[[\"combine_coefs\"]] # $coefs # [1] \"coefs.setosa\" \"coefs.versicolor\" \"coefs.virginica\" do.call(visNetwork, args = pip$get_graph()) |> visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\") pip$set_params(list(xVar = \"Sepal.Length\", yVar = \"Sepal.Width\")) pip$run() # INFO [2024-12-01 21:41:57.260] Start run of 'my-pipeline' pipeline: # INFO [2024-12-01 21:41:57.263] Step 1/10 data.setosa # INFO [2024-12-01 21:41:57.265] Step 2/10 fit.setosa # INFO [2024-12-01 21:41:57.268] Step 3/10 coefs.setosa # INFO [2024-12-01 21:41:57.270] Step 4/10 data.versicolor # INFO [2024-12-01 21:41:57.272] Step 5/10 fit.versicolor # INFO [2024-12-01 21:41:57.275] Step 6/10 coefs.versicolor # INFO [2024-12-01 21:41:57.277] Step 7/10 data.virginica # INFO [2024-12-01 21:41:57.279] Step 8/10 fit.virginica # INFO [2024-12-01 21:41:57.282] Step 9/10 coefs.virginica # INFO [2024-12-01 21:41:57.284] Step 10/10 combine_coefs # INFO [2024-12-01 21:41:57.286] Finished execution of steps. # INFO [2024-12-01 21:41:57.286] Done. pip$collect_out() # $combine_coefs # (Intercept) Sepal.Length # coefs.setosa -0.5694327 0.7985283 # coefs.versicolor 0.8721460 0.3197193 # coefs.virginica 1.4463054 0.2318905"},{"path":"https://github.com/rpahl/pipeflow/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Roman Pahl. Author, maintainer.","code":""},{"path":"https://github.com/rpahl/pipeflow/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Pahl R (2024). pipeflow: Implement Data Analysis Workflows Pipelines. R package version 0.1.1, https://github.com/rpahl/pipeflow, https://rpahl.github.io/pipeflow.","code":"@Manual{, title = {pipeflow: Implement Data Analysis Workflows with Pipelines}, author = {Roman Pahl}, year = {2024}, note = {R package version 0.1.1, https://github.com/rpahl/pipeflow}, url = {https://rpahl.github.io/pipeflow}, }"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"pipeflow-","dir":"","previous_headings":"","what":"Implement Data Analysis Workflows with Pipelines","title":"Implement Data Analysis Workflows with Pipelines","text":"pipeflow intuitive yet powerful framework building running data processing pipelines. Whether ’re working scientific computing, machine learning (AI), statistical reporting, pipeflow adapts seamlessly scenario—small exploratory analyses complex production workflows. Since pipeflow pipelines essentially sequences R functions, ’re easy learn can used interactively typical R users. Advanced features like dynamic branching self-modifying pipelines provide additional flexibility complex workflows.","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"why-use-pipeflow","dir":"","previous_headings":"","what":"Why use pipeflow","title":"Implement Data Analysis Workflows with Pipelines","text":"Easy learn yet powerful complex workflows Automatically manages function dependencies Promotes standardized, reproducible analysis Simplifies error handling, debugging, reusability","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"key-features","dir":"","previous_headings":"","what":"Key features","title":"Implement Data Analysis Workflows with Pipelines","text":"Flexible Application: Use interactively programmatically R Dependency Management: Dependencies checked definition, ensuring reliable workflows Comprehensive Logging: Logs step, customizable logger options Parameter Control: Easily view adjust function parameters one place Modular Composition: Modify, extend, combine pipelines effortlessly Intelligent Execution: Skip steps already --date, similar make Visualization: View pipelines tabular graphical formats","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"advanced-features","dir":"","previous_headings":"","what":"Advanced features","title":"Implement Data Analysis Workflows with Pipelines","text":"Dynamic Branching: Apply pipeline multiple datasets seamlessly Self-Modifying: Pipelines can adapt modify runtime","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Implement Data Analysis Workflows with Pipelines","text":"","code":"# Install release version from CRAN install.packages(\"pipeflow\") # Install development version from GitHub devtools::install_github(\"rpahl/pipeflow\")"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"usage","dir":"","previous_headings":"","what":"Usage","title":"Implement Data Analysis Workflows with Pipelines","text":"","code":"library(pipeflow)"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"getting-started","dir":"","previous_headings":"","what":"Getting Started","title":"Implement Data Analysis Workflows with Pipelines","text":"recommended read vignettes order listed : Get started pipeflow Modifying existing pipelines Combining pipelines Collecting output","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"advanced-topics","dir":"","previous_headings":"","what":"Advanced topics","title":"Implement Data Analysis Workflows with Pipelines","text":"use pipeflow split data sets pipelines can modify runtime","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":null,"dir":"Reference","previous_headings":"","what":"Pipeline Class — Pipeline","title":"Pipeline Class — Pipeline","text":"class implements analysis pipeline. pipeline consists sequence analysis steps, can added one one. added step may may depend one previous steps. pipeline keeps track dependencies among steps ensure dependencies met creation pipeline, , pipeline run. pipeline run, output stored pipeline along step can accessed later. Different pipelines can bound together preserving dependencies within pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Pipeline Class — Pipeline","text":"Roman Pahl","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"public-fields","dir":"Reference","previous_headings":"","what":"Public fields","title":"Pipeline Class — Pipeline","text":"name string name pipeline pipeline data.table pipeline row represents one step.","code":""},{"path":[]},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"public-methods","dir":"Reference","previous_headings":"","what":"Public methods","title":"Pipeline Class — Pipeline","text":"Pipeline$new() Pipeline$add() Pipeline$append() Pipeline$append_to_step_names() Pipeline$collect_out() Pipeline$discard_steps() Pipeline$get_data() Pipeline$get_depends() Pipeline$get_depends_down() Pipeline$get_depends_up() Pipeline$get_graph() Pipeline$get_out() Pipeline$get_params() Pipeline$get_params_at_step() Pipeline$get_params_unique() Pipeline$get_params_unique_json() Pipeline$get_step() Pipeline$get_step_names() Pipeline$get_step_number() Pipeline$has_step() Pipeline$insert_after() Pipeline$insert_before() Pipeline$length() Pipeline$lock_step() Pipeline$print() Pipeline$pop_step() Pipeline$pop_steps_after() Pipeline$pop_steps_from() Pipeline$remove_step() Pipeline$rename_step() Pipeline$replace_step() Pipeline$reset() Pipeline$run() Pipeline$run_step() Pipeline$set_data() Pipeline$set_data_split() Pipeline$set_keep_out() Pipeline$set_params() Pipeline$set_params_at_step() Pipeline$split() Pipeline$unlock_step() Pipeline$clone()","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-new-","dir":"Reference","previous_headings":"","what":"Method new()","title":"Pipeline Class — Pipeline","text":"constructor","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$new(name, data = NULL, logger = NULL)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"name name Pipeline data optional data used start pipeline. data also can set later using set_data function. logger custom logger used logging. logger provided, default logger used, sufficient use cases. want use custom log function, need provide function obeys following form: function(level, msg, ...) { custom logging code } level argument string one info, warn, error. msg argument string containing message logged. ... argument list named parameters, can used add additional information log message. Currently, used add context case step giving warning error. Note default logger, log layout can altered time via set_log_layout().","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:8)) p # Passing custom logger my_logger <- function(level, msg, ...) { cat(level, msg, \"\\n\") } p <- Pipeline$new(\"myPipe\", logger = my_logger)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-add-","dir":"Reference","previous_headings":"","what":"Method add()","title":"Pipeline Class — Pipeline","text":"Add pipeline step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-1","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$add( step, fun, params = list(), description = \"\", group = step, keepOut = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-1","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step. step name must unique. fun function name function applied step. existing lambda/anonymous functions can used. params list list parameters set overwrite parameters passed function. description string optional description step group string output collected pipeline execution (see function collect_out) grouped defined group names. default, name step, comes handy pipeline copy-appended multiple times keep results function/step grouped one place. keepOut logical FALSE (default) output step collected calling collect_out pipeline run. option used keep results matter skip intermediate results needed. See also function collect_out details.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-1","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-1","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Add steps with lambda functions p <- Pipeline$new(\"myPipe\", data = 1) p$add(\"s1\", \\(x = ~data) 2*x) # use input data p$add(\"s2\", \\(x = ~data, y = ~s1) x * y) try(p$add(\"s2\", \\(z = 3) 3)) # error: step 's2' exists already try(p$add(\"s3\", \\(z = ~foo) 3)) # dependency 'foo' not found p p <- Pipeline$new(\"myPipe\", data = c(1, 2, NA, 3, 4)) p$add(\"calc_mean\", mean, params = list(x = ~data, na.rm = TRUE)) p$run()$get_out(\"calc_mean\") p <- Pipeline$new(\"myPipe\", data = 1:10) p$add(\"s1\", \\(x = ~data) 2*x, description = \"multiply by 2\") print(p) print(p, verbose = TRUE) # print all columns p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:5, y = 1:5)) p$add(\"prep_x\", \\(data = ~data) data$x, group = \"prep\") p$add(\"prep_y\", \\(data = ~data) (data$y)^2, group = \"prep\") p$add(\"sum\", \\(x = ~prep_x, y = ~prep_y) x + y) p$run()$collect_out(all = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-append-","dir":"Reference","previous_headings":"","what":"Method append()","title":"Pipeline Class — Pipeline","text":"Append another pipeline. append takes care name clashes dependencies, changed append.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-2","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$append(p, outAsIn = FALSE, tryAutofixNames = TRUE, sep = \".\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-2","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"p Pipeline object appended. outAsIn logical TRUE, output first pipeline used input second pipeline. tryAutofixNames logical TRUE, name clashes tried automatically resolved appending 2nd pipeline's name. sep string separator used auto-resolving step names","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-2","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns new combined Pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-2","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Append pipeline p1 <- Pipeline$new(\"pipe1\") p1$add(\"step1\", \\(x = 1) x) p2 <- Pipeline$new(\"pipe2\") p2$add(\"step2\", \\(y = 1) y) p1$append(p2) p3 <- Pipeline$new(\"pipe3\") p3$add(\"step1\", \\(z = 1) z) p1$append(p2)$append(p3)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-append-to-step-names-","dir":"Reference","previous_headings":"","what":"Method append_to_step_names()","title":"Pipeline Class — Pipeline","text":"Append string step names. Also takes care updating dependencies accordingly.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-3","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$append_to_step_names(postfix, sep = \".\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-3","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"postfix string appended step name. sep string separator step name postfix.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-3","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-3","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\") p$add(\"step1\", \\(x = 1) x) p$add(\"step2\", \\(y = 1) y) p$append_to_step_names(\"new\") p p$append_to_step_names(\"new\", sep = \"_\") p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-collect-out-","dir":"Reference","previous_headings":"","what":"Method collect_out()","title":"Pipeline Class — Pipeline","text":"Collect output afer pipeline run, default, steps keepOut set TRUE. output grouped group names (see group parameter function add) set explicitly corresponds step names.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-4","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$collect_out(groupBy = \"group\", all = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-4","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"groupBy string column pipeline group output. logical TRUE output collected regardless keepOut flag. can useful debugging.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-4","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list containing output, named groups, , default, steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-4","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2) p$add(\"step2\", \\(x = ~step1) x + 2, keepOut = TRUE) p$run() p$collect_out() p$collect_out(all = TRUE) |> str() p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2, group = \"add\") p$add(\"step2\", \\(x = ~step1, y = 2) x + y, group = \"add\") p$add(\"step3\", \\(x = ~data) x * 3, group = \"mult\") p$add(\"step4\", \\(x = ~data, y = 2) x * y, group = \"mult\") p p$run() p$collect_out(all = TRUE) |> str() # Grouped by state p$set_params(list(y = 5)) p p$collect_out(groupBy = \"state\", all = TRUE) |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-discard-steps-","dir":"Reference","previous_headings":"","what":"Method discard_steps()","title":"Pipeline Class — Pipeline","text":"Discard steps match given pattern.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-5","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$discard_steps(pattern, recursive = FALSE, fixed = TRUE, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-5","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"pattern string containing regular expression (character string fixed = TRUE) matched. recursive logical TRUE step removed together downstream dependencies. fixed logical TRUE, pattern string matched . Overrides conflicting arguments. ... arguments passed grep().","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-5","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-5","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~add1) x + 2) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$discard_steps(\"mult\") p # Re-add steps p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p # Discard step 'add1' does'nt work as 'add2' and 'mult3' depend on it try(p$discard_steps(\"add1\")) p$discard_steps(\"add1\", recursive = TRUE) # this works p # Trying to discard non-existent steps is just ignored p$discard_steps(\"non-existent\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-data-","dir":"Reference","previous_headings":"","what":"Method get_data()","title":"Pipeline Class — Pipeline","text":"Get data","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-6","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_data()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-6","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"output defined data step, default first step pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-6","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$get_data() p$set_data(3:4) p$get_data()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-","dir":"Reference","previous_headings":"","what":"Method get_depends()","title":"Pipeline Class — Pipeline","text":"Get dependencies defined pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-7","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-7","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"named list dependencies step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-7","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$get_depends()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-down-","dir":"Reference","previous_headings":"","what":"Method get_depends_down()","title":"Pipeline Class — Pipeline","text":"Get downstream dependencies given step, default descending recursively.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-8","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends_down(step, recursive = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-6","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step recursive logical TRUE, dependencies dependencies also returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-8","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list downstream dependencies","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-8","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_down(\"add1\") p$get_depends_down(\"add1\", recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-up-","dir":"Reference","previous_headings":"","what":"Method get_depends_up()","title":"Pipeline Class — Pipeline","text":"Get upstream dependencies given step, default descending recursively.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-9","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends_up(step, recursive = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-7","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step recursive logical TRUE, dependencies dependencies also returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-9","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list upstream dependencies","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-9","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_up(\"mult4\") p$get_depends_up(\"mult4\", recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-graph-","dir":"Reference","previous_headings":"","what":"Method get_graph()","title":"Pipeline Class — Pipeline","text":"Visualize pipeline graph.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-10","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_graph(groups = NULL)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-8","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"groups character NULL, steps belonging given groups considered.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-10","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"two data frames, one nodes one edges ready used visNetwork package.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-10","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = ~add1, y = ~add2) x * y) if (require(\"visNetwork\", quietly = TRUE)) { do.call(visNetwork, args = p$get_graph()) }"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-out-","dir":"Reference","previous_headings":"","what":"Method get_out()","title":"Pipeline Class — Pipeline","text":"Get output given step pipeline run.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-11","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_out(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-9","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-11","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"output given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-11","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$run() p$get_out(\"add1\") p$get_out(\"add2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-","dir":"Reference","previous_headings":"","what":"Method get_params()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) function parameters defined pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-12","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-10","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-12","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list parameters, sorted named step. Steps parameters filtered .","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-12","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params() |> str() p$get_params(ignoreHidden = FALSE) |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-at-step-","dir":"Reference","previous_headings":"","what":"Method get_params_at_step()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) given step name.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-13","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_at_step(step, ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-11","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-13","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list parameters defined given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-13","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params_at_step(\"add2\") p$get_params_at_step(\"add2\", ignoreHidden = FALSE) p$get_params_at_step(\"add3\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-unique-","dir":"Reference","previous_headings":"","what":"Method get_params_unique()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) parameters defined pipeline, list parameter . values parameters, values first step parameter defined. particularly useful parameters set using set_params function, set value steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-14","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_unique(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-12","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-14","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list unique parameters","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-14","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique() p$get_params_unique(ignoreHidden = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-unique-json-","dir":"Reference","previous_headings":"","what":"Method get_params_unique_json()","title":"Pipeline Class — Pipeline","text":"Get unique function parameters json format.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-15","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_unique_json(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-13","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-15","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list flat unnamed json list unique function parameters","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-15","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique_json() p$get_params_unique_json(ignoreHidden = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-","dir":"Reference","previous_headings":"","what":"Method get_step()","title":"Pipeline Class — Pipeline","text":"Get step pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-16","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-14","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-16","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"data.table row containing step. step found, error given.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-16","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, z = ~add1) x + y + z) p$run() add1 <- p$get_step(\"add1\") print(add1) add1[[\"params\"]] add1[[\"out\"]] try() try(p$get_step(\"foo\")) # error: step 'foo' does not exist"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-names-","dir":"Reference","previous_headings":"","what":"Method get_step_names()","title":"Pipeline Class — Pipeline","text":"Get step names pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-17","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step_names()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-17","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector step names","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-17","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_names()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-number-","dir":"Reference","previous_headings":"","what":"Method get_step_number()","title":"Pipeline Class — Pipeline","text":"Get step number","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-18","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step_number(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-15","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-18","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"step number pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-18","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_number(\"f2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-has-step-","dir":"Reference","previous_headings":"","what":"Method has_step()","title":"Pipeline Class — Pipeline","text":"Determine whether pipeline given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-19","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$has_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-16","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-19","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"logical whether step exists","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-19","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$has_step(\"f2\") p$has_step(\"foo\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-insert-after-","dir":"Reference","previous_headings":"","what":"Method insert_after()","title":"Pipeline Class — Pipeline","text":"Insert step certain step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-20","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$insert_after(afterStep, step, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-17","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"afterStep string name step insert step string name step insert ... arguments passed add method pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-20","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-20","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_after(\"f1\", \"f3\", \\(x = ~f1) x) p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-insert-before-","dir":"Reference","previous_headings":"","what":"Method insert_before()","title":"Pipeline Class — Pipeline","text":"Insert step certain step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-21","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$insert_before(beforeStep, step, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-18","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"beforeStep string name step insert step string name step insert ... arguments passed add method pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-21","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-21","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_before(\"f2\", \"f3\", \\(x = ~f1) x) p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-length-","dir":"Reference","previous_headings":"","what":"Method length()","title":"Pipeline Class — Pipeline","text":"Length pipeline aka number pipeline steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-22","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$length()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-22","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"numeric length pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-22","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$length()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-lock-step-","dir":"Reference","previous_headings":"","what":"Method lock_step()","title":"Pipeline Class — Pipeline","text":"Locking step means parameters output (given output) locked. output, parameters locked. Locking step useful step happens share parameter names steps affected parameters set commonly entire pipeline (see function set_params ).","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-23","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$lock_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-19","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-23","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-23","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$run() p$get_out(\"add1\") p$get_out(\"add2\") p$lock_step(\"add1\") p$set_data(3) p$set_params(list(x = 3)) p$run() p$get_out(\"add1\") p$get_out(\"add2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-print-","dir":"Reference","previous_headings":"","what":"Method print()","title":"Pipeline Class — Pipeline","text":"Print pipeline table.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-24","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$print(verbose = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-20","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"verbose logical TRUE, print columns pipeline, otherwise subset columns printed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-24","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-24","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$print()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-step-","dir":"Reference","previous_headings":"","what":"Method pop_step()","title":"Pipeline Class — Pipeline","text":"Remove last step pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-25","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_step()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-25","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"string name step removed","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-25","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p p$pop_step() # \"f2\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-steps-after-","dir":"Reference","previous_headings":"","what":"Method pop_steps_after()","title":"Pipeline Class — Pipeline","text":"Remove steps given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-26","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_steps_after(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-21","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-26","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector steps removed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-26","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_after(\"f1\") # \"f2\", \"f3\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-steps-from-","dir":"Reference","previous_headings":"","what":"Method pop_steps_from()","title":"Pipeline Class — Pipeline","text":"Remove steps including given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-27","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_steps_from(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-22","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-27","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector steps removed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-27","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_from(\"f2\") # \"f2\", \"f3\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-remove-step-","dir":"Reference","previous_headings":"","what":"Method remove_step()","title":"Pipeline Class — Pipeline","text":"Remove certain step pipeline. step exist, error given. steps depend step removed, error given, unless recursive = TRUE.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-28","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$remove_step(step, recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-23","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step removed. recursive logical TRUE step removed together downstream dependencies.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-28","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-28","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = 1, y = ~add2) x * y) p$remove_step(\"mult1\") p try(p$remove_step(\"add1\")) # fails because \"add2\" depends on \"add1\" p$remove_step(\"add1\", recursive = TRUE) # removes \"add1\" and \"add2\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-rename-step-","dir":"Reference","previous_headings":"","what":"Method rename_step()","title":"Pipeline Class — Pipeline","text":"Safely rename step pipeline. new step name result name clash, error given.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-29","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$rename_step(from, to)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-24","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"string name step renamed. string new name step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-29","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-29","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p try(p$rename_step(\"add1\", \"add2\")) # fails because \"add2\" exists p$rename_step(\"add1\", \"first_add\") # Ok p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-replace-step-","dir":"Reference","previous_headings":"","what":"Method replace_step()","title":"Pipeline Class — Pipeline","text":"Replace pipeline step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-30","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$replace_step( step, fun, params = list(), description = \"\", group = step, keepOut = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-25","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step replaced. Step must exist. fun string function operation applied step. existing lambda/anonymous functions can used. params list list parameters overwrite default parameters existing functions. description string optional description step group string grouping information (default name step. output collected later (see function collect_out default put together group names. , example, comes handy pipeline copy-appended multiple times keep results function/step one place. keepOut logical FALSE output function cleaned end whole pipeline execution. option used keep results matter.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-30","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-30","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = 1, y = 2) x * y, keepOut = TRUE) p$run()$collect_out() p$replace_step(\"mult\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() try(p$replace_step(\"foo\", \\(x = 1) x)) # step 'foo' does not exist"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-reset-","dir":"Reference","previous_headings":"","what":"Method reset()","title":"Pipeline Class — Pipeline","text":"Resets pipeline state run. means output removed state steps reset 'New'.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-31","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$reset()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-31","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-31","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$run() p p$reset() p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-run-","dir":"Reference","previous_headings":"","what":"Method run()","title":"Pipeline Class — Pipeline","text":"Run new /outdated pipeline steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-32","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$run( force = FALSE, recursive = TRUE, cleanUnkept = FALSE, progress = NULL, showLog = TRUE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-26","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"force logical TRUE steps run regardless whether outdated . recursive logical TRUE step returns new pipeline, run current pipeline aborted new pipeline run recursively. cleanUnkept logical TRUE output marked kept removed pipeline run. option can useful temporary results require lot memory. progress function parameter can used provide custom progress function form function(value, detail), show progress pipeline run step, value current step number detail name step. showLog logical steps logged pipeline run?","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-32","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-32","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Simple pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"final\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() p$set_params(list(z = 4)) # outdates steps add2 and final p p$run()$collect_out() p$run(cleanUnkept = TRUE) # clean up temporary results p # Recursive pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"new_pipe\", \\(x = ~add1) { pp <- Pipeline$new(\"new_pipe\", data = x) pp$add(\"add1\", \\(x = ~data) x + 1) pp$add(\"add2\", \\(x = ~add1) x + 2, keepOut = TRUE) } ) p$run()$collect_out() # Run pipeline with progress bar p <- Pipeline$new(\"pipe\", data = 1) p$add(\"first step\", \\() Sys.sleep(1)) p$add(\"second step\", \\() Sys.sleep(1)) p$add(\"last step\", \\() Sys.sleep(1)) pb <- txtProgressBar(min = 1, max = p$length(), style = 3) fprogress <- function(value, detail) { setTxtProgressBar(pb, value) } p$run(progress = fprogress, showLog = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-run-step-","dir":"Reference","previous_headings":"","what":"Method run_step()","title":"Pipeline Class — Pipeline","text":"Run given pipeline step possibly together upstream downstream dependencies.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-33","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$run_step( step, upstream = TRUE, downstream = FALSE, cleanUnkept = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-27","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step upstream logical TRUE, run dependent upstream steps first. downstream logical TRUE, run depdendent downstream afterwards. cleanUnkept logical TRUE output marked kept removed pipeline run. option can useful temporary results require lot memory.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-33","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-33","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run_step(\"add2\") p$run_step(\"add2\", downstream = TRUE) p$run_step(\"mult\", upstream = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-data-","dir":"Reference","previous_headings":"","what":"Method set_data()","title":"Pipeline Class — Pipeline","text":"Set data first step pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-34","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_data(data)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-28","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"data data.frame initial data set.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-34","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-34","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$run()$collect_out() p$set_data(3) p$run()$collect_out()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-data-split-","dir":"Reference","previous_headings":"","what":"Method set_data_split()","title":"Pipeline Class — Pipeline","text":"Split-copy pipeline list data sets. sub-pipeline one data sets set input data. step names sub-pipelines original step names plus name data set.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-35","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_data_split( dataList, toStep = utils::tail(self$get_step_names(), 1), groupBySplit = TRUE, sep = \".\" )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-29","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"dataList list data sets toStep string step name marking optional subset pipeline, data split applied . groupBySplit logical whether set step groups according data split. sep string separator used step name data set name creating new step names.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-35","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"new combined Pipeline sub-pipeline set one data sets.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-35","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList) p3 p3$run()$collect_out() |> str() # Don't group output by split p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList, groupBySplit = FALSE) p3 p3$run()$collect_out() |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-keep-out-","dir":"Reference","previous_headings":"","what":"Method set_keep_out()","title":"Pipeline Class — Pipeline","text":"Change keepOut flag given pipeline step, determines whether output step collected calling collect_out() pipeline run.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-36","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_keep_out(step, keepOut = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-30","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step keepOut logical whether keep output step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-36","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-36","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run()$collect_out() p$set_keep_out(\"add1\", keepOut = FALSE) p$set_keep_out(\"mult\", keepOut = TRUE) p$collect_out()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-params-","dir":"Reference","previous_headings":"","what":"Method set_params()","title":"Pipeline Class — Pipeline","text":"Set parameters pipeline. parameter occurs several steps, parameter set commonly steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-37","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_params(params, warnUndefined = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-31","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"params list parameters set warnUndefined logical whether give warning parameter defined pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-37","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-37","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 1) x + y) p$add(\"mult\", \\(x = 1, z = 1) x * z) p$get_params() p$set_params(list(x = 3, y = 3)) p$get_params() p$set_params(list(x = 5, z = 3)) p$get_params() suppressWarnings( p$set_params(list(foo = 3)) # warning: trying to set undefined )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-params-at-step-","dir":"Reference","previous_headings":"","what":"Method set_params_at_step()","title":"Pipeline Class — Pipeline","text":"Set unbound parameter values given pipeline step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-38","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_params_at_step(step, params)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-32","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step params list parameters set","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-38","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-38","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1, z = 2) x + y) p$add(\"add2\", \\(x = ~data, y = 1, z = 2) x + y) p$set_params_at_step(\"add1\", list(y = 3, z = 3)) p$get_params() try(p$set_params_at_step(\"add1\", list(foo = 3))) # foo not defined"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-split-","dir":"Reference","previous_headings":"","what":"Method split()","title":"Pipeline Class — Pipeline","text":"Splits pipeline independent parts.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-39","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$split()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-39","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list Pipeline objects","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-39","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Example for two independent calculation paths p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = ~data) x) p$add(\"f2\", \\(x = 1) x) p$add(\"f3\", \\(x = ~f1) x) p$add(\"f4\", \\(x = ~f2) x) p$split() # Example of split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) pips <- p$set_data_split(dataList)$split()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-unlock-step-","dir":"Reference","previous_headings":"","what":"Method unlock_step()","title":"Pipeline Class — Pipeline","text":"Unlock previously locked step. step locked, command ignored.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-40","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$unlock_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-33","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-40","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-40","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$lock_step(\"add1\") p$set_params(list(x = 3)) p$get_params() p$unlock_step(\"add1\") p$set_params(list(x = 3)) p$get_params()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-clone-","dir":"Reference","previous_headings":"","what":"Method clone()","title":"Pipeline Class — Pipeline","text":"objects class cloneable method.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-41","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$clone(deep = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-34","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"deep Whether make deep clone.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"## ------------------------------------------------ ## Method `Pipeline$new` ## ------------------------------------------------ p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:8)) p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New # Passing custom logger my_logger <- function(level, msg, ...) { cat(level, msg, \"\\n\") } p <- Pipeline$new(\"myPipe\", logger = my_logger) ## ------------------------------------------------ ## Method `Pipeline$add` ## ------------------------------------------------ # Add steps with lambda functions p <- Pipeline$new(\"myPipe\", data = 1) p$add(\"s1\", \\(x = ~data) 2*x) # use input data p$add(\"s2\", \\(x = ~data, y = ~s1) x * y) try(p$add(\"s2\", \\(z = 3) 3)) # error: step 's2' exists already #> Error : step 's2' already exists try(p$add(\"s3\", \\(z = ~foo) 3)) # dependency 'foo' not found #> Error : step 's3': dependency 'foo' not found p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: s1 data [NULL] FALSE s1 New #> 3: s2 data,s1 [NULL] FALSE s2 New p <- Pipeline$new(\"myPipe\", data = c(1, 2, NA, 3, 4)) p$add(\"calc_mean\", mean, params = list(x = ~data, na.rm = TRUE)) p$run()$get_out(\"calc_mean\") #> INFO [2024-12-01 21:41:21.908] Start run of 'myPipe' pipeline: #> INFO [2024-12-01 21:41:21.933] Step 1/2 data #> INFO [2024-12-01 21:41:21.942] Step 2/2 calc_mean #> INFO [2024-12-01 21:41:21.961] Finished execution of steps. #> INFO [2024-12-01 21:41:21.962] Done. #> [1] 2.5 p <- Pipeline$new(\"myPipe\", data = 1:10) p$add(\"s1\", \\(x = ~data) 2*x, description = \"multiply by 2\") print(p) #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: s1 data [NULL] FALSE s1 New print(p, verbose = TRUE) # print all columns #> step fun funcName params depends out keepOut group #> #> 1: data function [NULL] FALSE data #> 2: s1 function data [NULL] FALSE s1 #> description time state #> #> 1: 2024-12-01 21:41:21 New #> 2: multiply by 2 2024-12-01 21:41:21 New p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:5, y = 1:5)) p$add(\"prep_x\", \\(data = ~data) data$x, group = \"prep\") p$add(\"prep_y\", \\(data = ~data) (data$y)^2, group = \"prep\") p$add(\"sum\", \\(x = ~prep_x, y = ~prep_y) x + y) p$run()$collect_out(all = TRUE) #> INFO [2024-12-01 21:41:21.975] Start run of 'myPipe' pipeline: #> INFO [2024-12-01 21:41:21.976] Step 1/4 data #> INFO [2024-12-01 21:41:21.979] Step 2/4 prep_x #> INFO [2024-12-01 21:41:21.981] Step 3/4 prep_y #> INFO [2024-12-01 21:41:21.983] Step 4/4 sum #> INFO [2024-12-01 21:41:21.984] Finished execution of steps. #> INFO [2024-12-01 21:41:21.985] Done. #> $data #> x y #> 1 1 1 #> 2 2 2 #> 3 3 3 #> 4 4 4 #> 5 5 5 #> #> $prep #> $prep$prep_x #> [1] 1 2 3 4 5 #> #> $prep$prep_y #> [1] 1 4 9 16 25 #> #> #> $sum #> [1] 2 6 12 20 30 #> ## ------------------------------------------------ ## Method `Pipeline$append` ## ------------------------------------------------ # Append pipeline p1 <- Pipeline$new(\"pipe1\") p1$add(\"step1\", \\(x = 1) x) p2 <- Pipeline$new(\"pipe2\") p2$add(\"step2\", \\(y = 1) y) p1$append(p2) #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: step1 [NULL] FALSE step1 New #> 3: data.pipe2 [NULL] FALSE data New #> 4: step2 [NULL] FALSE step2 New p3 <- Pipeline$new(\"pipe3\") p3$add(\"step1\", \\(z = 1) z) p1$append(p2)$append(p3) #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: step1 [NULL] FALSE step1 New #> 3: data.pipe2 [NULL] FALSE data New #> 4: step2 [NULL] FALSE step2 New #> 5: data.pipe3 [NULL] FALSE data New #> 6: step1.pipe3 [NULL] FALSE step1 New ## ------------------------------------------------ ## Method `Pipeline$append_to_step_names` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\") p$add(\"step1\", \\(x = 1) x) p$add(\"step2\", \\(y = 1) y) p$append_to_step_names(\"new\") p #> step depends out keepOut group state #> #> 1: data.new [NULL] FALSE data New #> 2: step1.new [NULL] FALSE step1 New #> 3: step2.new [NULL] FALSE step2 New p$append_to_step_names(\"new\", sep = \"_\") p #> step depends out keepOut group state #> #> 1: data.new_new [NULL] FALSE data New #> 2: step1.new_new [NULL] FALSE step1 New #> 3: step2.new_new [NULL] FALSE step2 New ## ------------------------------------------------ ## Method `Pipeline$collect_out` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2) p$add(\"step2\", \\(x = ~step1) x + 2, keepOut = TRUE) p$run() #> INFO [2024-12-01 21:41:22.118] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.119] Step 1/3 data #> INFO [2024-12-01 21:41:22.121] Step 2/3 step1 #> INFO [2024-12-01 21:41:22.123] Step 3/3 step2 #> INFO [2024-12-01 21:41:22.124] Finished execution of steps. #> INFO [2024-12-01 21:41:22.125] Done. p$collect_out() #> $step2 #> [1] 5 6 #> p$collect_out(all = TRUE) |> str() #> List of 3 #> $ data : int [1:2] 1 2 #> $ step1: num [1:2] 3 4 #> $ step2: num [1:2] 5 6 p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2, group = \"add\") p$add(\"step2\", \\(x = ~step1, y = 2) x + y, group = \"add\") p$add(\"step3\", \\(x = ~data) x * 3, group = \"mult\") p$add(\"step4\", \\(x = ~data, y = 2) x * y, group = \"mult\") p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: step1 data [NULL] FALSE add New #> 3: step2 step1 [NULL] FALSE add New #> 4: step3 data [NULL] FALSE mult New #> 5: step4 data [NULL] FALSE mult New p$run() #> INFO [2024-12-01 21:41:22.154] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.155] Step 1/5 data #> INFO [2024-12-01 21:41:22.157] Step 2/5 step1 #> INFO [2024-12-01 21:41:22.159] Step 3/5 step2 #> INFO [2024-12-01 21:41:22.160] Step 4/5 step3 #> INFO [2024-12-01 21:41:22.162] Step 5/5 step4 #> INFO [2024-12-01 21:41:22.163] Finished execution of steps. #> INFO [2024-12-01 21:41:22.164] Done. p$collect_out(all = TRUE) |> str() #> List of 3 #> $ data: int [1:2] 1 2 #> $ add :List of 2 #> ..$ step1: num [1:2] 3 4 #> ..$ step2: num [1:2] 5 6 #> $ mult:List of 2 #> ..$ step3: num [1:2] 3 6 #> ..$ step4: num [1:2] 2 4 # Grouped by state p$set_params(list(y = 5)) p #> step depends out keepOut group state #> #> 1: data 1,2 FALSE data Done #> 2: step1 data 3,4 FALSE add Done #> 3: step2 step1 5,6 FALSE add Outdated #> 4: step3 data 3,6 FALSE mult Done #> 5: step4 data 2,4 FALSE mult Outdated p$collect_out(groupBy = \"state\", all = TRUE) |> str() #> List of 2 #> $ Done :List of 3 #> ..$ data : int [1:2] 1 2 #> ..$ step1: num [1:2] 3 4 #> ..$ step3: num [1:2] 3 6 #> $ Outdated:List of 2 #> ..$ step2: num [1:2] 5 6 #> ..$ step4: num [1:2] 2 4 ## ------------------------------------------------ ## Method `Pipeline$discard_steps` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~add1) x + 2) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$discard_steps(\"mult\") #> step 'mult4' was removed #> step 'mult3' was removed p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: add1 data [NULL] FALSE add1 New #> 3: add2 add1 [NULL] FALSE add2 New # Re-add steps p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: add1 data [NULL] FALSE add1 New #> 3: add2 add1 [NULL] FALSE add2 New #> 4: mult3 add1 [NULL] FALSE mult3 New #> 5: mult4 add2 [NULL] FALSE mult4 New # Discard step 'add1' does'nt work as 'add2' and 'mult3' depend on it try(p$discard_steps(\"add1\")) #> Error in self$remove_step(step, recursive = recursive) : #> cannot remove step 'add1' because the following steps depend on it: 'add2', 'mult3' p$discard_steps(\"add1\", recursive = TRUE) # this works #> Removing step 'add1' and its downstream dependencies: 'add2', 'mult3', 'mult4' #> step 'add1' was removed p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New # Trying to discard non-existent steps is just ignored p$discard_steps(\"non-existent\") ## ------------------------------------------------ ## Method `Pipeline$get_data` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$get_data() #> [1] 1 2 p$set_data(3:4) p$get_data() #> [1] 3 4 ## ------------------------------------------------ ## Method `Pipeline$get_depends` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$get_depends() #> $data #> character(0) #> #> $add1 #> x #> \"data\" #> #> $add2 #> x y #> \"data\" \"add1\" #> ## ------------------------------------------------ ## Method `Pipeline$get_depends_down` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_down(\"add1\") #> [1] \"add2\" \"mult3\" \"mult4\" p$get_depends_down(\"add1\", recursive = FALSE) #> [1] \"add2\" \"mult3\" ## ------------------------------------------------ ## Method `Pipeline$get_depends_up` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_up(\"mult4\") #> [1] \"data\" \"add1\" \"add2\" p$get_depends_up(\"mult4\", recursive = FALSE) #> [1] \"add2\" ## ------------------------------------------------ ## Method `Pipeline$get_graph` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = ~add1, y = ~add2) x * y) if (require(\"visNetwork\", quietly = TRUE)) { do.call(visNetwork, args = p$get_graph()) } {\"x\":{\"nodes\":{\"id\":[1,2,3,4],\"label\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"group\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"shape\":[\"database\",\"box\",\"box\",\"box\"],\"color\":[\"lightblue\",\"lightblue\",\"lightblue\",\"lightblue\"],\"title\":[\"

    <\\/p>\",\"

    <\\/p>\",\"

    <\\/p>\",\"

    <\\/p>\"]},\"edges\":{\"from\":[1,2,2,3],\"to\":[2,3,4,4],\"arrows\":[\"to\",\"to\",\"to\",\"to\"]},\"nodesToDataframe\":true,\"edgesToDataframe\":true,\"options\":{\"width\":\"100%\",\"height\":\"100%\",\"nodes\":{\"shape\":\"dot\"},\"manipulation\":{\"enabled\":false}},\"groups\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"width\":null,\"height\":null,\"idselection\":{\"enabled\":false},\"byselection\":{\"enabled\":false},\"main\":null,\"submain\":null,\"footer\":null,\"background\":\"rgba(0, 0, 0, 0)\"},\"evals\":[],\"jsHooks\":[]} ## ------------------------------------------------ ## Method `Pipeline$get_out` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$run() #> INFO [2024-12-01 21:41:22.273] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.274] Step 1/3 data #> INFO [2024-12-01 21:41:22.276] Step 2/3 add1 #> INFO [2024-12-01 21:41:22.278] Step 3/3 add2 #> INFO [2024-12-01 21:41:22.279] Finished execution of steps. #> INFO [2024-12-01 21:41:22.280] Done. p$get_out(\"add1\") #> [1] 2 3 p$get_out(\"add2\") #> [1] 3 5 ## ------------------------------------------------ ## Method `Pipeline$get_params` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params() |> str() #> List of 2 #> $ add1:List of 1 #> ..$ x: num 1 #> $ add2:List of 2 #> ..$ x: num 1 #> ..$ y: num 2 p$get_params(ignoreHidden = FALSE) |> str() #> List of 2 #> $ add1:List of 1 #> ..$ x: num 1 #> $ add2:List of 3 #> ..$ x : num 1 #> ..$ y : num 2 #> ..$ .z: num 3 ## ------------------------------------------------ ## Method `Pipeline$get_params_at_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params_at_step(\"add2\") #> $x #> [1] 1 #> #> $y #> [1] 2 #> p$get_params_at_step(\"add2\", ignoreHidden = FALSE) #> $x #> [1] 1 #> #> $y #> [1] 2 #> #> $.z #> [1] 3 #> p$get_params_at_step(\"add3\") #> list() ## ------------------------------------------------ ## Method `Pipeline$get_params_unique` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique() #> $x #> [1] 1 #> #> $y #> [1] 2 #> p$get_params_unique(ignoreHidden = FALSE) #> $x #> [1] 1 #> #> $y #> [1] 2 #> #> $.z #> [1] 3 #> ## ------------------------------------------------ ## Method `Pipeline$get_params_unique_json` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique_json() #> [ #> { #> \"name\": \"x\", #> \"value\": 1 #> }, #> { #> \"name\": \"y\", #> \"value\": 2 #> } #> ] p$get_params_unique_json(ignoreHidden = FALSE) #> [ #> { #> \"name\": \"x\", #> \"value\": 1 #> }, #> { #> \"name\": \"y\", #> \"value\": 2 #> }, #> { #> \"name\": \".z\", #> \"value\": 3 #> } #> ] ## ------------------------------------------------ ## Method `Pipeline$get_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, z = ~add1) x + y + z) p$run() #> INFO [2024-12-01 21:41:22.322] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.323] Step 1/3 data #> INFO [2024-12-01 21:41:22.325] Step 2/3 add1 #> INFO [2024-12-01 21:41:22.326] Step 3/3 add2 #> INFO [2024-12-01 21:41:22.328] Finished execution of steps. #> INFO [2024-12-01 21:41:22.328] Done. add1 <- p$get_step(\"add1\") print(add1) #> step fun funcName params depends out keepOut group #> #> 1: add1 function data 2,3 FALSE add1 #> description time state #> #> 1: 2024-12-01 21:41:22 Done add1[[\"params\"]] #> [[1]] #> [[1]]$data #> ~data #> #> #> [[1]]$x #> [1] 1 #> #> add1[[\"out\"]] #> [[1]] #> [1] 2 3 #> try() #> Error in try() : argument \"expr\" is missing, with no default try(p$get_step(\"foo\")) # error: step 'foo' does not exist #> Error : step 'foo' does not exist ## ------------------------------------------------ ## Method `Pipeline$get_step_names` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_names() #> [1] \"data\" \"f1\" \"f2\" ## ------------------------------------------------ ## Method `Pipeline$get_step_number` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_number(\"f2\") #> [1] 3 ## ------------------------------------------------ ## Method `Pipeline$has_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$has_step(\"f2\") #> [1] TRUE p$has_step(\"foo\") #> [1] FALSE ## ------------------------------------------------ ## Method `Pipeline$insert_after` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_after(\"f1\", \"f3\", \\(x = ~f1) x) p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New #> 3: f3 f1 [NULL] FALSE f3 New #> 4: f2 f1 [NULL] FALSE f2 New ## ------------------------------------------------ ## Method `Pipeline$insert_before` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_before(\"f2\", \"f3\", \\(x = ~f1) x) p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New #> 3: f3 f1 [NULL] FALSE f3 New #> 4: f2 f1 [NULL] FALSE f2 New ## ------------------------------------------------ ## Method `Pipeline$length` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$length() #> [1] 3 ## ------------------------------------------------ ## Method `Pipeline$lock_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$run() #> INFO [2024-12-01 21:41:22.370] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.371] Step 1/3 data #> INFO [2024-12-01 21:41:22.373] Step 2/3 add1 #> INFO [2024-12-01 21:41:22.375] Step 3/3 add2 #> INFO [2024-12-01 21:41:22.376] Finished execution of steps. #> INFO [2024-12-01 21:41:22.376] Done. p$get_out(\"add1\") #> [1] 2 p$get_out(\"add2\") #> [1] 2 p$lock_step(\"add1\") p$set_data(3) p$set_params(list(x = 3)) #> skipping setting parameters x at locked step 'add1' p$run() #> INFO [2024-12-01 21:41:22.396] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.397] Step 1/3 data #> INFO [2024-12-01 21:41:22.399] Step 2/3 add1 - skip 'locked' step #> INFO [2024-12-01 21:41:22.400] Step 3/3 add2 #> INFO [2024-12-01 21:41:22.401] Finished execution of steps. #> INFO [2024-12-01 21:41:22.402] Done. p$get_out(\"add1\") #> [1] 2 p$get_out(\"add2\") #> [1] 6 ## ------------------------------------------------ ## Method `Pipeline$print` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$print() #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New #> 3: f2 [NULL] FALSE f2 New ## ------------------------------------------------ ## Method `Pipeline$pop_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New #> 3: f2 [NULL] FALSE f2 New p$pop_step() # \"f2\" #> [1] \"f2\" p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New ## ------------------------------------------------ ## Method `Pipeline$pop_steps_after` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_after(\"f1\") # \"f2\", \"f3\" #> [1] \"f2\" \"f3\" p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New ## ------------------------------------------------ ## Method `Pipeline$pop_steps_from` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_from(\"f2\") # \"f2\", \"f3\" #> [1] \"f2\" \"f3\" p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New ## ------------------------------------------------ ## Method `Pipeline$remove_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = 1, y = ~add2) x * y) p$remove_step(\"mult1\") p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: add1 data [NULL] FALSE add1 New #> 3: add2 add1 [NULL] FALSE add2 New try(p$remove_step(\"add1\")) # fails because \"add2\" depends on \"add1\" #> Error in p$remove_step(\"add1\") : #> cannot remove step 'add1' because the following steps depend on it: 'add2' p$remove_step(\"add1\", recursive = TRUE) # removes \"add1\" and \"add2\" #> Removing step 'add1' and its downstream dependencies: 'add2' p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New ## ------------------------------------------------ ## Method `Pipeline$rename_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: add1 data [NULL] FALSE add1 New #> 3: add2 add1 [NULL] FALSE add2 New try(p$rename_step(\"add1\", \"add2\")) # fails because \"add2\" exists #> Error : step 'add2' already exists p$rename_step(\"add1\", \"first_add\") # Ok p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: first_add data [NULL] FALSE add1 New #> 3: add2 first_add [NULL] FALSE add2 New ## ------------------------------------------------ ## Method `Pipeline$replace_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = 1, y = 2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO [2024-12-01 21:41:22.462] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.463] Step 1/4 data #> INFO [2024-12-01 21:41:22.465] Step 2/4 add1 #> INFO [2024-12-01 21:41:22.466] Step 3/4 add2 #> INFO [2024-12-01 21:41:22.468] Step 4/4 mult #> INFO [2024-12-01 21:41:22.469] Finished execution of steps. #> INFO [2024-12-01 21:41:22.469] Done. #> $mult #> [1] 2 #> p$replace_step(\"mult\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO [2024-12-01 21:41:22.472] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.473] Step 1/4 data - skip 'done' step #> INFO [2024-12-01 21:41:22.474] Step 2/4 add1 - skip 'done' step #> INFO [2024-12-01 21:41:22.475] Step 3/4 add2 - skip 'done' step #> INFO [2024-12-01 21:41:22.476] Step 4/4 mult #> INFO [2024-12-01 21:41:22.477] Finished execution of steps. #> INFO [2024-12-01 21:41:22.477] Done. #> $mult #> [1] 6 #> try(p$replace_step(\"foo\", \\(x = 1) x)) # step 'foo' does not exist #> Error : step 'foo' does not exist ## ------------------------------------------------ ## Method `Pipeline$reset` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$run() #> INFO [2024-12-01 21:41:22.493] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.494] Step 1/3 data #> INFO [2024-12-01 21:41:22.495] Step 2/3 f1 #> INFO [2024-12-01 21:41:22.496] Step 3/3 f2 #> INFO [2024-12-01 21:41:22.498] Finished execution of steps. #> INFO [2024-12-01 21:41:22.498] Done. p #> step depends out keepOut group state #> #> 1: data 1,2 FALSE data Done #> 2: f1 1 FALSE f1 Done #> 3: f2 1 FALSE f2 Done p$reset() p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 [NULL] FALSE f1 New #> 3: f2 [NULL] FALSE f2 New ## ------------------------------------------------ ## Method `Pipeline$run` ## ------------------------------------------------ # Simple pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"final\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO [2024-12-01 21:41:22.508] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.509] Step 1/4 data #> INFO [2024-12-01 21:41:22.511] Step 2/4 add1 #> INFO [2024-12-01 21:41:22.513] Step 3/4 add2 #> INFO [2024-12-01 21:41:22.515] Step 4/4 final #> INFO [2024-12-01 21:41:22.516] Finished execution of steps. #> INFO [2024-12-01 21:41:22.517] Done. #> $final #> [1] 8 #> p$set_params(list(z = 4)) # outdates steps add2 and final p #> step depends out keepOut group state #> #> 1: data 1 FALSE data Done #> 2: add1 data 2 FALSE add1 Done #> 3: add2 add1 4 FALSE add2 Outdated #> 4: final add1,add2 8 TRUE final Outdated p$run()$collect_out() #> INFO [2024-12-01 21:41:22.524] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.525] Step 1/4 data - skip 'done' step #> INFO [2024-12-01 21:41:22.526] Step 2/4 add1 - skip 'done' step #> INFO [2024-12-01 21:41:22.526] Step 3/4 add2 #> INFO [2024-12-01 21:41:22.528] Step 4/4 final #> INFO [2024-12-01 21:41:22.529] Finished execution of steps. #> INFO [2024-12-01 21:41:22.530] Done. #> $final #> [1] 12 #> p$run(cleanUnkept = TRUE) # clean up temporary results #> INFO [2024-12-01 21:41:22.531] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.532] Step 1/4 data - skip 'done' step #> INFO [2024-12-01 21:41:22.533] Step 2/4 add1 - skip 'done' step #> INFO [2024-12-01 21:41:22.534] Step 3/4 add2 - skip 'done' step #> INFO [2024-12-01 21:41:22.535] Step 4/4 final - skip 'done' step #> INFO [2024-12-01 21:41:22.535] Finished execution of steps. #> INFO [2024-12-01 21:41:22.536] Clean temporary results. #> INFO [2024-12-01 21:41:22.536] Done. p #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data Outdated #> 2: add1 data [NULL] FALSE add1 Outdated #> 3: add2 add1 [NULL] FALSE add2 Outdated #> 4: final add1,add2 12 TRUE final Done # Recursive pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"new_pipe\", \\(x = ~add1) { pp <- Pipeline$new(\"new_pipe\", data = x) pp$add(\"add1\", \\(x = ~data) x + 1) pp$add(\"add2\", \\(x = ~add1) x + 2, keepOut = TRUE) } ) p$run()$collect_out() #> INFO [2024-12-01 21:41:22.543] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:22.544] Step 1/3 data #> INFO [2024-12-01 21:41:22.546] Step 2/3 add1 #> INFO [2024-12-01 21:41:22.547] Step 3/3 new_pipe #> INFO [2024-12-01 21:41:22.550] Abort pipeline execution and restart on new. #> INFO [2024-12-01 21:41:22.551] Start run of 'new_pipe' pipeline: #> INFO [2024-12-01 21:41:22.552] Step 1/3 data #> INFO [2024-12-01 21:41:22.554] Step 2/3 add1 #> INFO [2024-12-01 21:41:22.556] Step 3/3 add2 #> INFO [2024-12-01 21:41:22.557] Finished execution of steps. #> INFO [2024-12-01 21:41:22.557] Done. #> $add2 #> [1] 5 #> # Run pipeline with progress bar p <- Pipeline$new(\"pipe\", data = 1) p$add(\"first step\", \\() Sys.sleep(1)) p$add(\"second step\", \\() Sys.sleep(1)) p$add(\"last step\", \\() Sys.sleep(1)) pb <- txtProgressBar(min = 1, max = p$length(), style = 3) fprogress <- function(value, detail) { setTxtProgressBar(pb, value) } p$run(progress = fprogress, showLog = FALSE) #> | | | 0% | |======================= | 33% | |=============================================== | 67% | |======================================================================| 100% ## ------------------------------------------------ ## Method `Pipeline$run_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run_step(\"add2\") #> INFO [2024-12-01 21:41:25.633] Start step run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.634] Step 1/3 data (upstream) #> INFO [2024-12-01 21:41:25.636] Step 2/3 add1 (upstream) #> INFO [2024-12-01 21:41:25.638] Step 3/3 add2 #> INFO [2024-12-01 21:41:25.640] Finished execution of steps. #> INFO [2024-12-01 21:41:25.640] Done. p$run_step(\"add2\", downstream = TRUE) #> INFO [2024-12-01 21:41:25.655] Start step run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.655] Step 1/4 data (upstream) #> INFO [2024-12-01 21:41:25.658] Step 2/4 add1 (upstream) #> INFO [2024-12-01 21:41:25.660] Step 3/4 add2 #> INFO [2024-12-01 21:41:25.664] Step 4/4 mult (downstream) #> INFO [2024-12-01 21:41:25.665] Finished execution of steps. #> INFO [2024-12-01 21:41:25.666] Done. p$run_step(\"mult\", upstream = TRUE) #> INFO [2024-12-01 21:41:25.667] Start step run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.668] Step 1/4 data (upstream) #> INFO [2024-12-01 21:41:25.670] Step 2/4 add1 (upstream) #> INFO [2024-12-01 21:41:25.672] Step 3/4 add2 (upstream) #> INFO [2024-12-01 21:41:25.674] Step 4/4 mult #> INFO [2024-12-01 21:41:25.675] Finished execution of steps. #> INFO [2024-12-01 21:41:25.675] Done. ## ------------------------------------------------ ## Method `Pipeline$set_data` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$run()$collect_out() #> INFO [2024-12-01 21:41:25.679] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.680] Step 1/2 data #> INFO [2024-12-01 21:41:25.682] Step 2/2 add1 #> INFO [2024-12-01 21:41:25.683] Finished execution of steps. #> INFO [2024-12-01 21:41:25.684] Done. #> $add1 #> [1] 2 #> p$set_data(3) p$run()$collect_out() #> INFO [2024-12-01 21:41:25.687] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.688] Step 1/2 data #> INFO [2024-12-01 21:41:25.690] Step 2/2 add1 #> INFO [2024-12-01 21:41:25.691] Finished execution of steps. #> INFO [2024-12-01 21:41:25.692] Done. #> $add1 #> [1] 4 #> ## ------------------------------------------------ ## Method `Pipeline$set_data_split` ## ------------------------------------------------ # Split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList) p3 #> step depends out keepOut group state #> #> 1: data.a [NULL] FALSE a New #> 2: add1.a data.a [NULL] TRUE a Outdated #> 3: mult.a data.a,add1.a [NULL] TRUE a Outdated #> 4: data.b [NULL] FALSE b New #> 5: add1.b data.b [NULL] TRUE b Outdated #> 6: mult.b data.b,add1.b [NULL] TRUE b Outdated #> 7: data.c [NULL] FALSE c New #> 8: add1.c data.c [NULL] TRUE c Outdated #> 9: mult.c data.c,add1.c [NULL] TRUE c Outdated p3$run()$collect_out() |> str() #> INFO [2024-12-01 21:41:25.714] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.715] Step 1/9 data.a #> INFO [2024-12-01 21:41:25.717] Step 2/9 add1.a #> INFO [2024-12-01 21:41:25.719] Step 3/9 mult.a #> INFO [2024-12-01 21:41:25.721] Step 4/9 data.b #> INFO [2024-12-01 21:41:25.723] Step 5/9 add1.b #> INFO [2024-12-01 21:41:25.725] Step 6/9 mult.b #> INFO [2024-12-01 21:41:25.726] Step 7/9 data.c #> INFO [2024-12-01 21:41:25.728] Step 8/9 add1.c #> INFO [2024-12-01 21:41:25.730] Step 9/9 mult.c #> INFO [2024-12-01 21:41:25.731] Finished execution of steps. #> INFO [2024-12-01 21:41:25.732] Done. #> List of 3 #> $ a:List of 2 #> ..$ add1.a: num 2 #> ..$ mult.a: num 2 #> $ b:List of 2 #> ..$ add1.b: num 3 #> ..$ mult.b: num 6 #> $ c:List of 2 #> ..$ add1.c: num 4 #> ..$ mult.c: num 12 # Don't group output by split p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList, groupBySplit = FALSE) p3 #> step depends out keepOut group state #> #> 1: data.a [NULL] FALSE data.a New #> 2: add1.a data.a [NULL] TRUE add1.a Outdated #> 3: mult.a data.a,add1.a [NULL] TRUE mult.a Outdated #> 4: data.b [NULL] FALSE data.b New #> 5: add1.b data.b [NULL] TRUE add1.b Outdated #> 6: mult.b data.b,add1.b [NULL] TRUE mult.b Outdated #> 7: data.c [NULL] FALSE data.c New #> 8: add1.c data.c [NULL] TRUE add1.c Outdated #> 9: mult.c data.c,add1.c [NULL] TRUE mult.c Outdated p3$run()$collect_out() |> str() #> INFO [2024-12-01 21:41:25.757] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.758] Step 1/9 data.a #> INFO [2024-12-01 21:41:25.760] Step 2/9 add1.a #> INFO [2024-12-01 21:41:25.762] Step 3/9 mult.a #> INFO [2024-12-01 21:41:25.764] Step 4/9 data.b #> INFO [2024-12-01 21:41:25.766] Step 5/9 add1.b #> INFO [2024-12-01 21:41:25.767] Step 6/9 mult.b #> INFO [2024-12-01 21:41:25.769] Step 7/9 data.c #> INFO [2024-12-01 21:41:25.771] Step 8/9 add1.c #> INFO [2024-12-01 21:41:25.773] Step 9/9 mult.c #> INFO [2024-12-01 21:41:25.774] Finished execution of steps. #> INFO [2024-12-01 21:41:25.775] Done. #> List of 6 #> $ add1.a: num 2 #> $ mult.a: num 2 #> $ add1.b: num 3 #> $ mult.b: num 6 #> $ add1.c: num 4 #> $ mult.c: num 12 ## ------------------------------------------------ ## Method `Pipeline$set_keep_out` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run()$collect_out() #> INFO [2024-12-01 21:41:25.783] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:25.784] Step 1/4 data #> INFO [2024-12-01 21:41:25.786] Step 2/4 add1 #> INFO [2024-12-01 21:41:25.788] Step 3/4 add2 #> INFO [2024-12-01 21:41:25.790] Step 4/4 mult #> INFO [2024-12-01 21:41:25.791] Finished execution of steps. #> INFO [2024-12-01 21:41:25.792] Done. #> $add1 #> [1] 2 #> p$set_keep_out(\"add1\", keepOut = FALSE) p$set_keep_out(\"mult\", keepOut = TRUE) p$collect_out() #> $mult #> [1] 6 #> ## ------------------------------------------------ ## Method `Pipeline$set_params` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 1) x + y) p$add(\"mult\", \\(x = 1, z = 1) x * z) p$get_params() #> $add1 #> $add1$y #> [1] 1 #> #> #> $add2 #> $add2$y #> [1] 1 #> #> #> $mult #> $mult$x #> [1] 1 #> #> $mult$z #> [1] 1 #> #> p$set_params(list(x = 3, y = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #> #> #> $add2 #> $add2$y #> [1] 3 #> #> #> $mult #> $mult$x #> [1] 3 #> #> $mult$z #> [1] 1 #> #> p$set_params(list(x = 5, z = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #> #> #> $add2 #> $add2$y #> [1] 3 #> #> #> $mult #> $mult$x #> [1] 5 #> #> $mult$z #> [1] 3 #> #> suppressWarnings( p$set_params(list(foo = 3)) # warning: trying to set undefined ) ## ------------------------------------------------ ## Method `Pipeline$set_params_at_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1, z = 2) x + y) p$add(\"add2\", \\(x = ~data, y = 1, z = 2) x + y) p$set_params_at_step(\"add1\", list(y = 3, z = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #> #> $add1$z #> [1] 3 #> #> #> $add2 #> $add2$y #> [1] 1 #> #> $add2$z #> [1] 2 #> #> try(p$set_params_at_step(\"add1\", list(foo = 3))) # foo not defined #> Error in p$set_params_at_step(\"add1\", list(foo = 3)) : #> Unable to set parameter(s) foo at step add1 - candidates are y, z ## ------------------------------------------------ ## Method `Pipeline$split` ## ------------------------------------------------ # Example for two independent calculation paths p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = ~data) x) p$add(\"f2\", \\(x = 1) x) p$add(\"f3\", \\(x = ~f1) x) p$add(\"f4\", \\(x = ~f2) x) p$split() #> [[1]] #> step depends out keepOut group state #> #> 1: data [NULL] FALSE data New #> 2: f1 data [NULL] FALSE f1 New #> 3: f3 f1 [NULL] FALSE f3 New #> #> [[2]] #> step depends out keepOut group state #> #> 1: f2 [NULL] FALSE f2 New #> 2: f4 f2 [NULL] FALSE f4 New #> # Example of split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) pips <- p$set_data_split(dataList)$split() ## ------------------------------------------------ ## Method `Pipeline$unlock_step` ## ------------------------------------------------ p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$lock_step(\"add1\") p$set_params(list(x = 3)) #> skipping setting parameters x at locked step 'add1' p$get_params() #> $add1 #> $add1$x #> [1] 1 #> #> #> $add2 #> $add2$x #> [1] 3 #> #> p$unlock_step(\"add1\") p$set_params(list(x = 3)) p$get_params() #> $add1 #> $add1$x #> [1] 3 #> #> #> $add2 #> $add2$x #> [1] 3 #> #>"},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":null,"dir":"Reference","previous_headings":"","what":"Pipeline alias functions — pipelineAliases","title":"Pipeline alias functions — pipelineAliases","text":"Alias functions, one member function Pipeline object.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline alias functions — pipelineAliases","text":"","code":"pipe_add(pip, ...) pipe_append(pip, ...) pipe_append_to_step_names(pip, ...) pipe_clone(pip, ...) pipe_collect_out(pip, ...) pipe_discard_steps(pip, ...) pipe_get_data(pip, ...) pipe_get_depends(pip, ...) pipe_get_depends_down(pip, ...) pipe_get_depends_up(pip, ...) pipe_get_graph(pip, ...) pipe_get_out(pip, ...) pipe_get_params(pip, ...) pipe_get_params_at_step(pip, ...) pipe_get_params_unique(pip, ...) pipe_get_params_unique_json(pip, ...) pipe_get_step(pip, ...) pipe_get_step_names(pip, ...) pipe_get_step_number(pip, ...) pipe_has_step(pip, ...) pipe_insert_after(pip, ...) pipe_insert_before(pip, ...) pipe_length(pip, ...) pipe_lock_step(pip, ...) pipe_new(...) pipe_print(pip, ...) pipe_pop_step(pip, ...) pipe_pop_steps_after(pip, ...) pipe_pop_steps_from(pip, ...) pipe_remove_step(pip, ...) pipe_rename_step(pip, ...) pipe_replace_step(pip, ...) pipe_reset(pip, ...) pipe_run(pip, ...) pipe_run_step(pip, ...) pipe_set_data(pip, ...) pipe_set_data_split(pip, ...) pipe_set_keep_out(pip, ...) pipe_set_params(pip, ...) pipe_set_params_at_step(pip, ...) pipe_split(pip, ...) pipe_unlock_step(pip, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline alias functions — pipelineAliases","text":"pip pipeline object ... Arguments passed respective pipeline method","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Pipeline alias functions — pipelineAliases","text":"result respective pipeline method","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":null,"dir":"Reference","previous_headings":"","what":"Set pipeflow log layout — set_log_layout","title":"Set pipeflow log layout — set_log_layout","text":"Set pipeflow log layout","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set pipeflow log layout — set_log_layout","text":"","code":"set_log_layout(layout)"},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set pipeflow log layout — set_log_layout","text":"layout Layout name","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set pipeflow log layout — set_log_layout","text":"invisibly returns logger object","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set pipeflow log layout — set_log_layout","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$run() #> INFO [2024-12-01 21:41:29.126] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:29.127] Step 1/2 data #> INFO [2024-12-01 21:41:29.129] Step 2/2 add1 #> INFO [2024-12-01 21:41:29.130] Finished execution of steps. #> INFO [2024-12-01 21:41:29.131] Done. lg <- set_log_layout(\"json\") print(lg) #> [info] pipeflow #> #> appenders: #> [[1]]: [all] -> console p$run() #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Start run of 'pipe' pipeline:\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Step 1/2 data - skip 'done' step\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Step 2/2 add1 - skip 'done' step\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Finished execution of steps.\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Done.\"} set_log_layout(\"text\") p$run() #> INFO [2024-12-01 21:41:29.169] Start run of 'pipe' pipeline: #> INFO [2024-12-01 21:41:29.170] Step 1/2 data - skip 'done' step #> INFO [2024-12-01 21:41:29.171] Step 2/2 add1 - skip 'done' step #> INFO [2024-12-01 21:41:29.171] Finished execution of steps. #> INFO [2024-12-01 21:41:29.172] Done."}]