diff --git a/_pkgdown.yml b/_pkgdown.yml
index 5f52493..694af7b 100644
--- a/_pkgdown.yml
+++ b/_pkgdown.yml
@@ -35,7 +35,7 @@ reference:
 navbar:
   structure:
     left: [home, reference, articles, myPackages]
-    right: [search, github, stackoverflow, bluesky, blog]
+    right: [github, stackoverflow, bluesky, blog]
   components:
     home:
       icon: fa-home fa-lg
diff --git a/docs/404.html b/docs/404.html
index 143c861..1411878 100644
--- a/docs/404.html
+++ b/docs/404.html
@@ -64,9 +64,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
diff --git a/docs/articles/collect-output.html b/docs/articles/collect-output.html
index 3194e88..ab0b486 100644
--- a/docs/articles/collect-output.html
+++ b/docs/articles/collect-output.html
@@ -68,9 +68,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
@@ -198,15 +195,15 @@ <h3 id="flag-output-steps">Flag output steps<a class="anchor" aria-label="anchor
 the flagged steps.</p>
 <div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.888] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.916] Step 1/6 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.925] Step 2/6 data_prep</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.948] Step 3/6 data_summary</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.950] Step 4/6 model_fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.954] Step 5/6 model_summary</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.956] Step 6/6 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.968] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:31.968] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.511] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.539] Step 1/6 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.548] Step 2/6 data_prep</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.572] Step 3/6 data_summary</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.573] Step 4/6 model_fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.577] Step 5/6 model_summary</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.580] Step 6/6 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.591] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.591] Done.</span></span>
 <span></span>
 <span><span class="va">out</span> <span class="op">&lt;-</span> <span class="va">pip</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
 <span></span>
@@ -310,15 +307,15 @@ <h3 id="grouping-output-steps">Grouping output steps<a class="anchor" aria-label
 we run the pipeline and collect the output.</p>
 <div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.198] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.199] Step 1/6 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.202] Step 2/6 data_prep</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.205] Step 3/6 used_data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.206] Step 4/6 model_fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.229] Step 5/6 model_summary</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.231] Step 6/6 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.236] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:32.237] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.822] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.823] Step 1/6 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.826] Step 2/6 data_prep</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.829] Step 3/6 used_data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.830] Step 4/6 model_fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.853] Step 5/6 model_summary</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.855] Step 6/6 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.860] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:27.861] Done.</span></span>
 <span></span>
 <span><span class="va">out</span> <span class="op">&lt;-</span> <span class="va">pip</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
 <span></span>
diff --git a/docs/articles/combine-pipelines.html b/docs/articles/combine-pipelines.html
index 8c7b9dc..9a74c1a 100644
--- a/docs/articles/combine-pipelines.html
+++ b/docs/articles/combine-pipelines.html
@@ -68,9 +68,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
@@ -291,8 +288,8 @@ <h4 id="relative-indexing">Relative indexing<a class="anchor" aria-label="anchor
 <span><span class="co"># [[1]]</span></span>
 <span><span class="co"># function (data = ~-1) </span></span>
 <span><span class="co"># data</span></span>
-<span><span class="co"># &lt;bytecode: 0x000001c01370b3f8&gt;</span></span>
-<span><span class="co"># &lt;environment: 0x000001c00f2b05a8&gt;</span></span></code></pre></div>
+<span><span class="co"># &lt;bytecode: 0x0000015b1cf930e8&gt;</span></span>
+<span><span class="co"># &lt;environment: 0x0000015b18bae928&gt;</span></span></code></pre></div>
 </div>
 </div>
 <div class="section level3">
@@ -301,15 +298,15 @@ <h3 id="run-combined-pipeline">Run combined pipeline<a class="anchor" aria-label
 <p>Let’s now run the combined pipeline and inspect the plot.</p>
 <div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.693] Start run of 'preprocessing.modeling' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.763] Step 1/6 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.772] Step 2/6 data_prep</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.788] Step 3/6 standardize</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.791] Step 4/6 data.modeling</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.793] Step 5/6 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.797] Step 6/6 plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.810] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:36.811] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.182] Start run of 'preprocessing.modeling' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.255] Step 1/6 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.263] Step 2/6 data_prep</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.280] Step 3/6 standardize</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.283] Step 4/6 data.modeling</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.285] Step 5/6 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.289] Step 6/6 plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.302] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.303] Done.</span></span></code></pre></div>
 <div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"plot"</span><span class="op">)</span></span>
 <span><span class="co"># Warning: Removed 37 rows containing missing values or values outside the scale range</span></span>
@@ -323,15 +320,15 @@ <h3 id="run-combined-pipeline">Run combined pipeline<a class="anchor" aria-label
 <div id="htmlwidget-10b3b7155e8045a1b2ad" style="width:700px;height:100px;" class="visNetwork html-widget"></div>
 <script type="application/json" data-for="htmlwidget-10b3b7155e8045a1b2ad">{"x":{"nodes":{"id":[1,2,3,4,5,6],"label":["data","data_prep","standardize","data.modeling","fit","plot"],"group":["data","data_prep","standardize","data.modeling","fit","plot"],"shape":["database","database","box","database","box","box"],"color":["lightgreen","lightgreen","lightgreen","lightgreen","orange","orange"],"title":["<p><\/p>","<p><\/p>","<p><\/p>","<p>output of last step of first pipeline<\/p>","<p><\/p>","<p><\/p>"]},"edges":{"from":[1,2,3,4,4,5],"to":[2,3,4,5,6,6],"arrows":["to","to","to","to","to","to"]},"nodesToDataframe":true,"edgesToDataframe":true,"options":{"width":"100%","height":"100%","nodes":{"shape":"dot"},"manipulation":{"enabled":false},"layout":{"hierarchical":{"enabled":true,"direction":"LR","sortMethod":"directed"}}},"groups":["data","data_prep","standardize","data.modeling","fit","plot"],"width":null,"height":100,"idselection":{"enabled":false},"byselection":{"enabled":false},"main":null,"submain":null,"footer":null,"background":"rgba(0, 0, 0, 0)"},"evals":[],"jsHooks":[]}</script><div class="sourceCode" id="cb13"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.360] Start run of 'preprocessing.modeling' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.361] Step 1/6 data - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.362] Step 2/6 data_prep - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.362] Step 3/6 standardize - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.363] Step 4/6 data.modeling - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.364] Step 5/6 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.368] Step 6/6 plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.385] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:37.386] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.852] Start run of 'preprocessing.modeling' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.853] Step 1/6 data - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.854] Step 2/6 data_prep - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.855] Step 3/6 standardize - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.856] Step 4/6 data.modeling - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.857] Step 5/6 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.861] Step 6/6 plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.878] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:32.879] Done.</span></span></code></pre></div>
 <div id="htmlwidget-4018eef1a407a0df6b52" style="width:700px;height:100px;" class="visNetwork html-widget"></div>
 <script type="application/json" data-for="htmlwidget-4018eef1a407a0df6b52">{"x":{"nodes":{"id":[1,2,3,4,5,6],"label":["data","data_prep","standardize","data.modeling","fit","plot"],"group":["data","data_prep","standardize","data.modeling","fit","plot"],"shape":["database","database","box","database","box","box"],"color":["lightgreen","lightgreen","lightgreen","lightgreen","lightgreen","lightgreen"],"title":["<p><\/p>","<p><\/p>","<p><\/p>","<p>output of last step of first pipeline<\/p>","<p><\/p>","<p><\/p>"]},"edges":{"from":[1,2,3,4,4,5],"to":[2,3,4,5,6,6],"arrows":["to","to","to","to","to","to"]},"nodesToDataframe":true,"edgesToDataframe":true,"options":{"width":"100%","height":"100%","nodes":{"shape":"dot"},"manipulation":{"enabled":false},"layout":{"hierarchical":{"enabled":true,"direction":"LR","sortMethod":"directed"}}},"groups":["data","data_prep","standardize","data.modeling","fit","plot"],"width":null,"height":100,"idselection":{"enabled":false},"byselection":{"enabled":false},"main":null,"submain":null,"footer":null,"background":"rgba(0, 0, 0, 0)"},"evals":[],"jsHooks":[]}</script><div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"plot"</span><span class="op">)</span></span>
diff --git a/docs/articles/get-started.html b/docs/articles/get-started.html
index a713267..c70b34b 100644
--- a/docs/articles/get-started.html
+++ b/docs/articles/get-started.html
@@ -68,9 +68,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
@@ -232,13 +229,13 @@ <h3 id="run-the-pipeline-and-inspect-the-output">Run the pipeline and inspect th
 the pipeline, which will produce the following output:</p>
 <div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.560] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.586] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.597] Step 2/4 data_prep</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.620] Step 3/4 model_fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.623] Step 4/4 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.635] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:41.636] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.075] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.101] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.112] Step 2/4 data_prep</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.134] Step 3/4 model_fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.138] Step 4/4 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.149] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:37.150] Done.</span></span></code></pre></div>
 <p>Let’s inspect the pipeline again.</p>
 <div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span></span>
@@ -323,13 +320,13 @@ <h3 id="change-parameters-of-the-pipeline">Change parameters of the pipeline<a c
 the pipeline again.</p>
 <div class="sourceCode" id="cb17"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.547] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.549] Step 1/4 data - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.550] Step 2/4 data_prep - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.551] Step 3/4 model_fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.555] Step 4/4 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.564] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.565] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.057] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.058] Step 1/4 data - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.059] Step 2/4 data_prep - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.060] Step 3/4 model_fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.064] Step 4/4 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.074] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.074] Done.</span></span></code></pre></div>
 <p>The outdated steps were re-run as expected and the output was updated
 accordingly.</p>
 <div class="sourceCode" id="cb18"><pre class="downlit sourceCode r">
@@ -348,13 +345,13 @@ <h3 id="change-parameters-of-the-pipeline">Change parameters of the pipeline<a c
 <span><span class="co"># 4: model_plot model_fit,data_prep            &lt;gg[11]&gt;   FALSE model_plot Outdated</span></span></code></pre></div>
 <div class="sourceCode" id="cb20"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"model_plot"</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.919] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.920] Step 1/4 data - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.921] Step 2/4 data_prep - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.922] Step 3/4 model_fit - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.923] Step 4/4 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.931] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:42.931] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.428] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.429] Step 1/4 data - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.430] Step 2/4 data_prep - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.431] Step 3/4 model_fit - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.432] Step 4/4 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.440] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.440] Done.</span></span></code></pre></div>
 <p><img src="get-started_files/figure-html/inspect-plot-after-title-change-1.png" alt="model-plot" width="700"></p>
 <p>If instead we were to change the input data, since all steps depend
 on it, we expect all steps to be rerun.</p>
@@ -369,13 +366,13 @@ <h3 id="change-parameters-of-the-pipeline">Change parameters of the pipeline<a c
 <span><span class="co"># 4: model_plot model_fit,data_prep            &lt;gg[11]&gt;   FALSE model_plot Outdated</span></span></code></pre></div>
 <div class="sourceCode" id="cb22"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"model_plot"</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.202] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.203] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.206] Step 2/4 data_prep</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.211] Step 3/4 model_fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.214] Step 4/4 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.219] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:43.220] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.710] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.711] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.714] Step 2/4 data_prep</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.718] Step 3/4 model_fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.720] Step 4/4 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.726] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:38.726] Done.</span></span></code></pre></div>
 <p><img src="get-started_files/figure-html/inspect-plot-after-data-change-1.png" alt="model-plot" width="700"></p>
 <p>Next, let’s see how to <a href="modify-pipeline.html">modify the
 pipeline</a>.</p>
diff --git a/docs/articles/index.html b/docs/articles/index.html
index cc812aa..f15ca9d 100644
--- a/docs/articles/index.html
+++ b/docs/articles/index.html
@@ -37,9 +37,7 @@
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/GroupSeq/">GroupSeq</a></li>
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/pipeflow/">pipeflow</a></li>
   </ul></li>
-      </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li>
-<li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
+      </ul><ul class="navbar-nav"><li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://rpahl.github.io/r-some-blog/" aria-label="blog"><span class="fa fas fa-blog"></span></a></li>
diff --git a/docs/articles/modify-pipeline.html b/docs/articles/modify-pipeline.html
index 50a4244..035572e 100644
--- a/docs/articles/modify-pipeline.html
+++ b/docs/articles/modify-pipeline.html
@@ -68,9 +68,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
@@ -213,14 +210,14 @@ <h3 id="replace-existing-steps">Replace existing steps<a class="anchor" aria-lab
 <div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>xVar <span class="op">=</span> <span class="st">"Solar.R"</span>, yVar <span class="op">=</span> <span class="st">"Wind"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.485] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.486] Step 1/5 data - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.487] Step 2/5 data_prep - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.488] Step 3/5 standardize</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.490] Step 4/5 model_fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.493] Step 5/5 model_plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.498] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:47.498] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.011] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.012] Step 1/5 data - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.013] Step 2/5 data_prep - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.014] Step 3/5 standardize</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.016] Step 4/5 model_fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.019] Step 5/5 model_plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.024] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:43.025] Done.</span></span></code></pre></div>
 <div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"model_fit"</span><span class="op">)</span> <span class="op">|&gt;</span> <span class="fu"><a href="https://rdrr.io/r/stats/coef.html" class="external-link">coefficients</a></span><span class="op">(</span><span class="op">)</span></span>
 <span><span class="co">#  (Intercept)      Solar.R </span></span>
diff --git a/docs/articles/self-modify-pipeline.html b/docs/articles/self-modify-pipeline.html
index 03d5e2d..d6b9dd3 100644
--- a/docs/articles/self-modify-pipeline.html
+++ b/docs/articles/self-modify-pipeline.html
@@ -68,9 +68,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
@@ -162,13 +159,13 @@ <h3 id="changing-pipeline-parameters-at-runtime">Changing pipeline parameters at
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="va">airquality</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>xVar <span class="op">=</span> <span class="st">"Ozone"</span>, yVar <span class="op">=</span> <span class="st">"Temp"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.403] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.430] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.444] Step 2/4 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.462] Step 3/4 residual_shapiro_p_value</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.464] Step 4/4 plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.481] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.482] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.851] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.880] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.895] Step 2/4 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.912] Step 3/4 residual_shapiro_p_value</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.914] Step 4/4 plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.930] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:46.931] Done.</span></span>
 <span><span class="co"># $residual_shapiro_p_value</span></span>
 <span><span class="co"># [1] 0.00022598</span></span>
 <span><span class="co"># </span></span>
@@ -187,13 +184,13 @@ <h3 id="changing-pipeline-parameters-at-runtime">Changing pipeline parameters at
 <span>    <span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>pointColor <span class="op">=</span> <span class="st">"red"</span><span class="op">)</span><span class="op">)</span></span>
 <span>    <span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
 <span><span class="op">}</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.876] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.877] Step 1/4 data - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.878] Step 2/4 fit - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.879] Step 3/4 residual_shapiro_p_value - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.880] Step 4/4 plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.893] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:51.894] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.325] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.326] Step 1/4 data - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.327] Step 2/4 fit - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.328] Step 3/4 residual_shapiro_p_value - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.329] Step 4/4 plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.342] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.342] Done.</span></span>
 <span><span class="co"># $residual_shapiro_p_value</span></span>
 <span><span class="co"># [1] 0.00022598</span></span>
 <span><span class="co"># </span></span>
@@ -231,13 +228,13 @@ <h3 id="changing-pipeline-parameters-at-runtime">Changing pipeline parameters at
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="va">airquality</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>xVar <span class="op">=</span> <span class="st">"Ozone"</span>, yVar <span class="op">=</span> <span class="st">"Temp"</span>, .self <span class="op">=</span> <span class="va">pip</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.189] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.190] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.192] Step 2/4 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.196] Step 3/4 residual_shapiro_p_value</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.200] Step 4/4 plot</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.207] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.207] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.638] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.640] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.642] Step 2/4 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.646] Step 3/4 residual_shapiro_p_value</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.650] Step 4/4 plot</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.657] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.658] Done.</span></span>
 <span><span class="co"># $residual_shapiro_p_value</span></span>
 <span><span class="co"># [1] 0.00022598</span></span>
 <span><span class="co"># </span></span>
@@ -281,13 +278,13 @@ <h3 id="changing-the-pipeline-structure-at-runtime">Changing the pipeline struct
 respectively.</p>
 <div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="fl">1</span><span class="op">)</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.449] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.450] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.452] Step 2/4 f1</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.454] Step 3/4 f2</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.456] Step 4/4 f3</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.457] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.458] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.901] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.902] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.904] Step 2/4 f1</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.907] Step 3/4 f2</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.908] Step 4/4 f3</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.910] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:47.910] Done.</span></span>
 <span><span class="va">pip</span></span>
 <span><span class="co">#      step depends    out keepOut  group  state</span></span>
 <span><span class="co">#    &lt;char&gt;  &lt;list&gt; &lt;list&gt;  &lt;lgcl&gt; &lt;char&gt; &lt;char&gt;</span></span>
@@ -340,13 +337,13 @@ <h4 id="modify-a-step">Modify a step<a class="anchor" aria-label="anchor" href="
 <div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>.self <span class="op">=</span> <span class="va">pip</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="fl">1</span><span class="op">)</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.575] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.576] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.578] Step 2/4 f1</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.580] Step 3/4 f2</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.582] Step 4/4 f3</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.584] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.584] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.028] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.029] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.031] Step 2/4 f1</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.034] Step 3/4 f2</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.035] Step 4/4 f3</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.037] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.037] Done.</span></span>
 <span><span class="va">pip</span></span>
 <span><span class="co">#      step depends    out keepOut  group  state</span></span>
 <span><span class="co">#    &lt;char&gt;  &lt;list&gt; &lt;list&gt;  &lt;lgcl&gt; &lt;char&gt; &lt;char&gt;</span></span>
@@ -357,13 +354,13 @@ <h4 id="modify-a-step">Modify a step<a class="anchor" aria-label="anchor" href="
 <p>Now let’s try it with an input of 10.</p>
 <div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="fl">10</span><span class="op">)</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.643] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.644] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.646] Step 2/4 f1</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.649] Step 3/4 f2</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.654] Step 4/4 f3</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.656] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.656] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.096] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.097] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.099] Step 2/4 f1</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.102] Step 3/4 f2</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.108] Step 4/4 f3</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.109] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.110] Done.</span></span>
 <span><span class="va">pip</span></span>
 <span><span class="co">#      step depends    out keepOut  group  state</span></span>
 <span><span class="co">#    &lt;char&gt;  &lt;list&gt; &lt;list&gt;  &lt;lgcl&gt; &lt;char&gt; &lt;char&gt;</span></span>
@@ -452,19 +449,19 @@ <h4 id="insert-and-remove-steps">Insert and remove steps<a class="anchor" aria-l
 <div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>.self <span class="op">=</span> <span class="va">pip</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="fl">10</span><span class="op">)</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.829] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.830] Step 1/4 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.832] Step 2/4 f1</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.834] Step 3/4 f2</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.848] Abort pipeline execution and restart on new.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.848] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.849] Step 1/5 data - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.850] Step 2/5 f1 - skip 'done' step</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.851] Step 3/5 f2a</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.856] Step 4/5 f2b</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.858] Step 5/5 f3</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.859] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:52.860] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.282] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.283] Step 1/4 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.285] Step 2/4 f1</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.287] Step 3/4 f2</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.300] Abort pipeline execution and restart on new.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.301] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.302] Step 1/5 data - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.302] Step 2/5 f1 - skip 'done' step</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.303] Step 3/5 f2a</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.308] Step 4/5 f2b</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.310] Step 5/5 f3</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.312] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:48.312] Done.</span></span></code></pre></div>
 <p>The log output shows the abort and re-run of the pipeline. Let’s see
 the final structure and step outputs.</p>
 <div class="sourceCode" id="cb15"><pre class="downlit sourceCode r">
diff --git a/docs/articles/split-and-combine.html b/docs/articles/split-and-combine.html
index 855d968..8978c94 100644
--- a/docs/articles/split-and-combine.html
+++ b/docs/articles/split-and-combine.html
@@ -68,9 +68,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
@@ -168,12 +165,12 @@ <h3 id="define-pipeline">Define pipeline<a class="anchor" aria-label="anchor" hr
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="va">iris</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>xVar <span class="op">=</span> <span class="st">"Sepal.Length"</span>, yVar <span class="op">=</span> <span class="st">"Sepal.Width"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:55.998] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.024] Step 1/3 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.035] Step 2/3 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.055] Step 3/3 coefs</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.056] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.056] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.463] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.487] Step 1/3 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.498] Step 2/3 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.517] Step 3/3 coefs</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.519] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.519] Done.</span></span></code></pre></div>
 <div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
 <span><span class="co"># $coefs</span></span>
@@ -195,24 +192,24 @@ <h3 id="split-data-set">Split data set<a class="anchor" aria-label="anchor" href
 <span><span class="op">}</span></span>
 <span></span>
 <span><span class="va">results</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/lapply.html" class="external-link">lapply</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/split.html" class="external-link">split</a></span><span class="op">(</span><span class="va">iris</span>, <span class="va">iris</span><span class="op">$</span><span class="va">Species</span><span class="op">)</span>, FUN <span class="op">=</span> <span class="va">run_pipeline</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.192] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.193] Step 1/3 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.195] Step 2/3 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.199] Step 3/3 coefs</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.203] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.204] Done.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.222] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.223] Step 1/3 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.225] Step 2/3 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.227] Step 3/3 coefs</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.228] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.229] Done.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.232] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.233] Step 1/3 data</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.235] Step 2/3 fit</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.238] Step 3/3 coefs</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.239] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.239] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.654] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.655] Step 1/3 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.657] Step 2/3 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.661] Step 3/3 coefs</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.665] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.666] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.683] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.684] Step 1/3 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.686] Step 2/3 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.688] Step 3/3 coefs</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.690] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.690] Done.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.692] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.693] Step 1/3 data</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.695] Step 2/3 fit</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.698] Step 3/3 coefs</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.699] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:51.699] Done.</span></span></code></pre></div>
 <div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">results</span></span>
 <span><span class="co"># $setosa</span></span>
@@ -272,18 +269,18 @@ <h3 id="split-data-set">Split data set<a class="anchor" aria-label="anchor" href
 pipeline.</p>
 <div class="sourceCode" id="cb13"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.710] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.711] Step 1/9 data.setosa</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.713] Step 2/9 fit.setosa</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.716] Step 3/9 coefs.setosa</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.717] Step 4/9 data.versicolor</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.719] Step 5/9 fit.versicolor</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.722] Step 6/9 coefs.versicolor</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.723] Step 7/9 data.virginica</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.725] Step 8/9 fit.virginica</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.728] Step 9/9 coefs.virginica</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.729] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:56.730] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.167] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.168] Step 1/9 data.setosa</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.170] Step 2/9 fit.setosa</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.172] Step 3/9 coefs.setosa</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.174] Step 4/9 data.versicolor</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.176] Step 5/9 fit.versicolor</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.179] Step 6/9 coefs.versicolor</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.180] Step 7/9 data.virginica</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.182] Step 8/9 fit.virginica</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.185] Step 9/9 coefs.virginica</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.186] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.187] Done.</span></span></code></pre></div>
 <div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
 <span><span class="co"># $setosa</span></span>
@@ -415,19 +412,19 @@ <h3 id="combine-output">Combine output<a class="anchor" aria-label="anchor" href
 <div class="sourceCode" id="cb21"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>xVar <span class="op">=</span> <span class="st">"Sepal.Length"</span>, yVar <span class="op">=</span> <span class="st">"Sepal.Width"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">pip</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.260] Start run of 'my-pipeline' pipeline:</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.263] Step 1/10 data.setosa</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.265] Step 2/10 fit.setosa</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.268] Step 3/10 coefs.setosa</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.270] Step 4/10 data.versicolor</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.272] Step 5/10 fit.versicolor</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.275] Step 6/10 coefs.versicolor</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.277] Step 7/10 data.virginica</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.279] Step 8/10 fit.virginica</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.282] Step 9/10 coefs.virginica</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.284] Step 10/10 combine_coefs</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.286] Finished execution of steps.</span></span>
-<span><span class="co"># INFO  [2024-12-01 21:41:57.286] Done.</span></span></code></pre></div>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.680] Start run of 'my-pipeline' pipeline:</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.681] Step 1/10 data.setosa</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.683] Step 2/10 fit.setosa</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.686] Step 3/10 coefs.setosa</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.688] Step 4/10 data.versicolor</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.691] Step 5/10 fit.versicolor</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.694] Step 6/10 coefs.versicolor</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.696] Step 7/10 data.virginica</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.698] Step 8/10 fit.virginica</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.701] Step 9/10 coefs.virginica</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.702] Step 10/10 combine_coefs</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.704] Finished execution of steps.</span></span>
+<span><span class="co"># INFO  [2024-12-01 21:46:52.705] Done.</span></span></code></pre></div>
 <div class="sourceCode" id="cb22"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">pip</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span>
 <span><span class="co"># $combine_coefs</span></span>
diff --git a/docs/authors.html b/docs/authors.html
index 530bbb1..2046911 100644
--- a/docs/authors.html
+++ b/docs/authors.html
@@ -37,9 +37,7 @@
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/GroupSeq/">GroupSeq</a></li>
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/pipeflow/">pipeflow</a></li>
   </ul></li>
-      </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="search.json"></form></li>
-<li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
+      </ul><ul class="navbar-nav"><li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://rpahl.github.io/r-some-blog/" aria-label="blog"><span class="fa fas fa-blog"></span></a></li>
diff --git a/docs/index.html b/docs/index.html
index c8f6893..bad2b1d 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -66,9 +66,6 @@
 </li>
       </ul>
 <ul class="navbar-nav">
-<li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="search.json">
-</form></li>
 <li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml
index 2ad7bfb..d3d8a4f 100644
--- a/docs/pkgdown.yml
+++ b/docs/pkgdown.yml
@@ -8,7 +8,7 @@ articles:
   modify-pipeline: modify-pipeline.html
   self-modify-pipeline: self-modify-pipeline.html
   split-and-combine: split-and-combine.html
-last_built: 2024-12-01T20:41Z
+last_built: 2024-12-01T20:46Z
 urls:
   reference: https://github.com/rpahl/pipeflow/reference
   article: https://github.com/rpahl/pipeflow/articles
diff --git a/docs/reference/Pipeline.html b/docs/reference/Pipeline.html
index 0d6a960..48b9b1e 100644
--- a/docs/reference/Pipeline.html
+++ b/docs/reference/Pipeline.html
@@ -53,9 +53,7 @@
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/GroupSeq/">GroupSeq</a></li>
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/pipeflow/">pipeflow</a></li>
   </ul></li>
-      </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li>
-<li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
+      </ul><ul class="navbar-nav"><li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://rpahl.github.io/r-some-blog/" aria-label="blog"><span class="fa fas fa-blog"></span></a></li>
@@ -1802,11 +1800,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span> <span class="op">&lt;-</span> <span class="va">Pipeline</span><span class="op">$</span><span class="fu">new</span><span class="op">(</span><span class="st">"myPipe"</span>, data <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">2</span>, <span class="cn">NA</span>, <span class="fl">3</span>, <span class="fl">4</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"calc_mean"</span>, <span class="va">mean</span>, params <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>x <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, na.rm <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"calc_mean"</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.908] Start run of 'myPipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.933] Step 1/2 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.942] Step 2/2 calc_mean</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.961] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.962] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.666] Start run of 'myPipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.694] Step 1/2 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.703] Step 2/2 calc_mean</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.723] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.724] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2.5</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="va">p</span> <span class="op">&lt;-</span> <span class="va">Pipeline</span><span class="op">$</span><span class="fu">new</span><span class="op">(</span><span class="st">"myPipe"</span>, data <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">10</span><span class="op">)</span></span></span>
@@ -1823,21 +1821,21 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span> 2:     s1 &lt;function[1]&gt; function &lt;list[1]&gt;    data [NULL]   FALSE     s1</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>      description                time  state</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>           &lt;char&gt;              &lt;POSc&gt; &lt;char&gt;</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> 1:               2024-12-01 21:41:21    New</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> 2: multiply by 2 2024-12-01 21:41:21    New</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> 1:               2024-12-01 21:46:17    New</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> 2: multiply by 2 2024-12-01 21:46:17    New</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="va">p</span> <span class="op">&lt;-</span> <span class="va">Pipeline</span><span class="op">$</span><span class="fu">new</span><span class="op">(</span><span class="st">"myPipe"</span>, data <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>x <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">5</span>, y <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">5</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"prep_x"</span>, \<span class="op">(</span><span class="va">data</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span><span class="op">)</span> <span class="va">data</span><span class="op">$</span><span class="va">x</span>, group <span class="op">=</span> <span class="st">"prep"</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"prep_y"</span>, \<span class="op">(</span><span class="va">data</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span><span class="op">)</span> <span class="op">(</span><span class="va">data</span><span class="op">$</span><span class="va">y</span><span class="op">)</span><span class="op">^</span><span class="fl">2</span>, group <span class="op">=</span> <span class="st">"prep"</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"sum"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">prep_x</span>, <span class="va">y</span> <span class="op">=</span> <span class="op">~</span><span class="va">prep_y</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span>all <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.975] Start run of 'myPipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.976] Step 1/4 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.979] Step 2/4 prep_x</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.981] Step 3/4 prep_y</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.983] Step 4/4 sum</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.984] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:21.985] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.739] Start run of 'myPipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.740] Step 1/4 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.742] Step 2/4 prep_x</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.744] Step 3/4 prep_y</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.746] Step 4/4 sum</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.747] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.748] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $data</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>   x y</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> 1 1 1</span>
@@ -1917,12 +1915,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"step1"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="fl">2</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"step2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">step1</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="fl">2</span>, keepOut <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.118] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.119] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.121] Step 2/3 step1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.123] Step 3/3 step2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.124] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.125] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.887] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.892] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.894] Step 2/3 step1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.896] Step 3/3 step2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.897] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.898] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $step2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 5 6</span>
@@ -1947,14 +1945,14 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span> 4:  step3    data [NULL]   FALSE   mult    New</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> 5:  step4    data [NULL]   FALSE   mult    New</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.154] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.155] Step 1/5 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.157] Step 2/5 step1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.159] Step 3/5 step2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.160] Step 4/5 step3</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.162] Step 5/5 step4</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.163] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.164] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.924] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.925] Step 1/5 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.928] Step 2/5 step1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.930] Step 3/5 step2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.931] Step 4/5 step3</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.933] Step 5/5 step4</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.934] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:17.935] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span>all <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span> <span class="op">|&gt;</span> <span class="fu"><a href="https://rdrr.io/r/utils/str.html" class="external-link">str</a></span><span class="op">(</span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> List of 3</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>  $ data: int [1:2] 1 2</span>
@@ -2110,12 +2108,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add1"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="fl">1</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, <span class="va">y</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.273] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.274] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.276] Step 2/3 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.278] Step 3/3 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.279] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.280] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.047] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.048] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.050] Step 2/3 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.052] Step 3/3 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.053] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.053] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"add1"</span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2 3</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"add2"</span><span class="op">)</span></span></span>
@@ -2242,12 +2240,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add1"</span>, \<span class="op">(</span><span class="va">data</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, <span class="va">x</span> <span class="op">=</span> <span class="fl">1</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">data</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="fl">1</span>, <span class="va">y</span> <span class="op">=</span> <span class="fl">2</span>, <span class="va">z</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">y</span> <span class="op">+</span> <span class="va">z</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.322] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.323] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.325] Step 2/3 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.326] Step 3/3 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.328] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.328] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.092] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.093] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.095] Step 2/3 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.097] Step 3/3 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.098] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.099] Done.</span>
 <span class="r-in"><span><span class="va">add1</span> <span class="op">&lt;-</span> <span class="va">p</span><span class="op">$</span><span class="fu">get_step</span><span class="op">(</span><span class="st">"add1"</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu"><a href="https://rdrr.io/r/base/print.html" class="external-link">print</a></span><span class="op">(</span><span class="va">add1</span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>      step           fun funcName    params depends    out keepOut  group</span>
@@ -2255,12 +2253,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span> 1:   add1 &lt;function[1]&gt; function &lt;list[2]&gt;    data    2,3   FALSE   add1</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>    description                time  state</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>         &lt;char&gt;              &lt;POSc&gt; &lt;char&gt;</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> 1:             2024-12-01 21:41:22   Done</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> 1:             2024-12-01 21:46:18   Done</span>
 <span class="r-in"><span><span class="va">add1</span><span class="op">[[</span><span class="st">"params"</span><span class="op">]</span><span class="op">]</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [[1]]</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [[1]]$data</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> ~data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> &lt;environment: 0x000002328176fc38&gt;</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> &lt;environment: 0x00000296abb38438&gt;</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [[1]]$x</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 1</span>
@@ -2357,12 +2355,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add1"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="fl">1</span>, <span class="va">data</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">data</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="fl">1</span>, <span class="va">data</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">data</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.370] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.371] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.373] Step 2/3 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.375] Step 3/3 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.376] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.376] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.141] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.142] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.144] Step 2/3 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.146] Step 3/3 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.147] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.147] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"add1"</span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"add2"</span><span class="op">)</span></span></span>
@@ -2373,12 +2371,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">set_params</span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>x <span class="op">=</span> <span class="fl">3</span><span class="op">)</span><span class="op">)</span></span></span>
 <span class="r-msg co"><span class="r-pr">#&gt;</span> skipping setting parameters x at locked step 'add1'</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.396] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.397] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.399] Step 2/3 add1 - skip 'locked' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.400] Step 3/3 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.401] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.402] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.168] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.169] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.171] Step 2/3 add1 - skip 'locked' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.172] Step 3/3 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.173] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.173] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"add1"</span><span class="op">)</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">get_out</span><span class="op">(</span><span class="st">"add2"</span><span class="op">)</span></span></span>
@@ -2508,25 +2506,25 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, <span class="va">y</span> <span class="op">=</span> <span class="fl">2</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"mult"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="fl">1</span>, <span class="va">y</span> <span class="op">=</span> <span class="fl">2</span><span class="op">)</span> <span class="va">x</span> <span class="op">*</span> <span class="va">y</span>, keepOut <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.462] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.463] Step 1/4 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.465] Step 2/4 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.466] Step 3/4 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.468] Step 4/4 mult</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.469] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.469] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.233] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.234] Step 1/4 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.236] Step 2/4 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.237] Step 3/4 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.239] Step 4/4 mult</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.240] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.240] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $mult</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">replace_step</span><span class="op">(</span><span class="st">"mult"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span>, <span class="va">y</span> <span class="op">=</span> <span class="op">~</span><span class="va">add2</span><span class="op">)</span> <span class="va">x</span> <span class="op">*</span> <span class="va">y</span>, keepOut <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.472] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.473] Step 1/4 data - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.474] Step 2/4 add1 - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.475] Step 3/4 add2 - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.476] Step 4/4 mult</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.477] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.477] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.243] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.244] Step 1/4 data - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.245] Step 2/4 add1 - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.246] Step 3/4 add2 - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.246] Step 4/4 mult</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.248] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.248] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $mult</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 6</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
@@ -2541,12 +2539,12 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"f1"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="fl">1</span><span class="op">)</span> <span class="va">x</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"f2"</span>, \<span class="op">(</span><span class="va">y</span> <span class="op">=</span> <span class="fl">1</span><span class="op">)</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.493] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.494] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.495] Step 2/3 f1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.496] Step 3/3 f2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.498] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.498] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.264] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.265] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.266] Step 2/3 f1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.268] Step 3/3 f2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.269] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.269] Done.</span>
 <span class="r-in"><span><span class="va">p</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>      step depends    out keepOut  group  state</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>    &lt;char&gt;  &lt;list&gt; &lt;list&gt;  &lt;lgcl&gt; &lt;char&gt; &lt;char&gt;</span>
@@ -2571,13 +2569,13 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span>, <span class="va">z</span> <span class="op">=</span> <span class="fl">2</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">z</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"final"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span>, <span class="va">y</span> <span class="op">=</span> <span class="op">~</span><span class="va">add2</span><span class="op">)</span> <span class="va">x</span> <span class="op">*</span> <span class="va">y</span>, keepOut <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.508] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.509] Step 1/4 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.511] Step 2/4 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.513] Step 3/4 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.515] Step 4/4 final</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.516] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.517] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.280] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.280] Step 1/4 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.283] Step 2/4 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.285] Step 3/4 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.287] Step 4/4 final</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.288] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.288] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $final</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 8</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
@@ -2590,25 +2588,25 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span> 3:   add2      add1      4   FALSE   add2 Outdated</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> 4:  final add1,add2      8    TRUE  final Outdated</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.524] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.525] Step 1/4 data - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.526] Step 2/4 add1 - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.526] Step 3/4 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.528] Step 4/4 final</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.529] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.530] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.295] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.296] Step 1/4 data - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.297] Step 2/4 add1 - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.298] Step 3/4 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.299] Step 4/4 final</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.301] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.301] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $final</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 12</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span>cleanUnkept <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span>  <span class="co"># clean up temporary results</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.531] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.532] Step 1/4 data - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.533] Step 2/4 add1 - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.534] Step 3/4 add2 - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.535] Step 4/4 final - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.535] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.536] Clean temporary results.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.536] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.303] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.303] Step 1/4 data - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.304] Step 2/4 add1 - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.305] Step 3/4 add2 - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.306] Step 4/4 final - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.306] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.307] Clean temporary results.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.307] Done.</span>
 <span class="r-in"><span><span class="va">p</span></span></span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>      step   depends    out keepOut  group    state</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>    &lt;char&gt;    &lt;list&gt; &lt;list&gt;  &lt;lgcl&gt; &lt;char&gt;   &lt;char&gt;</span>
@@ -2627,17 +2625,17 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span>    <span class="op">}</span></span></span>
 <span class="r-in"><span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.543] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.544] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.546] Step 2/3 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.547] Step 3/3 new_pipe</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.550] Abort pipeline execution and restart on new.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.551] Start run of 'new_pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.552] Step 1/3 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.554] Step 2/3 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.556] Step 3/3 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.557] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:22.557] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.314] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.315] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.317] Step 2/3 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.319] Step 3/3 new_pipe</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.322] Abort pipeline execution and restart on new.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.322] Start run of 'new_pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.323] Step 1/3 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.325] Step 2/3 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.327] Step 3/3 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.328] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:18.329] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $add2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 5</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
@@ -2663,28 +2661,28 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span>, <span class="va">z</span> <span class="op">=</span> <span class="fl">2</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">z</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"mult"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span>, <span class="va">y</span> <span class="op">=</span> <span class="op">~</span><span class="va">add2</span><span class="op">)</span> <span class="va">x</span> <span class="op">*</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run_step</span><span class="op">(</span><span class="st">"add2"</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.633] Start step run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.634] Step 1/3 data (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.636] Step 2/3 add1 (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.638] Step 3/3 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.640] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.640] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.408] Start step run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.409] Step 1/3 data (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.411] Step 2/3 add1 (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.413] Step 3/3 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.414] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.415] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run_step</span><span class="op">(</span><span class="st">"add2"</span>, downstream <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.655] Start step run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.655] Step 1/4 data (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.658] Step 2/4 add1 (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.660] Step 3/4 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.664] Step 4/4 mult (downstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.665] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.666] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.432] Start step run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.432] Step 1/4 data (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.434] Step 2/4 add1 (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.436] Step 3/4 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.438] Step 4/4 mult (downstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.439] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.439] Done.</span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run_step</span><span class="op">(</span><span class="st">"mult"</span>, upstream <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.667] Start step run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.668] Step 1/4 data (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.670] Step 2/4 add1 (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.672] Step 3/4 add2 (upstream)</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.674] Step 4/4 mult</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.675] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.675] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.441] Start step run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.441] Step 1/4 data (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.444] Step 2/4 add1 (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.446] Step 3/4 add2 (upstream)</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.448] Step 4/4 mult</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.449] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.449] Done.</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="co">## ------------------------------------------------</span></span></span>
 <span class="r-in"><span><span class="co">## Method `Pipeline$set_data`</span></span></span>
@@ -2693,21 +2691,21 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span> <span class="op">&lt;-</span> <span class="va">Pipeline</span><span class="op">$</span><span class="fu">new</span><span class="op">(</span><span class="st">"pipe"</span>, data <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add1"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, <span class="va">y</span> <span class="op">=</span> <span class="fl">1</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">y</span>, keepOut <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.679] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.680] Step 1/2 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.682] Step 2/2 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.683] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.684] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.453] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.454] Step 1/2 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.456] Step 2/2 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.457] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.457] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $add1</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">set_data</span><span class="op">(</span><span class="fl">3</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.687] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.688] Step 1/2 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.690] Step 2/2 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.691] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.692] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.460] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.461] Step 1/2 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.464] Step 2/2 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.465] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.465] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $add1</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 4</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
@@ -2735,18 +2733,18 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span> 8: add1.c        data.c [NULL]    TRUE      c Outdated</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> 9: mult.c data.c,add1.c [NULL]    TRUE      c Outdated</span>
 <span class="r-in"><span><span class="va">p3</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span> <span class="op">|&gt;</span> <span class="fu"><a href="https://rdrr.io/r/utils/str.html" class="external-link">str</a></span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.714] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.715] Step 1/9 data.a</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.717] Step 2/9 add1.a</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.719] Step 3/9 mult.a</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.721] Step 4/9 data.b</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.723] Step 5/9 add1.b</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.725] Step 6/9 mult.b</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.726] Step 7/9 data.c</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.728] Step 8/9 add1.c</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.730] Step 9/9 mult.c</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.731] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.732] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.488] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.488] Step 1/9 data.a</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.490] Step 2/9 add1.a</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.492] Step 3/9 mult.a</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.494] Step 4/9 data.b</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.496] Step 5/9 add1.b</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.498] Step 6/9 mult.b</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.500] Step 7/9 data.c</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.502] Step 8/9 add1.c</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.503] Step 9/9 mult.c</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.505] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.505] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> List of 3</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>  $ a:List of 2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>   ..$ add1.a: num 2</span>
@@ -2776,18 +2774,18 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span> 8: add1.c        data.c [NULL]    TRUE add1.c Outdated</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> 9: mult.c data.c,add1.c [NULL]    TRUE mult.c Outdated</span>
 <span class="r-in"><span><span class="va">p3</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span> <span class="op">|&gt;</span> <span class="fu"><a href="https://rdrr.io/r/utils/str.html" class="external-link">str</a></span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.757] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.758] Step 1/9 data.a</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.760] Step 2/9 add1.a</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.762] Step 3/9 mult.a</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.764] Step 4/9 data.b</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.766] Step 5/9 add1.b</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.767] Step 6/9 mult.b</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.769] Step 7/9 data.c</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.771] Step 8/9 add1.c</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.773] Step 9/9 mult.c</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.774] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.775] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.531] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.532] Step 1/9 data.a</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.534] Step 2/9 add1.a</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.536] Step 3/9 mult.a</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.537] Step 4/9 data.b</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.539] Step 5/9 add1.b</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.541] Step 6/9 mult.b</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.543] Step 7/9 data.c</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.545] Step 8/9 add1.c</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.546] Step 9/9 mult.c</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.548] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.548] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> List of 6</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>  $ add1.a: num 2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span>  $ mult.a: num 2</span>
@@ -2805,13 +2803,13 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add2"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, <span class="va">y</span> <span class="op">=</span> <span class="fl">2</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"mult"</span>, \<span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="op">~</span><span class="va">add1</span>, <span class="va">y</span> <span class="op">=</span> <span class="op">~</span><span class="va">add2</span><span class="op">)</span> <span class="va">x</span> <span class="op">*</span> <span class="va">y</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span><span class="op">$</span><span class="fu">collect_out</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.783] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.784] Step 1/4 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.786] Step 2/4 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.788] Step 3/4 add2</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.790] Step 4/4 mult</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.791] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:25.792] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.557] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.558] Step 1/4 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.560] Step 2/4 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.562] Step 3/4 add2</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.566] Step 4/4 mult</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.567] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:21.568] Done.</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> $add1</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> [1] 2</span>
 <span class="r-out co"><span class="r-pr">#&gt;</span> </span>
diff --git a/docs/reference/index.html b/docs/reference/index.html
index 0bdbf62..17d1b0c 100644
--- a/docs/reference/index.html
+++ b/docs/reference/index.html
@@ -37,9 +37,7 @@
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/GroupSeq/">GroupSeq</a></li>
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/pipeflow/">pipeflow</a></li>
   </ul></li>
-      </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li>
-<li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
+      </ul><ul class="navbar-nav"><li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://rpahl.github.io/r-some-blog/" aria-label="blog"><span class="fa fas fa-blog"></span></a></li>
diff --git a/docs/reference/pipelineAliases.html b/docs/reference/pipelineAliases.html
index 7dfbf8b..bc18f26 100644
--- a/docs/reference/pipelineAliases.html
+++ b/docs/reference/pipelineAliases.html
@@ -39,9 +39,7 @@
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/GroupSeq/">GroupSeq</a></li>
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/pipeflow/">pipeflow</a></li>
   </ul></li>
-      </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li>
-<li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
+      </ul><ul class="navbar-nav"><li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://rpahl.github.io/r-some-blog/" aria-label="blog"><span class="fa fas fa-blog"></span></a></li>
diff --git a/docs/reference/set_log_layout.html b/docs/reference/set_log_layout.html
index 6efc545..0520c3e 100644
--- a/docs/reference/set_log_layout.html
+++ b/docs/reference/set_log_layout.html
@@ -37,9 +37,7 @@
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/GroupSeq/">GroupSeq</a></li>
     <li><a class="external-link dropdown-item" href="https://rpahl.github.io/pipeflow/">pipeflow</a></li>
   </ul></li>
-      </ul><ul class="navbar-nav"><li class="nav-item"><form class="form-inline" role="search">
- <input class="form-control" type="search" name="search-input" id="search-input" autocomplete="off" aria-label="Search site" placeholder="Search for" data-search-index="../search.json"></form></li>
-<li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
+      </ul><ul class="navbar-nav"><li class="nav-item"><a class="nav-link" href="https://github.com/rpahl/pipeflow/" aria-label="github"><span class="fa fab fa-github"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://stackoverflow.com/users/8120617/rpahl" aria-label="stackoverflow"><span class="fa fab fa-stack-overflow"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://bsky.app/profile/rpahl.bsky.social" aria-label="bluesky"><span class="fa fab fa-bluesky"></span></a></li>
 <li class="nav-item"><a class="external-link nav-link" href="https://rpahl.github.io/r-some-blog/" aria-label="blog"><span class="fa fas fa-blog"></span></a></li>
@@ -82,11 +80,11 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
     <div class="sourceCode"><pre class="sourceCode r"><code><span class="r-in"><span><span class="va">p</span> <span class="op">&lt;-</span> <span class="va"><a href="Pipeline.html">Pipeline</a></span><span class="op">$</span><span class="fu">new</span><span class="op">(</span><span class="st">"pipe"</span>, data <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fl">2</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">add</span><span class="op">(</span><span class="st">"add1"</span>, \<span class="op">(</span><span class="va">data</span> <span class="op">=</span> <span class="op">~</span><span class="va">data</span>, <span class="va">x</span> <span class="op">=</span> <span class="fl">1</span><span class="op">)</span> <span class="va">x</span> <span class="op">+</span> <span class="va">data</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.126] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.127] Step 1/2 data</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.129] Step 2/2 add1</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.130] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.131] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.778] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.779] Step 1/2 data</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.781] Step 2/2 add1</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.782] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.783] Done.</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="va">lg</span> <span class="op">&lt;-</span> <span class="fu">set_log_layout</span><span class="op">(</span><span class="st">"json"</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="fu"><a href="https://rdrr.io/r/base/print.html" class="external-link">print</a></span><span class="op">(</span><span class="va">lg</span><span class="op">)</span></span></span>
@@ -96,19 +94,19 @@ <h2 id="ref-examples">Examples<a class="anchor" aria-label="anchor" href="#ref-e
 <span class="r-out co"><span class="r-pr">#&gt;</span>   [[1]]: &lt;AppenderConsole&gt; [all] -&gt; console</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Start run of 'pipe' pipeline:"}</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Step 1/2 data - skip 'done' step"}</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Step 2/2 add1 - skip 'done' step"}</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Finished execution of steps."}</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:41:29+1:00","message":"Done."}</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:46:24+1:00","message":"Start run of 'pipe' pipeline:"}</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:46:24+1:00","message":"Step 1/2 data - skip 'done' step"}</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:46:24+1:00","message":"Step 2/2 add1 - skip 'done' step"}</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:46:24+1:00","message":"Finished execution of steps."}</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> {"application":"pipeflow","level":"info","time":"2024-12-01T21:46:24+1:00","message":"Done."}</span>
 <span class="r-in"><span></span></span>
 <span class="r-in"><span><span class="fu">set_log_layout</span><span class="op">(</span><span class="st">"text"</span><span class="op">)</span></span></span>
 <span class="r-in"><span><span class="va">p</span><span class="op">$</span><span class="fu">run</span><span class="op">(</span><span class="op">)</span></span></span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.169] Start run of 'pipe' pipeline:</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.170] Step 1/2 data - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.171] Step 2/2 add1 - skip 'done' step</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.171] Finished execution of steps.</span>
-<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:41:29.172] Done.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.821] Start run of 'pipe' pipeline:</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.822] Step 1/2 data - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.823] Step 2/2 add1 - skip 'done' step</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.823] Finished execution of steps.</span>
+<span class="r-out co"><span class="r-pr">#&gt;</span> INFO  [2024-12-01 21:46:24.824] Done.</span>
 </code></pre></div>
     </div>
   </main><aside class="col-md-3"><nav id="toc" aria-label="Table of contents"><h2>On this page</h2>
diff --git a/docs/search.json b/docs/search.json
index 3608847..fce3613 100644
--- a/docs/search.json
+++ b/docs/search.json
@@ -1 +1 @@
-[{"path":"https://github.com/rpahl/pipeflow/articles/collect-output.html","id":"flag-output-steps","dir":"Articles","previous_headings":"","what":"Flag output steps","title":"Collecting output","text":"Output steps flagged settting keepOut argument TRUE adding step pipeline. following example, want keep output steps data_summary, model_summary, model_plot. Looking pipeline, see steps data_summary, model-summary, model_plot flagged accordingly (see column keepOut). Graphically, steps flagged keepOut = TRUE displayed circle shape “normal” steps shown rectangle boxes. Now let’s run collect output flagged steps using collect_out method, returns list output flagged steps. expected, output list contains output flagged steps.","code":"library(pipeflow) library(ggplot2)  pip <- pipe_new(         \"my-pipeline\",         data = airquality     ) |>      pipe_add(         \"data_prep\",         function(data = ~data) {             replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)         }     ) |>      pipe_add(         \"data_summary\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             data[, c(xVar, yVar)]         },         keepOut = TRUE              # <- keep this     ) |>      pipe_add(         \"model_fit\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"model_summary\",         function(             fit = ~model_fit         ) {             summary(fit)         },         keepOut = TRUE              # <- keep this     ) |>      pipe_add(         \"model_plot\",         function(             model = ~model_fit,             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\",             title = \"Linear model fit\"         ) {             coeffs <- coefficients(model)             ggplot(data) +                 geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) +                 geom_abline(intercept = coeffs[1], slope = coeffs[2]) +                 labs(title = title)         },         keepOut = TRUE              # <- keep this     ) pip #             step             depends    out keepOut         group  state #           <char>              <list> <list>  <lgcl>        <char> <char> # 1:          data                     [NULL]   FALSE          data    New # 2:     data_prep                data [NULL]   FALSE     data_prep    New # 3:  data_summary           data_prep [NULL]    TRUE  data_summary    New # 4:     model_fit           data_prep [NULL]   FALSE     model_fit    New # 5: model_summary           model_fit [NULL]    TRUE model_summary    New # 6:    model_plot model_fit,data_prep [NULL]    TRUE    model_plot    New pip$run() # INFO  [2024-12-01 21:41:31.888] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:31.916] Step 1/6 data # INFO  [2024-12-01 21:41:31.925] Step 2/6 data_prep # INFO  [2024-12-01 21:41:31.948] Step 3/6 data_summary # INFO  [2024-12-01 21:41:31.950] Step 4/6 model_fit # INFO  [2024-12-01 21:41:31.954] Step 5/6 model_summary # INFO  [2024-12-01 21:41:31.956] Step 6/6 model_plot # INFO  [2024-12-01 21:41:31.968] Finished execution of steps. # INFO  [2024-12-01 21:41:31.968] Done.  out <- pip$collect_out()  names(out) # [1] \"data_summary\"  \"model_summary\" \"model_plot\" str(out, max.level = 1) # List of 3 #  $ data_summary :'data.frame':    153 obs. of  2 variables: #  $ model_summary:List of 12 #   ..- attr(*, \"class\")= chr \"summary.lm\" #  $ model_plot   :List of 11 #   ..- attr(*, \"class\")= chr [1:2] \"gg\" \"ggplot\""},{"path":"https://github.com/rpahl/pipeflow/articles/collect-output.html","id":"grouping-output-steps","dir":"Articles","previous_headings":"","what":"Grouping output steps","title":"Collecting output","text":"Often certain output steps related grouped together. can achieved conveniently setting group argument adding step pipeline. Let’s illustrate slightly modifying previous example. Looking pipeline, defined groups shown group column. see, default, group identical step name, , step represents trivial case one-sized group. , run pipeline collect output. can see, output related modelling grouped one sublist named Model.","code":"pip <- Pipeline$new(\"my-pipeline\", data = airquality) |>      pipe_add(         \"data_prep\",         function(data = ~data) {             replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)         }     ) |>      pipe_add(         \"used_data\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             data[, c(xVar, yVar)]         },         keepOut = TRUE,         group = \"Data\"                 # <- define 'Data' group here     ) |>      pipe_add(         \"model_fit\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"model_summary\",         function(             fit = ~model_fit         ) {             summary(fit)         },         keepOut = TRUE,         group = \"Model\"                # <- define 'Model' group here     ) |>      pipe_add(         \"model_plot\",         function(             model = ~model_fit,             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\",             title = \"Linear model fit\"         ) {             coeffs <- coefficients(model)             ggplot(data) +                 geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) +                 geom_abline(intercept = coeffs[1], slope = coeffs[2]) +                 labs(title = title)         },         keepOut = TRUE,         group = \"Model\"                # <- define 'Model' group here     ) pip #             step             depends    out keepOut     group  state #           <char>              <list> <list>  <lgcl>    <char> <char> # 1:          data                     [NULL]   FALSE      data    New # 2:     data_prep                data [NULL]   FALSE data_prep    New # 3:     used_data           data_prep [NULL]    TRUE      Data    New # 4:     model_fit           data_prep [NULL]   FALSE model_fit    New # 5: model_summary           model_fit [NULL]    TRUE     Model    New # 6:    model_plot model_fit,data_prep [NULL]    TRUE     Model    New pip$run() # INFO  [2024-12-01 21:41:32.198] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:32.199] Step 1/6 data # INFO  [2024-12-01 21:41:32.202] Step 2/6 data_prep # INFO  [2024-12-01 21:41:32.205] Step 3/6 used_data # INFO  [2024-12-01 21:41:32.206] Step 4/6 model_fit # INFO  [2024-12-01 21:41:32.229] Step 5/6 model_summary # INFO  [2024-12-01 21:41:32.231] Step 6/6 model_plot # INFO  [2024-12-01 21:41:32.236] Finished execution of steps. # INFO  [2024-12-01 21:41:32.237] Done.  out <- pip$collect_out()  names(out) # [1] \"Data\"  \"Model\" str(out, max.level = 2) # List of 2 #  $ Data :'data.frame':    153 obs. of  2 variables: #   ..$ Temp.Celsius: num [1:153] 19.4 22.2 23.3 16.7 13.3 ... #   ..$ Ozone       : int [1:153] 41 36 12 18 NA 28 23 19 8 NA ... #  $ Model:List of 2 #   ..$ model_summary:List of 12 #   .. ..- attr(*, \"class\")= chr \"summary.lm\" #   ..$ model_plot   :List of 11 #   .. ..- attr(*, \"class\")= chr [1:2] \"gg\" \"ggplot\""},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"define-two-pipelines","dir":"Articles","previous_headings":"","what":"Define two pipelines","title":"Combining pipelines","text":"Let’s define one pipeline used data_preprocessing one modeling. Data preprocessing pipeline: Modeling pipeline: Graphically, two pipelines look follows:","code":"library(pipeflow) library(ggplot2)  pip1 <- pipe_new(         \"preprocessing\",         data = airquality     ) |>      pipe_add(         \"data_prep\",         function(data = ~data) {             replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)         }     ) |>      pipe_add(         \"standardize\",         function(             data = ~data_prep,             yVar = \"Ozone\"         ) {             data[, yVar] <- scale(data[, yVar])             data         }     ) pip1 #           step   depends    out keepOut       group  state #         <char>    <list> <list>  <lgcl>      <char> <char> # 1:        data           [NULL]   FALSE        data    New # 2:   data_prep      data [NULL]   FALSE   data_prep    New # 3: standardize data_prep [NULL]   FALSE standardize    New pip2 <- pipe_new(         \"modeling\",         data = airquality     ) |>      pipe_add(         \"fit\",         function(             data = ~data,             xVar = \"Temp\",             yVar = \"Ozone\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"plot\",         function(             model = ~fit,             data = ~data,             xVar = \"Temp\",             yVar = \"Ozone\",             title = \"Linear model fit\"         ) {             coeffs <- coefficients(model)             ggplot(data) +                 geom_point(aes(.data[[xVar]], .data[[yVar]])) +                 geom_abline(intercept = coeffs[1], slope = coeffs[2]) +                 labs(title = title)         }     ) pip2 #      step  depends    out keepOut  group  state #    <char>   <list> <list>  <lgcl> <char> <char> # 1:   data          [NULL]   FALSE   data    New # 2:    fit     data [NULL]   FALSE    fit    New # 3:   plot fit,data [NULL]   FALSE   plot    New"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"combine-pipelines","dir":"Articles","previous_headings":"","what":"Combine pipelines","title":"Combining pipelines","text":"Next combine two pipelines. can using append function. First , note data step second pipeline appended name second pipeline. particular, first step second pipeline renamed data data.modeling (line 4 step column) likewise dependencies second pipeline updated (see lines 5-6 depends column). , appending two pipelines, pipeflow ensures step names remain unique resulting combined pipeline therefore automatically renames duplicated step names necessary. Now, can also seen graphical representation pipeline, two pipelines yet connected. make actual use combined pipeline, therefore want use output first pipeline input second pipeline, , want use output standardize step data parameter input data.modeling step. One way achieve use replace function described earlier vignette modify pipeline, example:","code":"pip <- pip1$append(pip2)  pip #             step           depends    out keepOut       group  state #           <char>            <list> <list>  <lgcl>      <char> <char> # 1:          data                   [NULL]   FALSE        data    New # 2:     data_prep              data [NULL]   FALSE   data_prep    New # 3:   standardize         data_prep [NULL]   FALSE standardize    New # 4: data.modeling                   [NULL]   FALSE        data    New # 5:           fit     data.modeling [NULL]   FALSE         fit    New # 6:          plot fit,data.modeling [NULL]   FALSE        plot    New pip$replace_step(\"data.modeling\", function(data = ~standardize) data)  pip #             step           depends    out keepOut         group    state #           <char>            <list> <list>  <lgcl>        <char>   <char> # 1:          data                   [NULL]   FALSE          data      New # 2:     data_prep              data [NULL]   FALSE     data_prep      New # 3:   standardize         data_prep [NULL]   FALSE   standardize      New # 4: data.modeling       standardize [NULL]   FALSE data.modeling      New # 5:           fit     data.modeling [NULL]   FALSE           fit Outdated # 6:          plot fit,data.modeling [NULL]   FALSE          plot Outdated"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"relative-indexing","dir":"Articles","previous_headings":"Combine pipelines","what":"Relative indexing","title":"Combining pipelines","text":"Since name last step might always known1, pipeflow package also provides relative position indexing mechanism, allows rewrite command follows: Generally speaking, relative indexing mechanism allows refer steps positioned current step. index ~-1 can interpreted “go one step back”, ~-2 “go two steps back”, . Since scenario connecting two pipelines common avoid replacement steps manually, append function actually provides argument outAsIn allows appending “connecting” pipelines one go: inspect data.modeling step, see “hood” original step indeed replaced output last step first pipeline using relative indexing mechanism manually .","code":"pip$replace_step(\"data.modeling\", function(data = ~-1) data)  pip #             step           depends    out keepOut         group    state #           <char>            <list> <list>  <lgcl>        <char>   <char> # 1:          data                   [NULL]   FALSE          data      New # 2:     data_prep              data [NULL]   FALSE     data_prep      New # 3:   standardize         data_prep [NULL]   FALSE   standardize      New # 4: data.modeling       standardize [NULL]   FALSE data.modeling      New # 5:           fit     data.modeling [NULL]   FALSE           fit Outdated # 6:          plot fit,data.modeling [NULL]   FALSE          plot Outdated pip <- pip1$append(pip2, outAsIn = TRUE)  pip #             step           depends    out keepOut         group    state #           <char>            <list> <list>  <lgcl>        <char>   <char> # 1:          data                   [NULL]   FALSE          data      New # 2:     data_prep              data [NULL]   FALSE     data_prep      New # 3:   standardize         data_prep [NULL]   FALSE   standardize      New # 4: data.modeling       standardize [NULL]   FALSE data.modeling      New # 5:           fit     data.modeling [NULL]   FALSE           fit Outdated # 6:          plot fit,data.modeling [NULL]   FALSE          plot Outdated pip$get_step(\"data.modeling\")[[\"fun\"]] # [[1]] # function (data = ~-1)  # data # <bytecode: 0x000001c01370b3f8> # <environment: 0x000001c00f2b05a8>"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"run-combined-pipeline","dir":"Articles","previous_headings":"","what":"Run combined pipeline","title":"Combining pipelines","text":"Let’s now run combined pipeline inspect plot.  can see, plot shows linear model fit using standardized data. can now go ahead example change x-variable model rerun pipeline.  creating pipelines, usually lot steps calculating intermediate results steps contain final results interested . example, interested final plot output. real-world scenario, pipeline contain many steps interest us. see conveniently tag, collect possibly group output final steps, see next vignette Collecting output.","code":"pip$run() # INFO  [2024-12-01 21:41:36.693] Start run of 'preprocessing.modeling' pipeline: # INFO  [2024-12-01 21:41:36.763] Step 1/6 data # INFO  [2024-12-01 21:41:36.772] Step 2/6 data_prep # INFO  [2024-12-01 21:41:36.788] Step 3/6 standardize # INFO  [2024-12-01 21:41:36.791] Step 4/6 data.modeling # INFO  [2024-12-01 21:41:36.793] Step 5/6 fit # INFO  [2024-12-01 21:41:36.797] Step 6/6 plot # INFO  [2024-12-01 21:41:36.810] Finished execution of steps. # INFO  [2024-12-01 21:41:36.811] Done. pip$get_out(\"plot\") # Warning: Removed 37 rows containing missing values or values outside the scale range # (`geom_point()`). pip$set_params(list(xVar = \"Temp.Celsius\")) pip$run() # INFO  [2024-12-01 21:41:37.360] Start run of 'preprocessing.modeling' pipeline: # INFO  [2024-12-01 21:41:37.361] Step 1/6 data - skip 'done' step # INFO  [2024-12-01 21:41:37.362] Step 2/6 data_prep - skip 'done' step # INFO  [2024-12-01 21:41:37.362] Step 3/6 standardize - skip 'done' step # INFO  [2024-12-01 21:41:37.363] Step 4/6 data.modeling - skip 'done' step # INFO  [2024-12-01 21:41:37.364] Step 5/6 fit # INFO  [2024-12-01 21:41:37.368] Step 6/6 plot # INFO  [2024-12-01 21:41:37.385] Finished execution of steps. # INFO  [2024-12-01 21:41:37.386] Done. pip$get_out(\"plot\") # Warning: Removed 37 rows containing missing values or values outside the scale range # (`geom_point()`)."},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"a-simple-example-to-get-started","dir":"Articles","previous_headings":"","what":"A simple example to get started","title":"Get started with pipeflow","text":"example, ’ll use base R’s airquality dataset. goal create analysis pipeline performs following steps: add new data column Temp.Celsius containing temperature degrees Celsius fit linear model data plot data model fit. following, ’ll show define run pipeline, inspect output specifics steps, finally re-run pipeline different parameter settings, one selling points using pipeline.","code":"head(airquality) #   Ozone Solar.R Wind Temp Month Day # 1    41     190  7.4   67     5   1 # 2    36     118  8.0   72     5   2 # 3    12     149 12.6   74     5   3 # 4    18     313 11.5   62     5   4 # 5    NA      NA 14.3   56     5   5 # 6    28      NA 14.9   66     5   6"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"define-the-pipeline","dir":"Articles","previous_headings":"A simple example to get started","what":"Define the pipeline","title":"Get started with pipeflow","text":"easier understanding, go step step. First, create new pipeline name “-pipeline” airquality dataset input data. Let’s view pipeline ’s table form initialization. , step represented one row table step column lists name step state column shows current state step. now, don’t worry columns - explained later. can see, data always first step pipeline initially state New, means pipeline run yet. Next, add step called data_prep, consists function takes output data step first argument, adds new column returns modified data output. refer output earlier pipeline step, always use tilde (~) operator. , let’s inspect pipeline. second step called data_prep added depends data step can seen column depends, list dependencies step. Next, add step called model_fit, fits linear model data. function takes output data_prep defines parameter xVar, used specify variable used predictor linear model. Lastly, add step called model_plot, plots data linear model fit. function uses output model_fit data_prep step. also defines xVar parameter parameter title, used title plot. completes pipeline, now looks follows: last line, see model_plot step depends model_fit data_prep step. visualize pipeline, can use get_graph() method, returns list arguments can passed visNetwork function visNetwork package.","code":"library(pipeflow) library(ggplot2)  pip <- Pipeline$new(\"my-pipeline\", data = airquality) pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data         [NULL]   FALSE   data    New pip$add(     \"data_prep\",     function(data = ~data) {         replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)     } ) pip #         step depends    out keepOut     group  state #       <char>  <list> <list>  <lgcl>    <char> <char> # 1:      data         [NULL]   FALSE      data    New # 2: data_prep    data [NULL]   FALSE data_prep    New pip$add(     \"model_fit\",     function(         data = ~data_prep,         xVar = \"Temp.Celsius\"     ) {         lm(paste(\"Ozone ~\", xVar), data = data)     } ) pip$add(     \"model_plot\",     function(         model = ~model_fit,         data = ~data_prep,         xVar = \"Temp.Celsius\",         title = \"Linear model fit\"     ) {         coeffs <- coefficients(model)         ggplot(data) +             geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) +             geom_abline(intercept = coeffs[1], slope = coeffs[2]) +             labs(title = title)     } ) pip #          step             depends    out keepOut      group  state #        <char>              <list> <list>  <lgcl>     <char> <char> # 1:       data                     [NULL]   FALSE       data    New # 2:  data_prep                data [NULL]   FALSE  data_prep    New # 3:  model_fit           data_prep [NULL]   FALSE  model_fit    New # 4: model_plot model_fit,data_prep [NULL]   FALSE model_plot    New library(visNetwork) do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\")"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"run-the-pipeline-and-inspect-the-output","dir":"Articles","previous_headings":"A simple example to get started","what":"Run the pipeline and inspect the output","title":"Get started with pipeflow","text":"run pipeline, simply call run() method pipeline, produce following output: Let’s inspect pipeline . can see state steps changed New Done, graphically represented color change blue green. addition, output added column. can inspect output specific step calling get_out() method pipeline.","code":"pip$run() # INFO  [2024-12-01 21:41:41.560] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:41.586] Step 1/4 data # INFO  [2024-12-01 21:41:41.597] Step 2/4 data_prep # INFO  [2024-12-01 21:41:41.620] Step 3/4 model_fit # INFO  [2024-12-01 21:41:41.623] Step 4/4 model_plot # INFO  [2024-12-01 21:41:41.635] Finished execution of steps. # INFO  [2024-12-01 21:41:41.636] Done. pip #          step             depends                 out keepOut      group  state #        <char>              <list>              <list>  <lgcl>     <char> <char> # 1:       data                     <data.frame[153x6]>   FALSE       data   Done # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep   Done # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit   Done # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot   Done pip$get_out(\"model_fit\") #  # Call: # lm(formula = paste(\"Ozone ~\", xVar), data = data) #  # Coefficients: #  (Intercept)  Temp.Celsius   #      -69.277         4.372 pip$get_out(\"model_plot\")"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"change-parameters-of-the-pipeline","dir":"Articles","previous_headings":"A simple example to get started","what":"Change parameters of the pipeline","title":"Get started with pipeflow","text":"parameter pipeline bound specific step (.e. defined tilde (~) operator) can changed calling set_params() method pipeline. Let’s first list unbound parameters: Now let’s assume want change xVar entire pipeline \"Solar.R\". can simply follows: Changing parameters pipeline least effect steps depend changed parameter. can see model_fit model_plot steps now state Outdated (graphically indicated orange color), means need rerun. can calling run() method pipeline . outdated steps re-run expected output updated accordingly.  just change title plot, obviously model_plot step needs rerun.  instead change input data, since steps depend , expect steps rerun.  Next, let’s see modify pipeline.","code":"pip$get_params() # $model_fit # $model_fit$xVar # [1] \"Temp.Celsius\" #  #  # $model_plot # $model_plot$xVar # [1] \"Temp.Celsius\" #  # $model_plot$title # [1] \"Linear model fit\" pip$set_params(list(xVar = \"Solar.R\")) pip$get_params() # $model_fit # $model_fit$xVar # [1] \"Solar.R\" #  #  # $model_plot # $model_plot$xVar # [1] \"Solar.R\" #  # $model_plot$title # [1] \"Linear model fit\" pip #          step             depends                 out keepOut      group    state #        <char>              <list>              <list>  <lgcl>     <char>   <char> # 1:       data                     <data.frame[153x6]>   FALSE       data     Done # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep     Done # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit Outdated # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot Outdated pip$run() # INFO  [2024-12-01 21:41:42.547] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:42.549] Step 1/4 data - skip 'done' step # INFO  [2024-12-01 21:41:42.550] Step 2/4 data_prep - skip 'done' step # INFO  [2024-12-01 21:41:42.551] Step 3/4 model_fit # INFO  [2024-12-01 21:41:42.555] Step 4/4 model_plot # INFO  [2024-12-01 21:41:42.564] Finished execution of steps. # INFO  [2024-12-01 21:41:42.565] Done. pip$get_out(\"model_plot\") pip$set_params(list(title = \"Some new title\")) pip #          step             depends                 out keepOut      group    state #        <char>              <list>              <list>  <lgcl>     <char>   <char> # 1:       data                     <data.frame[153x6]>   FALSE       data     Done # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep     Done # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit     Done # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot Outdated pip$run()$get_out(\"model_plot\") # INFO  [2024-12-01 21:41:42.919] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:42.920] Step 1/4 data - skip 'done' step # INFO  [2024-12-01 21:41:42.921] Step 2/4 data_prep - skip 'done' step # INFO  [2024-12-01 21:41:42.922] Step 3/4 model_fit - skip 'done' step # INFO  [2024-12-01 21:41:42.923] Step 4/4 model_plot # INFO  [2024-12-01 21:41:42.931] Finished execution of steps. # INFO  [2024-12-01 21:41:42.931] Done. pip$set_data(airquality[1:10, ]) pip #          step             depends                 out keepOut      group    state #        <char>              <list>              <list>  <lgcl>     <char>   <char> # 1:       data                                  [NULL]   FALSE       data      New # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep Outdated # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit Outdated # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot Outdated pip$run()$get_out(\"model_plot\") # INFO  [2024-12-01 21:41:43.202] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:43.203] Step 1/4 data # INFO  [2024-12-01 21:41:43.206] Step 2/4 data_prep # INFO  [2024-12-01 21:41:43.211] Step 3/4 model_fit # INFO  [2024-12-01 21:41:43.214] Step 4/4 model_plot # INFO  [2024-12-01 21:41:43.219] Finished execution of steps. # INFO  [2024-12-01 21:41:43.220] Done."},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"existing-pipeline","dir":"Articles","previous_headings":"","what":"Existing pipeline","title":"Modifying existing pipelines","text":"Let’s start left Get started pipeflow vignette, , following pipeline following set data","code":"pip #          step             depends                out keepOut      group  state #        <char>              <list>             <list>  <lgcl>     <char> <char> # 1:       data                     <data.frame[10x6]>   FALSE       data   Done # 2:  data_prep                data <data.frame[10x7]>   FALSE  data_prep   Done # 3:  model_fit           data_prep           <lm[13]>   FALSE  model_fit   Done # 4: model_plot model_fit,data_prep           <gg[11]>   FALSE model_plot   Done pip$get_data() |> head(3) #   Ozone Solar.R Wind Temp Month Day # 1    41     190  7.4   67     5   1 # 2    36     118  8.0   72     5   2 # 3    12     149 12.6   74     5   3"},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"insert-new-step","dir":"Articles","previous_headings":"","what":"Insert new step","title":"Modifying existing pipelines","text":"Let’s say want insert new step data_prep step standardizes y-variable. can see, standardize step now part pipeline, far used step.","code":"pip$insert_after(     afterStep = \"data_prep\",     step = \"standardize\",     function(         data = ~`data_prep`,         yVar = \"Ozone\"     ) {         data[, yVar] <- scale(data[, yVar])         data     } ) pip #           step             depends                out keepOut       group  state #         <char>              <list>             <list>  <lgcl>      <char> <char> # 1:        data                     <data.frame[10x6]>   FALSE        data   Done # 2:   data_prep                data <data.frame[10x7]>   FALSE   data_prep   Done # 3: standardize           data_prep             [NULL]   FALSE standardize    New # 4:   model_fit           data_prep           <lm[13]>   FALSE   model_fit   Done # 5:  model_plot model_fit,data_prep           <gg[11]>   FALSE  model_plot   Done library(visNetwork) do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\")"},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"replace-existing-steps","dir":"Articles","previous_headings":"","what":"Replace existing steps","title":"Modifying existing pipelines","text":"Let’s revisit function definition model_fit step use standardized data, need change data dependency refers standardize step. Also instead fixed y-variable model, want pass paramter. model_plot step needs updated similar way. updated pipeline now looks follows. see model_fit model_plot steps now use standardized data. Let’s re-run pipeline inspect output.","code":"pip$get_step(\"model_fit\")[[\"fun\"]] # [[1]] # function (data = ~data_prep, xVar = \"Temp.Celsius\")  # { #     lm(paste(\"Ozone ~\", xVar), data = data) # } pip$replace_step(     \"model_fit\",     function(         data = ~standardize,        # <- changed data reference         xVar = \"Temp.Celsius\",         yVar = \"Ozone\"              # <- new y-variable     ) {         lm(paste(yVar, \"~\", xVar), data = data)     } ) pip$replace_step(     \"model_plot\",     function(         model = ~model_fit,         data = ~standardize,         # <- changed data reference         xVar = \"Temp.Celsius\",         yVar = \"Ozone\",              # <- new y-variable         title = \"Linear model fit\"     ) {         coeffs <- coefficients(model)         ggplot(data) +             geom_point(aes(.data[[xVar]], .data[[yVar]])) +             geom_abline(intercept = coeffs[1], slope = coeffs[2]) +             labs(title = title)     } ) pip #           step               depends                out keepOut       group  state #         <char>                <list>             <list>  <lgcl>      <char> <char> # 1:        data                       <data.frame[10x6]>   FALSE        data   Done # 2:   data_prep                  data <data.frame[10x7]>   FALSE   data_prep   Done # 3: standardize             data_prep             [NULL]   FALSE standardize    New # 4:   model_fit           standardize             [NULL]   FALSE   model_fit    New # 5:  model_plot model_fit,standardize             [NULL]   FALSE  model_plot    New pip$set_params(list(xVar = \"Solar.R\", yVar = \"Wind\")) pip$run() # INFO  [2024-12-01 21:41:47.485] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:47.486] Step 1/5 data - skip 'done' step # INFO  [2024-12-01 21:41:47.487] Step 2/5 data_prep - skip 'done' step # INFO  [2024-12-01 21:41:47.488] Step 3/5 standardize # INFO  [2024-12-01 21:41:47.490] Step 4/5 model_fit # INFO  [2024-12-01 21:41:47.493] Step 5/5 model_plot # INFO  [2024-12-01 21:41:47.498] Finished execution of steps. # INFO  [2024-12-01 21:41:47.498] Done. pip$get_out(\"model_fit\") |> coefficients() #  (Intercept)      Solar.R  #  0.979672739 -0.006625601 pip$get_out(\"model_plot\") # Warning: Removed 2 rows containing missing values or values outside the scale range # (`geom_point()`)."},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"removing-steps","dir":"Articles","previous_headings":"","what":"Removing steps","title":"Modifying existing pipelines","text":"Let’s see pipeline . trying remove step, pipeflow default checks step used step. , raise error. enforce removing step together downstream dependencies, can use recursive argument. Naturally, last step never downstream dependencies, can removed without issues. another way just remove last step. Replacing steps pipeline shown vignette allow re-use existing pipelines adapt programmatically new requirements. Another way re-using pipelines combine , shown Combining pipelines vignette.","code":"pip #           step               depends                out keepOut       group  state #         <char>                <list>             <list>  <lgcl>      <char> <char> # 1:        data                       <data.frame[10x6]>   FALSE        data   Done # 2:   data_prep                  data <data.frame[10x7]>   FALSE   data_prep   Done # 3: standardize             data_prep <data.frame[10x7]>   FALSE standardize   Done # 4:   model_fit           standardize           <lm[13]>   FALSE   model_fit   Done # 5:  model_plot model_fit,standardize           <gg[11]>   FALSE  model_plot   Done try(pip$remove_step(\"standardize\")) # Error in pip$remove_step(\"standardize\") :  #   cannot remove step 'standardize' because the following steps depend on it: 'model_fit', 'model_plot' pip$remove_step(\"standardize\", recursive = TRUE) # Removing step 'standardize' and its downstream dependencies: 'model_fit', 'model_plot' pip #         step depends                out keepOut     group  state #       <char>  <list>             <list>  <lgcl>    <char> <char> # 1:      data         <data.frame[10x6]>   FALSE      data   Done # 2: data_prep    data <data.frame[10x7]>   FALSE data_prep   Done pip$pop_step() # [1] \"data_prep\" pip #      step depends                out keepOut  group  state #    <char>  <list>             <list>  <lgcl> <char> <char> # 1:   data         <data.frame[10x6]>   FALSE   data   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"changing-pipeline-parameters-at-runtime","dir":"Articles","previous_headings":"","what":"Changing pipeline parameters at runtime","title":"Advanced: How pipelines can modify themselves at runtime","text":"Let’s first define pipeline, fits linear model, checks ’s residuals normality using Shapiro-Wilk test, plots residuals. pipeline looks like can run like  Now let’s imagine, want change color points plot depending Shapiro-Wilk test result. obvious way change plot step passing test result plot step function change color . However, interested another way keep plot function unchanged. example, run pipeline second time follows:  mentioned another vignette, solution ideal, requires run additional code outside pipeline framework. solve issue, therefore basically set parameter within pipeline execution. , make pipeline aware , can done passing pipeline object parameter. Let’s update residual_shapiro_p_value step example. Now just make sure set .self parameter.  simple “trick” opens wide range possibilities pipeline modifications runtime. show next section, limited changing parameters can also used modify pipeline structure .","code":"library(pipeflow) library(ggplot2)  pip <- pipe_new(\"my-pipeline\") |>      pipe_add(         \"fit\",         function(data = ~data, xVar = \"x\", yVar = \"y\")         {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"residual_shapiro_p_value\",         function(fit = ~fit)         {             residuals <- residuals(fit)             p <- shapiro.test(residuals)$p.value             p         },         keepOut = TRUE     ) |>      pipe_add(         \"plot\",         function(fit = ~fit, pointColor = \"black\")         {             data <- data.frame(                 fitted = predict(fit),                 residuals = residuals(fit)             )              ggplot(data, aes(x = fitted, y = residuals)) +                 geom_point(shape = 21, color = pointColor) +                 geom_hline(yintercept = 0, linetype = \"dashed\") +                 theme_minimal()         },         keepOut = TRUE     ) pip #                        step depends    out keepOut                    group  state #                      <char>  <list> <list>  <lgcl>                   <char> <char> # 1:                     data         [NULL]   FALSE                     data    New # 2:                      fit    data [NULL]   FALSE                      fit    New # 3: residual_shapiro_p_value     fit [NULL]    TRUE residual_shapiro_p_value    New # 4:                     plot     fit [NULL]    TRUE                     plot    New pip$set_data(airquality) pip$set_params(list(xVar = \"Ozone\", yVar = \"Temp\")) pip$run()$collect_out() # INFO  [2024-12-01 21:41:51.403] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:51.430] Step 1/4 data # INFO  [2024-12-01 21:41:51.444] Step 2/4 fit # INFO  [2024-12-01 21:41:51.462] Step 3/4 residual_shapiro_p_value # INFO  [2024-12-01 21:41:51.464] Step 4/4 plot # INFO  [2024-12-01 21:41:51.481] Finished execution of steps. # INFO  [2024-12-01 21:41:51.482] Done. # $residual_shapiro_p_value # [1] 0.00022598 #  # $plot if (pip$get_out(\"residual_shapiro_p_value\") < 0.05) {     pip$set_params(list(pointColor = \"red\"))     pip$run()$collect_out() } # INFO  [2024-12-01 21:41:51.876] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:51.877] Step 1/4 data - skip 'done' step # INFO  [2024-12-01 21:41:51.878] Step 2/4 fit - skip 'done' step # INFO  [2024-12-01 21:41:51.879] Step 3/4 residual_shapiro_p_value - skip 'done' step # INFO  [2024-12-01 21:41:51.880] Step 4/4 plot # INFO  [2024-12-01 21:41:51.893] Finished execution of steps. # INFO  [2024-12-01 21:41:51.894] Done. # $residual_shapiro_p_value # [1] 0.00022598 #  # $plot pip$replace_step(     \"residual_shapiro_p_value\",     function(         fit = ~fit,         .self = NULL     ) {         residuals <- residuals(fit)         p <- shapiro.test(residuals)$p.value          if (!is.null(.self) && p < 0.05) {             .self$set_params(list(pointColor = \"red\"))         }          p     },     keepOut = TRUE ) pip$set_data(airquality) pip$set_params(list(xVar = \"Ozone\", yVar = \"Temp\", .self = pip)) pip$run()$collect_out() # INFO  [2024-12-01 21:41:52.189] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:52.190] Step 1/4 data # INFO  [2024-12-01 21:41:52.192] Step 2/4 fit # INFO  [2024-12-01 21:41:52.196] Step 3/4 residual_shapiro_p_value # INFO  [2024-12-01 21:41:52.200] Step 4/4 plot # INFO  [2024-12-01 21:41:52.207] Finished execution of steps. # INFO  [2024-12-01 21:41:52.207] Done. # $residual_shapiro_p_value # [1] 0.00022598 #  # $plot"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"changing-the-pipeline-structure-at-runtime","dir":"Articles","previous_headings":"","what":"Changing the pipeline structure at runtime","title":"Advanced: How pipelines can modify themselves at runtime","text":"Subsequently, pipeline steps comprised basic functions order keep examples simple. focus pipeline structure can modified runtime. pipeline just adds 1, 2, 3 input data, respectively. column table shows output step. Now let’s modify last step pipeline input greater 10, pipeline replace last step new step now instead f2 references f1 subtracts 3 input.","code":"pip <- pipe_new(\"my-pipeline\") |>      pipe_add(         \"f1\",         function(x = ~data) {             x + 1         }     ) |>      pipe_add(         \"f2\",         function(x = ~f1) {             x + 2         }     ) |>      pipe_add(         \"f3\",         function(x = ~f2) {             x + 3         }     ) pip$set_data(1)$run() # INFO  [2024-12-01 21:41:52.449] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:52.450] Step 1/4 data # INFO  [2024-12-01 21:41:52.452] Step 2/4 f1 # INFO  [2024-12-01 21:41:52.454] Step 3/4 f2 # INFO  [2024-12-01 21:41:52.456] Step 4/4 f3 # INFO  [2024-12-01 21:41:52.457] Finished execution of steps. # INFO  [2024-12-01 21:41:52.458] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data              1   FALSE   data   Done # 2:     f1    data      2   FALSE     f1   Done # 3:     f2      f1      4   FALSE     f2   Done # 4:     f3      f2      7   FALSE     f3   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"modify-a-step","dir":"Articles","previous_headings":"Changing the pipeline structure at runtime","what":"Modify a step","title":"Advanced: How pipelines can modify themselves at runtime","text":"run pipeline , nothing change. Now let’s try input 10. see output pipeline dependencies last step changed.","code":"pip <- pipe_new(\"my-pipeline\") |>      pipe_add(         \"f1\",         function(x = ~data) {             x + 1         }     ) |>      pipe_add(         \"f2\",         function(x = ~f1, .self = NULL)         {             if (x > 10 && !is.null(.self))             {                 .self$replace_step(                     \"f3\",                     function(x = ~f1) {                         x - 3                     }                 )             }             x + 2         }     ) |>      pipe_add(         \"f3\",         function(x = ~f2) {             x + 3         }     ) pip$set_params(list(.self = pip)) pip$set_data(1)$run() # INFO  [2024-12-01 21:41:52.575] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:52.576] Step 1/4 data # INFO  [2024-12-01 21:41:52.578] Step 2/4 f1 # INFO  [2024-12-01 21:41:52.580] Step 3/4 f2 # INFO  [2024-12-01 21:41:52.582] Step 4/4 f3 # INFO  [2024-12-01 21:41:52.584] Finished execution of steps. # INFO  [2024-12-01 21:41:52.584] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data              1   FALSE   data   Done # 2:     f1    data      2   FALSE     f1   Done # 3:     f2      f1      4   FALSE     f2   Done # 4:     f3      f2      7   FALSE     f3   Done pip$set_data(10)$run() # INFO  [2024-12-01 21:41:52.643] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:52.644] Step 1/4 data # INFO  [2024-12-01 21:41:52.646] Step 2/4 f1 # INFO  [2024-12-01 21:41:52.649] Step 3/4 f2 # INFO  [2024-12-01 21:41:52.654] Step 4/4 f3 # INFO  [2024-12-01 21:41:52.656] Finished execution of steps. # INFO  [2024-12-01 21:41:52.656] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data             10   FALSE   data   Done # 2:     f1    data     11   FALSE     f1   Done # 3:     f2      f1     13   FALSE     f2   Done # 4:     f3      f1      8   FALSE     f3   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"insert-and-remove-steps","dir":"Articles","previous_headings":"Changing the pipeline structure at runtime","what":"Insert and remove steps","title":"Advanced: How pipelines can modify themselves at runtime","text":"last example, instead just replacing, go bit insert remove steps. pipeline definition follows: Basically, input greater 10, insert two new steps f1, remove f2, replace f3 new step adds 30 input. Also note return pipeline object case. important, pipeline’s run function argument recursive, default set TRUE means step returns pipeline, run current pipeline aborted returned pipeline re-run. Let’s see pipeline structure running . now let’s run input 10. log output shows abort re-run pipeline. Let’s see final structure step outputs. final structure expected new steps inserted old step removed. mentioned , just simple example show possibilities. leave user come sensible complex use cases.","code":"pip <- pipe_new(         \"my-pipeline\"     ) |>      pipe_add(         \"f1\",         function(x = ~data) {             x + 1         }     ) |>      pipe_add(         \"f2\",         function(x = ~f1, .self = NULL)         {             if (x > 10 && !is.null(.self)) {                 .self$insert_after(                     afterStep = \"f1\",                     step = \"f2a\",                     function(x = ~f1) {                         x + 21                     }                 )                 .self$insert_after(                     afterStep = \"f2a\",                     step = \"f2b\",                     function(x = ~f2a) {                         x + 22                     }                 )                 .self$replace_step(                     \"f3\",                     function(x = ~f2b) {                         x + 30                     }                 )                 .self$remove_step(\"f2\")                 return(.self)             }             x + 2         }     ) |>      pipe_add(         \"f3\",         function(x = ~f2) {             x + 3         }     ) pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data         [NULL]   FALSE   data    New # 2:     f1    data [NULL]   FALSE     f1    New # 3:     f2      f1 [NULL]   FALSE     f2    New # 4:     f3      f2 [NULL]   FALSE     f3    New pip$set_params(list(.self = pip)) pip$set_data(10)$run() # INFO  [2024-12-01 21:41:52.829] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:52.830] Step 1/4 data # INFO  [2024-12-01 21:41:52.832] Step 2/4 f1 # INFO  [2024-12-01 21:41:52.834] Step 3/4 f2 # INFO  [2024-12-01 21:41:52.848] Abort pipeline execution and restart on new. # INFO  [2024-12-01 21:41:52.848] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:52.849] Step 1/5 data - skip 'done' step # INFO  [2024-12-01 21:41:52.850] Step 2/5 f1 - skip 'done' step # INFO  [2024-12-01 21:41:52.851] Step 3/5 f2a # INFO  [2024-12-01 21:41:52.856] Step 4/5 f2b # INFO  [2024-12-01 21:41:52.858] Step 5/5 f3 # INFO  [2024-12-01 21:41:52.859] Finished execution of steps. # INFO  [2024-12-01 21:41:52.860] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data             10   FALSE   data   Done # 2:     f1    data     11   FALSE     f1   Done # 3:    f2a      f1     32   FALSE    f2a   Done # 4:    f2b     f2a     54   FALSE    f2b   Done # 5:     f3     f2b     84   FALSE     f3   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"motivation","dir":"Articles","previous_headings":"","what":"Motivation","title":"Advanced: How to use pipeflow with split data sets","text":"common scenario split data set subsets apply analysis part. context pipelines, means like apply pipeline multiple times data subset. additon, may want combine parts individual output. see, pipeflow provides built-function handle scenario.","code":""},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"define-pipeline","dir":"Articles","previous_headings":"","what":"Define pipeline","title":"Advanced: How to use pipeflow with split data sets","text":"Let’s first define pipeline, , keep matters simple, just fits linear model outputs model coefficients. pipeline looks like : graphically: use iris data set working example. First, apply pipeline whole data set.","code":"library(pipeflow)  pip <- pipe_new(         \"my-pipeline\"     ) |>      pipe_add(         \"fit\",         function(             data = ~data,             xVar = \"x\",             yVar = \"y\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"coefs\",         function(             fit = ~fit         ) {             coefficients(fit)         },         keepOut = TRUE     ) pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data         [NULL]   FALSE   data    New # 2:    fit    data [NULL]   FALSE    fit    New # 3:  coefs     fit [NULL]    TRUE  coefs    New library(visNetwork) do.call(visNetwork, args = c(pip$get_graph(), list(height = 100))) |>     visHierarchicalLayout(direction = \"LR\") head(iris) #   Sepal.Length Sepal.Width Petal.Length Petal.Width Species # 1          5.1         3.5          1.4         0.2  setosa # 2          4.9         3.0          1.4         0.2  setosa # 3          4.7         3.2          1.3         0.2  setosa # 4          4.6         3.1          1.5         0.2  setosa # 5          5.0         3.6          1.4         0.2  setosa # 6          5.4         3.9          1.7         0.4  setosa pip$set_data(iris) pip$set_params(list(xVar = \"Sepal.Length\", yVar = \"Sepal.Width\")) pip$run() # INFO  [2024-12-01 21:41:55.998] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:56.024] Step 1/3 data # INFO  [2024-12-01 21:41:56.035] Step 2/3 fit # INFO  [2024-12-01 21:41:56.055] Step 3/3 coefs # INFO  [2024-12-01 21:41:56.056] Finished execution of steps. # INFO  [2024-12-01 21:41:56.056] Done. pip$collect_out() # $coefs #  (Intercept) Sepal.Length  #    3.4189468   -0.0618848"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"split-data-set","dir":"Articles","previous_headings":"","what":"Split data set","title":"Advanced: How to use pipeflow with split data sets","text":"Next, want apply pipeline species separately. One way use R’s split function. can split Species column run pipeline subset. example: Unfortunately, approach create additional code run outside pipeline framework. Let’s now see handle scenario within pipeline framework. reminder, pipeline looks like : deal split data sets, use built-function set_data_split. function actually transforms pipeline: can see, pipeline now replicated data subset. Note set_data_split accepts list data frames, just output split. Now let’s run pipeline. can see, output now obtained without need write additional code outside pipeline framework. Finally, side note another built-function named split, can used split pipeline independent parts. works pipeline, working example, naturally split pipeline parts defined data split. function especially useful want separate parts pipeline code order run parallel.","code":"run_pipeline <- function(data) {     pip$set_data(data)     pip$run()     pip$collect_out() }  results <- lapply(split(iris, iris$Species), FUN = run_pipeline) # INFO  [2024-12-01 21:41:56.192] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:56.193] Step 1/3 data # INFO  [2024-12-01 21:41:56.195] Step 2/3 fit # INFO  [2024-12-01 21:41:56.199] Step 3/3 coefs # INFO  [2024-12-01 21:41:56.203] Finished execution of steps. # INFO  [2024-12-01 21:41:56.204] Done. # INFO  [2024-12-01 21:41:56.222] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:56.223] Step 1/3 data # INFO  [2024-12-01 21:41:56.225] Step 2/3 fit # INFO  [2024-12-01 21:41:56.227] Step 3/3 coefs # INFO  [2024-12-01 21:41:56.228] Finished execution of steps. # INFO  [2024-12-01 21:41:56.229] Done. # INFO  [2024-12-01 21:41:56.232] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:56.233] Step 1/3 data # INFO  [2024-12-01 21:41:56.235] Step 2/3 fit # INFO  [2024-12-01 21:41:56.238] Step 3/3 coefs # INFO  [2024-12-01 21:41:56.239] Finished execution of steps. # INFO  [2024-12-01 21:41:56.239] Done. results # $setosa # $setosa$coefs #  (Intercept) Sepal.Length  #   -0.5694327    0.7985283  #  #  # $versicolor # $versicolor$coefs #  (Intercept) Sepal.Length  #    0.8721460    0.3197193  #  #  # $virginica # $virginica$coefs #  (Intercept) Sepal.Length  #    1.4463054    0.2318905 pip #      step depends                 out keepOut  group  state #    <char>  <list>              <list>  <lgcl> <char> <char> # 1:   data          <data.frame[50x5]>   FALSE   data   Done # 2:    fit    data            <lm[12]>   FALSE    fit   Done # 3:  coefs     fit 1.4463054,0.2318905    TRUE  coefs   Done splitData <- split(iris, iris$Species) pip$set_data_split(splitData) pip #                step         depends                 out keepOut      group    state #              <char>          <list>              <list>  <lgcl>     <char>   <char> # 1:      data.setosa                              [NULL]   FALSE     setosa      New # 2:       fit.setosa     data.setosa            <lm[12]>   FALSE     setosa Outdated # 3:     coefs.setosa      fit.setosa 1.4463054,0.2318905    TRUE     setosa Outdated # 4:  data.versicolor                              [NULL]   FALSE versicolor      New # 5:   fit.versicolor data.versicolor            <lm[12]>   FALSE versicolor Outdated # 6: coefs.versicolor  fit.versicolor 1.4463054,0.2318905    TRUE versicolor Outdated # 7:   data.virginica                              [NULL]   FALSE  virginica      New # 8:    fit.virginica  data.virginica            <lm[12]>   FALSE  virginica Outdated # 9:  coefs.virginica   fit.virginica 1.4463054,0.2318905    TRUE  virginica Outdated do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\") pip$run() # INFO  [2024-12-01 21:41:56.710] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:56.711] Step 1/9 data.setosa # INFO  [2024-12-01 21:41:56.713] Step 2/9 fit.setosa # INFO  [2024-12-01 21:41:56.716] Step 3/9 coefs.setosa # INFO  [2024-12-01 21:41:56.717] Step 4/9 data.versicolor # INFO  [2024-12-01 21:41:56.719] Step 5/9 fit.versicolor # INFO  [2024-12-01 21:41:56.722] Step 6/9 coefs.versicolor # INFO  [2024-12-01 21:41:56.723] Step 7/9 data.virginica # INFO  [2024-12-01 21:41:56.725] Step 8/9 fit.virginica # INFO  [2024-12-01 21:41:56.728] Step 9/9 coefs.virginica # INFO  [2024-12-01 21:41:56.729] Finished execution of steps. # INFO  [2024-12-01 21:41:56.730] Done. pip$collect_out() # $setosa #  (Intercept) Sepal.Length  #   -0.5694327    0.7985283  #  # $versicolor #  (Intercept) Sepal.Length  #    0.8721460    0.3197193  #  # $virginica #  (Intercept) Sepal.Length  #    1.4463054    0.2318905 pip$split() # [[1]] #            step     depends                   out keepOut  group  state #          <char>      <list>                <list>  <lgcl> <char> <char> # 1:  data.setosa                <data.frame[50x5]>   FALSE setosa   Done # 2:   fit.setosa data.setosa              <lm[12]>   FALSE setosa   Done # 3: coefs.setosa  fit.setosa -0.5694327, 0.7985283    TRUE setosa   Done #  # [[2]] #                step         depends                 out keepOut      group  state #              <char>          <list>              <list>  <lgcl>     <char> <char> # 1:  data.versicolor                  <data.frame[50x5]>   FALSE versicolor   Done # 2:   fit.versicolor data.versicolor            <lm[12]>   FALSE versicolor   Done # 3: coefs.versicolor  fit.versicolor 0.8721460,0.3197193    TRUE versicolor   Done #  # [[3]] #               step        depends                 out keepOut     group  state #             <char>         <list>              <list>  <lgcl>    <char> <char> # 1:  data.virginica                 <data.frame[50x5]>   FALSE virginica   Done # 2:   fit.virginica data.virginica            <lm[12]>   FALSE virginica   Done # 3: coefs.virginica  fit.virginica 1.4463054,0.2318905    TRUE virginica   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"combine-output","dir":"Articles","previous_headings":"","what":"Combine output","title":"Advanced: How to use pipeflow with split data sets","text":"cases, may want (re-)combine output pipeline parts. example, may want combine coefficients linear models one table. Let’s matter extend pipeline one step end. pipeline looks now. Next want apply set_data_split function, basically need make sure pipeline split except last step combines everything. achieve using toStep parameter, basically tells pipeline split certain step. see last step replicated data subset now contains list dependencies, namely: Graphically becomes even clear: Finally, let’s see plays run pipeline.","code":"pip <- pipe_new(         \"my-pipeline\"     ) |>      pipe_add(         \"fit\",         function(             data = ~data,             xVar = \"x\",             yVar = \"y\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"coefs\",         function(             fit = ~fit         ) {             coefficients(fit)         }     )  |>      pipe_add(         \"combine_coefs\",         function(             coefs = ~coefs         ) {             coefs |> do.call(rbind, args = _) |> as.data.frame()         },         keepOut = TRUE     ) pip #             step depends    out keepOut         group  state #           <char>  <list> <list>  <lgcl>        <char> <char> # 1:          data         [NULL]   FALSE          data    New # 2:           fit    data [NULL]   FALSE           fit    New # 3:         coefs     fit [NULL]   FALSE         coefs    New # 4: combine_coefs   coefs [NULL]    TRUE combine_coefs    New pip$set_data_split(split(iris, iris$Species), toStep = \"coefs\") pip #                 step         depends    out keepOut         group    state #               <char>          <list> <list>  <lgcl>        <char>   <char> #  1:      data.setosa                 [NULL]   FALSE        setosa      New #  2:       fit.setosa     data.setosa [NULL]   FALSE        setosa Outdated #  3:     coefs.setosa      fit.setosa [NULL]   FALSE        setosa Outdated #  4:  data.versicolor                 [NULL]   FALSE    versicolor      New #  5:   fit.versicolor data.versicolor [NULL]   FALSE    versicolor Outdated #  6: coefs.versicolor  fit.versicolor [NULL]   FALSE    versicolor Outdated #  7:   data.virginica                 [NULL]   FALSE     virginica      New #  8:    fit.virginica  data.virginica [NULL]   FALSE     virginica Outdated #  9:  coefs.virginica   fit.virginica [NULL]   FALSE     virginica Outdated # 10:    combine_coefs       <list[1]> [NULL]    TRUE combine_coefs      New pip$get_depends()[[\"combine_coefs\"]] # $coefs # [1] \"coefs.setosa\"     \"coefs.versicolor\" \"coefs.virginica\" do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\") pip$set_params(list(xVar = \"Sepal.Length\", yVar = \"Sepal.Width\")) pip$run() # INFO  [2024-12-01 21:41:57.260] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:41:57.263] Step 1/10 data.setosa # INFO  [2024-12-01 21:41:57.265] Step 2/10 fit.setosa # INFO  [2024-12-01 21:41:57.268] Step 3/10 coefs.setosa # INFO  [2024-12-01 21:41:57.270] Step 4/10 data.versicolor # INFO  [2024-12-01 21:41:57.272] Step 5/10 fit.versicolor # INFO  [2024-12-01 21:41:57.275] Step 6/10 coefs.versicolor # INFO  [2024-12-01 21:41:57.277] Step 7/10 data.virginica # INFO  [2024-12-01 21:41:57.279] Step 8/10 fit.virginica # INFO  [2024-12-01 21:41:57.282] Step 9/10 coefs.virginica # INFO  [2024-12-01 21:41:57.284] Step 10/10 combine_coefs # INFO  [2024-12-01 21:41:57.286] Finished execution of steps. # INFO  [2024-12-01 21:41:57.286] Done. pip$collect_out() # $combine_coefs #                  (Intercept) Sepal.Length # coefs.setosa      -0.5694327    0.7985283 # coefs.versicolor   0.8721460    0.3197193 # coefs.virginica    1.4463054    0.2318905"},{"path":"https://github.com/rpahl/pipeflow/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Roman Pahl. Author, maintainer.","code":""},{"path":"https://github.com/rpahl/pipeflow/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Pahl R (2024). pipeflow: Implement Data Analysis Workflows Pipelines. R package version 0.1.1, https://github.com/rpahl/pipeflow, https://rpahl.github.io/pipeflow.","code":"@Manual{,   title = {pipeflow: Implement Data Analysis Workflows with Pipelines},   author = {Roman Pahl},   year = {2024},   note = {R package version 0.1.1, https://github.com/rpahl/pipeflow},   url = {https://rpahl.github.io/pipeflow}, }"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"pipeflow-","dir":"","previous_headings":"","what":"Implement Data Analysis Workflows with Pipelines","title":"Implement Data Analysis Workflows with Pipelines","text":"pipeflow intuitive yet powerful framework building running data processing pipelines. Whether ’re working scientific computing, machine learning (AI), statistical reporting, pipeflow adapts seamlessly scenario—small exploratory analyses complex production workflows. Since pipeflow pipelines essentially sequences R functions, ’re easy learn can used interactively typical R users. Advanced features like dynamic branching self-modifying pipelines provide additional flexibility complex workflows.","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"why-use-pipeflow","dir":"","previous_headings":"","what":"Why use pipeflow","title":"Implement Data Analysis Workflows with Pipelines","text":"Easy learn yet powerful complex workflows Automatically manages function dependencies Promotes standardized, reproducible analysis Simplifies error handling, debugging, reusability","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"key-features","dir":"","previous_headings":"","what":"Key features","title":"Implement Data Analysis Workflows with Pipelines","text":"Flexible Application: Use interactively programmatically R Dependency Management: Dependencies checked definition, ensuring reliable workflows Comprehensive Logging: Logs step, customizable logger options Parameter Control: Easily view adjust function parameters one place Modular Composition: Modify, extend, combine pipelines effortlessly Intelligent Execution: Skip steps already --date, similar make Visualization: View pipelines tabular graphical formats","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"advanced-features","dir":"","previous_headings":"","what":"Advanced features","title":"Implement Data Analysis Workflows with Pipelines","text":"Dynamic Branching: Apply pipeline multiple datasets seamlessly Self-Modifying: Pipelines can adapt modify runtime","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Implement Data Analysis Workflows with Pipelines","text":"","code":"# Install release version from CRAN install.packages(\"pipeflow\")  # Install development version from GitHub devtools::install_github(\"rpahl/pipeflow\")"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"usage","dir":"","previous_headings":"","what":"Usage","title":"Implement Data Analysis Workflows with Pipelines","text":"","code":"library(pipeflow)"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"getting-started","dir":"","previous_headings":"","what":"Getting Started","title":"Implement Data Analysis Workflows with Pipelines","text":"recommended read vignettes order listed : Get started pipeflow Modifying existing pipelines Combining pipelines Collecting output","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"advanced-topics","dir":"","previous_headings":"","what":"Advanced topics","title":"Implement Data Analysis Workflows with Pipelines","text":"use pipeflow split data sets pipelines can modify runtime","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":null,"dir":"Reference","previous_headings":"","what":"Pipeline Class — Pipeline","title":"Pipeline Class — Pipeline","text":"class implements analysis pipeline. pipeline consists sequence analysis steps, can added one one. added step may may depend one previous steps. pipeline keeps track dependencies among steps ensure dependencies met creation pipeline, , pipeline run. pipeline run, output stored pipeline along step can accessed later. Different pipelines can bound together preserving dependencies within pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Pipeline Class — Pipeline","text":"Roman Pahl","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"public-fields","dir":"Reference","previous_headings":"","what":"Public fields","title":"Pipeline Class — Pipeline","text":"name string name pipeline pipeline data.table pipeline row represents one step.","code":""},{"path":[]},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"public-methods","dir":"Reference","previous_headings":"","what":"Public methods","title":"Pipeline Class — Pipeline","text":"Pipeline$new() Pipeline$add() Pipeline$append() Pipeline$append_to_step_names() Pipeline$collect_out() Pipeline$discard_steps() Pipeline$get_data() Pipeline$get_depends() Pipeline$get_depends_down() Pipeline$get_depends_up() Pipeline$get_graph() Pipeline$get_out() Pipeline$get_params() Pipeline$get_params_at_step() Pipeline$get_params_unique() Pipeline$get_params_unique_json() Pipeline$get_step() Pipeline$get_step_names() Pipeline$get_step_number() Pipeline$has_step() Pipeline$insert_after() Pipeline$insert_before() Pipeline$length() Pipeline$lock_step() Pipeline$print() Pipeline$pop_step() Pipeline$pop_steps_after() Pipeline$pop_steps_from() Pipeline$remove_step() Pipeline$rename_step() Pipeline$replace_step() Pipeline$reset() Pipeline$run() Pipeline$run_step() Pipeline$set_data() Pipeline$set_data_split() Pipeline$set_keep_out() Pipeline$set_params() Pipeline$set_params_at_step() Pipeline$split() Pipeline$unlock_step() Pipeline$clone()","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-new-","dir":"Reference","previous_headings":"","what":"Method new()","title":"Pipeline Class — Pipeline","text":"constructor","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$new(name, data = NULL, logger = NULL)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"name name Pipeline data optional data used start pipeline. data also can set later using set_data function. logger custom logger used logging. logger provided, default logger used, sufficient use cases. want use custom log function, need provide function obeys following form: function(level, msg, ...) { custom logging code } level argument string one info, warn, error. msg argument string containing message logged. ... argument list named parameters, can used add additional information log message. Currently, used add context case step giving warning error. Note default logger, log layout can altered time via set_log_layout().","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:8)) p  # Passing custom logger my_logger <- function(level, msg, ...) {    cat(level, msg, \"\\n\") } p <- Pipeline$new(\"myPipe\", logger = my_logger)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-add-","dir":"Reference","previous_headings":"","what":"Method add()","title":"Pipeline Class — Pipeline","text":"Add pipeline step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-1","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$add(   step,   fun,   params = list(),   description = \"\",   group = step,   keepOut = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-1","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step. step name must unique. fun function name function applied step. existing lambda/anonymous functions can used. params list list parameters set overwrite parameters passed function. description string optional description step group string output collected pipeline execution (see function collect_out) grouped defined group names. default, name step, comes handy pipeline copy-appended multiple times keep results function/step grouped one place. keepOut logical FALSE (default) output step collected calling collect_out pipeline run. option used keep results matter skip intermediate results needed. See also function collect_out details.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-1","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-1","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Add steps with lambda functions p <- Pipeline$new(\"myPipe\", data = 1) p$add(\"s1\", \\(x = ~data) 2*x)  # use input data p$add(\"s2\", \\(x = ~data, y = ~s1) x * y) try(p$add(\"s2\", \\(z = 3) 3)) # error: step 's2' exists already try(p$add(\"s3\", \\(z = ~foo) 3)) # dependency 'foo' not found p  p <- Pipeline$new(\"myPipe\", data = c(1, 2, NA, 3, 4)) p$add(\"calc_mean\", mean, params = list(x = ~data, na.rm = TRUE)) p$run()$get_out(\"calc_mean\")  p <- Pipeline$new(\"myPipe\", data = 1:10) p$add(\"s1\", \\(x = ~data) 2*x, description = \"multiply by 2\") print(p) print(p, verbose = TRUE) # print all columns  p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:5, y = 1:5)) p$add(\"prep_x\", \\(data = ~data) data$x, group = \"prep\") p$add(\"prep_y\", \\(data = ~data) (data$y)^2, group = \"prep\") p$add(\"sum\", \\(x = ~prep_x, y = ~prep_y) x + y) p$run()$collect_out(all = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-append-","dir":"Reference","previous_headings":"","what":"Method append()","title":"Pipeline Class — Pipeline","text":"Append another pipeline. append takes care name clashes dependencies, changed append.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-2","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$append(p, outAsIn = FALSE, tryAutofixNames = TRUE, sep = \".\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-2","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"p Pipeline object appended. outAsIn logical TRUE, output first pipeline used input second pipeline. tryAutofixNames logical TRUE, name clashes tried automatically resolved appending 2nd pipeline's name. sep string separator used auto-resolving step names","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-2","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns new combined Pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-2","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Append pipeline p1 <- Pipeline$new(\"pipe1\") p1$add(\"step1\", \\(x = 1) x) p2 <- Pipeline$new(\"pipe2\") p2$add(\"step2\", \\(y = 1) y) p1$append(p2)  p3 <- Pipeline$new(\"pipe3\") p3$add(\"step1\", \\(z = 1) z) p1$append(p2)$append(p3)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-append-to-step-names-","dir":"Reference","previous_headings":"","what":"Method append_to_step_names()","title":"Pipeline Class — Pipeline","text":"Append string step names. Also takes care updating dependencies accordingly.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-3","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$append_to_step_names(postfix, sep = \".\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-3","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"postfix string appended step name. sep string separator step name postfix.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-3","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-3","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\") p$add(\"step1\", \\(x = 1) x) p$add(\"step2\", \\(y = 1) y) p$append_to_step_names(\"new\") p p$append_to_step_names(\"new\", sep = \"_\") p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-collect-out-","dir":"Reference","previous_headings":"","what":"Method collect_out()","title":"Pipeline Class — Pipeline","text":"Collect output afer pipeline run, default, steps keepOut set TRUE. output grouped group names (see group parameter function add) set explicitly corresponds step names.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-4","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$collect_out(groupBy = \"group\", all = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-4","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"groupBy string column pipeline group output. logical TRUE output collected regardless keepOut flag. can useful debugging.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-4","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list containing output, named groups, , default, steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-4","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2) p$add(\"step2\", \\(x = ~step1) x + 2, keepOut = TRUE) p$run() p$collect_out() p$collect_out(all = TRUE) |> str()  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2, group = \"add\") p$add(\"step2\", \\(x = ~step1, y = 2) x + y, group = \"add\") p$add(\"step3\", \\(x = ~data) x * 3, group = \"mult\") p$add(\"step4\", \\(x = ~data, y = 2) x * y, group = \"mult\") p p$run() p$collect_out(all = TRUE) |> str()  # Grouped by state p$set_params(list(y = 5)) p p$collect_out(groupBy = \"state\", all = TRUE) |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-discard-steps-","dir":"Reference","previous_headings":"","what":"Method discard_steps()","title":"Pipeline Class — Pipeline","text":"Discard steps match given pattern.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-5","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$discard_steps(pattern, recursive = FALSE, fixed = TRUE, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-5","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"pattern string containing regular expression (character string fixed = TRUE) matched. recursive logical TRUE step removed together downstream dependencies. fixed logical TRUE, pattern string matched . Overrides conflicting arguments. ... arguments passed grep().","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-5","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-5","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~add1) x + 2) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$discard_steps(\"mult\") p  # Re-add steps p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p # Discard step 'add1' does'nt work as 'add2' and 'mult3' depend on it try(p$discard_steps(\"add1\")) p$discard_steps(\"add1\", recursive = TRUE)   # this works p  # Trying to discard non-existent steps is just ignored p$discard_steps(\"non-existent\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-data-","dir":"Reference","previous_headings":"","what":"Method get_data()","title":"Pipeline Class — Pipeline","text":"Get data","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-6","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_data()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-6","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"output defined data step, default first step pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-6","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$get_data() p$set_data(3:4) p$get_data()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-","dir":"Reference","previous_headings":"","what":"Method get_depends()","title":"Pipeline Class — Pipeline","text":"Get dependencies defined pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-7","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-7","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"named list dependencies step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-7","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$get_depends()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-down-","dir":"Reference","previous_headings":"","what":"Method get_depends_down()","title":"Pipeline Class — Pipeline","text":"Get downstream dependencies given step, default descending recursively.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-8","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends_down(step, recursive = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-6","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step recursive logical TRUE, dependencies dependencies also returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-8","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list downstream dependencies","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-8","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_down(\"add1\") p$get_depends_down(\"add1\", recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-up-","dir":"Reference","previous_headings":"","what":"Method get_depends_up()","title":"Pipeline Class — Pipeline","text":"Get upstream dependencies given step, default descending recursively.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-9","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends_up(step, recursive = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-7","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step recursive logical TRUE, dependencies dependencies also returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-9","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list upstream dependencies","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-9","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_up(\"mult4\") p$get_depends_up(\"mult4\", recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-graph-","dir":"Reference","previous_headings":"","what":"Method get_graph()","title":"Pipeline Class — Pipeline","text":"Visualize pipeline graph.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-10","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_graph(groups = NULL)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-8","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"groups character NULL, steps belonging given groups considered.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-10","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"two data frames, one nodes one edges ready used visNetwork package.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-10","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = ~add1, y = ~add2) x * y) if (require(\"visNetwork\", quietly = TRUE)) {     do.call(visNetwork, args = p$get_graph()) }"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-out-","dir":"Reference","previous_headings":"","what":"Method get_out()","title":"Pipeline Class — Pipeline","text":"Get output given step pipeline run.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-11","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_out(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-9","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-11","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"output given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-11","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$run() p$get_out(\"add1\") p$get_out(\"add2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-","dir":"Reference","previous_headings":"","what":"Method get_params()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) function parameters defined pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-12","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-10","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-12","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list parameters, sorted named step. Steps parameters filtered .","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-12","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params() |> str() p$get_params(ignoreHidden = FALSE) |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-at-step-","dir":"Reference","previous_headings":"","what":"Method get_params_at_step()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) given step name.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-13","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_at_step(step, ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-11","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-13","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list parameters defined given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-13","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params_at_step(\"add2\") p$get_params_at_step(\"add2\", ignoreHidden = FALSE) p$get_params_at_step(\"add3\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-unique-","dir":"Reference","previous_headings":"","what":"Method get_params_unique()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) parameters defined pipeline, list parameter . values parameters, values first step parameter defined. particularly useful parameters set using set_params function, set value steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-14","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_unique(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-12","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-14","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list unique parameters","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-14","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique() p$get_params_unique(ignoreHidden = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-unique-json-","dir":"Reference","previous_headings":"","what":"Method get_params_unique_json()","title":"Pipeline Class — Pipeline","text":"Get unique function parameters json format.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-15","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_unique_json(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-13","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-15","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list flat unnamed json list unique function parameters","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-15","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique_json() p$get_params_unique_json(ignoreHidden = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-","dir":"Reference","previous_headings":"","what":"Method get_step()","title":"Pipeline Class — Pipeline","text":"Get step pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-16","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-14","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-16","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"data.table row containing step. step found, error given.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-16","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, z = ~add1) x + y + z) p$run() add1 <- p$get_step(\"add1\") print(add1) add1[[\"params\"]] add1[[\"out\"]] try() try(p$get_step(\"foo\")) # error: step 'foo' does not exist"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-names-","dir":"Reference","previous_headings":"","what":"Method get_step_names()","title":"Pipeline Class — Pipeline","text":"Get step names pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-17","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step_names()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-17","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector step names","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-17","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_names()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-number-","dir":"Reference","previous_headings":"","what":"Method get_step_number()","title":"Pipeline Class — Pipeline","text":"Get step number","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-18","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step_number(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-15","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-18","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"step number pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-18","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_number(\"f2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-has-step-","dir":"Reference","previous_headings":"","what":"Method has_step()","title":"Pipeline Class — Pipeline","text":"Determine whether pipeline given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-19","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$has_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-16","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-19","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"logical whether step exists","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-19","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$has_step(\"f2\") p$has_step(\"foo\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-insert-after-","dir":"Reference","previous_headings":"","what":"Method insert_after()","title":"Pipeline Class — Pipeline","text":"Insert step certain step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-20","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$insert_after(afterStep, step, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-17","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"afterStep string name step insert step string name step insert ... arguments passed add method pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-20","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-20","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_after(\"f1\", \"f3\", \\(x = ~f1) x) p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-insert-before-","dir":"Reference","previous_headings":"","what":"Method insert_before()","title":"Pipeline Class — Pipeline","text":"Insert step certain step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-21","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$insert_before(beforeStep, step, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-18","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"beforeStep string name step insert step string name step insert ... arguments passed add method pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-21","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-21","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_before(\"f2\", \"f3\", \\(x = ~f1) x) p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-length-","dir":"Reference","previous_headings":"","what":"Method length()","title":"Pipeline Class — Pipeline","text":"Length pipeline aka number pipeline steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-22","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$length()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-22","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"numeric length pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-22","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$length()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-lock-step-","dir":"Reference","previous_headings":"","what":"Method lock_step()","title":"Pipeline Class — Pipeline","text":"Locking step means parameters output (given output) locked. output, parameters locked. Locking step useful step happens share parameter names steps affected parameters set commonly entire pipeline (see function set_params ).","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-23","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$lock_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-19","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-23","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-23","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$run() p$get_out(\"add1\") p$get_out(\"add2\") p$lock_step(\"add1\")  p$set_data(3) p$set_params(list(x = 3)) p$run() p$get_out(\"add1\") p$get_out(\"add2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-print-","dir":"Reference","previous_headings":"","what":"Method print()","title":"Pipeline Class — Pipeline","text":"Print pipeline table.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-24","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$print(verbose = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-20","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"verbose logical TRUE, print columns pipeline, otherwise subset columns printed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-24","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-24","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$print()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-step-","dir":"Reference","previous_headings":"","what":"Method pop_step()","title":"Pipeline Class — Pipeline","text":"Remove last step pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-25","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_step()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-25","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"string name step removed","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-25","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p p$pop_step() # \"f2\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-steps-after-","dir":"Reference","previous_headings":"","what":"Method pop_steps_after()","title":"Pipeline Class — Pipeline","text":"Remove steps given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-26","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_steps_after(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-21","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-26","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector steps removed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-26","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_after(\"f1\")  # \"f2\", \"f3\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-steps-from-","dir":"Reference","previous_headings":"","what":"Method pop_steps_from()","title":"Pipeline Class — Pipeline","text":"Remove steps including given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-27","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_steps_from(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-22","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-27","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector steps removed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-27","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_from(\"f2\")  # \"f2\", \"f3\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-remove-step-","dir":"Reference","previous_headings":"","what":"Method remove_step()","title":"Pipeline Class — Pipeline","text":"Remove certain step pipeline. step exist, error given. steps depend step removed, error given, unless recursive = TRUE.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-28","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$remove_step(step, recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-23","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step removed. recursive logical TRUE step removed together downstream dependencies.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-28","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-28","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = 1, y = ~add2) x * y) p$remove_step(\"mult1\") p try(p$remove_step(\"add1\"))  # fails because \"add2\" depends on \"add1\" p$remove_step(\"add1\", recursive = TRUE)  # removes \"add1\" and \"add2\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-rename-step-","dir":"Reference","previous_headings":"","what":"Method rename_step()","title":"Pipeline Class — Pipeline","text":"Safely rename step pipeline. new step name result name clash, error given.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-29","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$rename_step(from, to)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-24","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"string name step renamed. string new name step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-29","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-29","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p try(p$rename_step(\"add1\", \"add2\"))  # fails because \"add2\" exists p$rename_step(\"add1\", \"first_add\")  # Ok p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-replace-step-","dir":"Reference","previous_headings":"","what":"Method replace_step()","title":"Pipeline Class — Pipeline","text":"Replace pipeline step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-30","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$replace_step(   step,   fun,   params = list(),   description = \"\",   group = step,   keepOut = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-25","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step replaced. Step must exist. fun string function operation applied step. existing lambda/anonymous functions can used. params list list parameters overwrite default parameters existing functions. description string optional description step group string grouping information (default name step. output collected later (see function collect_out default put together group names. , example, comes handy pipeline copy-appended multiple times keep results function/step one place. keepOut logical FALSE output function cleaned end whole pipeline execution. option used keep results matter.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-30","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-30","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = 1, y = 2) x * y, keepOut = TRUE) p$run()$collect_out() p$replace_step(\"mult\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() try(p$replace_step(\"foo\", \\(x = 1) x))   # step 'foo' does not exist"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-reset-","dir":"Reference","previous_headings":"","what":"Method reset()","title":"Pipeline Class — Pipeline","text":"Resets pipeline state run. means output removed state steps reset 'New'.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-31","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$reset()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-31","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-31","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$run() p p$reset() p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-run-","dir":"Reference","previous_headings":"","what":"Method run()","title":"Pipeline Class — Pipeline","text":"Run new /outdated pipeline steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-32","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$run(   force = FALSE,   recursive = TRUE,   cleanUnkept = FALSE,   progress = NULL,   showLog = TRUE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-26","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"force logical TRUE steps run regardless whether outdated . recursive logical TRUE step returns new pipeline, run current pipeline aborted new pipeline run recursively. cleanUnkept logical TRUE output marked kept removed pipeline run. option can useful temporary results require lot memory. progress function parameter can used provide custom progress function form function(value, detail), show progress pipeline run step, value current step number detail name step. showLog logical steps logged pipeline run?","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-32","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-32","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Simple pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"final\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() p$set_params(list(z = 4))  # outdates steps add2 and final p p$run()$collect_out() p$run(cleanUnkept = TRUE)  # clean up temporary results p  # Recursive pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"new_pipe\", \\(x = ~add1) {     pp <- Pipeline$new(\"new_pipe\", data = x)     pp$add(\"add1\", \\(x = ~data) x + 1)     pp$add(\"add2\", \\(x = ~add1) x + 2, keepOut = TRUE)     } ) p$run()$collect_out()  # Run pipeline with progress bar p <- Pipeline$new(\"pipe\", data = 1) p$add(\"first step\", \\() Sys.sleep(1)) p$add(\"second step\", \\() Sys.sleep(1)) p$add(\"last step\", \\() Sys.sleep(1)) pb <- txtProgressBar(min = 1, max = p$length(), style = 3) fprogress <- function(value, detail) {    setTxtProgressBar(pb, value) } p$run(progress = fprogress, showLog = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-run-step-","dir":"Reference","previous_headings":"","what":"Method run_step()","title":"Pipeline Class — Pipeline","text":"Run given pipeline step possibly together upstream downstream dependencies.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-33","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$run_step(   step,   upstream = TRUE,   downstream = FALSE,   cleanUnkept = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-27","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step upstream logical TRUE, run dependent upstream steps first. downstream logical TRUE, run depdendent downstream afterwards. cleanUnkept logical TRUE output marked kept removed pipeline run. option can useful temporary results require lot memory.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-33","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-33","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run_step(\"add2\") p$run_step(\"add2\", downstream = TRUE) p$run_step(\"mult\", upstream = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-data-","dir":"Reference","previous_headings":"","what":"Method set_data()","title":"Pipeline Class — Pipeline","text":"Set data first step pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-34","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_data(data)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-28","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"data data.frame initial data set.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-34","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-34","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$run()$collect_out() p$set_data(3) p$run()$collect_out()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-data-split-","dir":"Reference","previous_headings":"","what":"Method set_data_split()","title":"Pipeline Class — Pipeline","text":"Split-copy pipeline list data sets. sub-pipeline one data sets set input data. step names sub-pipelines original step names plus name data set.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-35","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_data_split(   dataList,   toStep = utils::tail(self$get_step_names(), 1),   groupBySplit = TRUE,   sep = \".\" )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-29","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"dataList list data sets toStep string step name marking optional subset pipeline, data split applied . groupBySplit logical whether set step groups according data split. sep string separator used step name data set name creating new step names.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-35","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"new combined Pipeline sub-pipeline set one data sets.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-35","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList) p3 p3$run()$collect_out() |> str()  # Don't group output by split p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList, groupBySplit = FALSE) p3 p3$run()$collect_out() |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-keep-out-","dir":"Reference","previous_headings":"","what":"Method set_keep_out()","title":"Pipeline Class — Pipeline","text":"Change keepOut flag given pipeline step, determines whether output step collected calling collect_out() pipeline run.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-36","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_keep_out(step, keepOut = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-30","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step keepOut logical whether keep output step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-36","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-36","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run()$collect_out() p$set_keep_out(\"add1\", keepOut = FALSE) p$set_keep_out(\"mult\", keepOut = TRUE) p$collect_out()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-params-","dir":"Reference","previous_headings":"","what":"Method set_params()","title":"Pipeline Class — Pipeline","text":"Set parameters pipeline. parameter occurs several steps, parameter set commonly steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-37","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_params(params, warnUndefined = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-31","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"params list parameters set warnUndefined logical whether give warning parameter defined pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-37","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-37","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 1) x + y) p$add(\"mult\", \\(x = 1, z = 1) x * z) p$get_params() p$set_params(list(x = 3, y = 3)) p$get_params() p$set_params(list(x = 5, z = 3)) p$get_params() suppressWarnings(     p$set_params(list(foo = 3))  # warning: trying to set undefined )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-params-at-step-","dir":"Reference","previous_headings":"","what":"Method set_params_at_step()","title":"Pipeline Class — Pipeline","text":"Set unbound parameter values given pipeline step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-38","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_params_at_step(step, params)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-32","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step params list parameters set","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-38","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-38","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1, z = 2) x + y) p$add(\"add2\", \\(x = ~data, y = 1, z = 2) x + y) p$set_params_at_step(\"add1\", list(y = 3, z = 3)) p$get_params() try(p$set_params_at_step(\"add1\", list(foo = 3))) # foo not defined"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-split-","dir":"Reference","previous_headings":"","what":"Method split()","title":"Pipeline Class — Pipeline","text":"Splits pipeline independent parts.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-39","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$split()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-39","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list Pipeline objects","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-39","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Example for two independent calculation paths p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = ~data) x) p$add(\"f2\", \\(x = 1) x) p$add(\"f3\", \\(x = ~f1) x) p$add(\"f4\", \\(x = ~f2) x) p$split()  # Example of split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) pips <- p$set_data_split(dataList)$split()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-unlock-step-","dir":"Reference","previous_headings":"","what":"Method unlock_step()","title":"Pipeline Class — Pipeline","text":"Unlock previously locked step. step locked, command ignored.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-40","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$unlock_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-33","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-40","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-40","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$lock_step(\"add1\") p$set_params(list(x = 3)) p$get_params() p$unlock_step(\"add1\") p$set_params(list(x = 3)) p$get_params()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-clone-","dir":"Reference","previous_headings":"","what":"Method clone()","title":"Pipeline Class — Pipeline","text":"objects class cloneable method.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-41","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$clone(deep = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-34","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"deep Whether make deep clone.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"## ------------------------------------------------ ## Method `Pipeline$new` ## ------------------------------------------------  p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:8)) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New  # Passing custom logger my_logger <- function(level, msg, ...) {    cat(level, msg, \"\\n\") } p <- Pipeline$new(\"myPipe\", logger = my_logger)  ## ------------------------------------------------ ## Method `Pipeline$add` ## ------------------------------------------------  # Add steps with lambda functions p <- Pipeline$new(\"myPipe\", data = 1) p$add(\"s1\", \\(x = ~data) 2*x)  # use input data p$add(\"s2\", \\(x = ~data, y = ~s1) x * y) try(p$add(\"s2\", \\(z = 3) 3)) # error: step 's2' exists already #> Error : step 's2' already exists try(p$add(\"s3\", \\(z = ~foo) 3)) # dependency 'foo' not found #> Error : step 's3': dependency 'foo' not found p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     s1    data [NULL]   FALSE     s1    New #> 3:     s2 data,s1 [NULL]   FALSE     s2    New  p <- Pipeline$new(\"myPipe\", data = c(1, 2, NA, 3, 4)) p$add(\"calc_mean\", mean, params = list(x = ~data, na.rm = TRUE)) p$run()$get_out(\"calc_mean\") #> INFO  [2024-12-01 21:41:21.908] Start run of 'myPipe' pipeline: #> INFO  [2024-12-01 21:41:21.933] Step 1/2 data #> INFO  [2024-12-01 21:41:21.942] Step 2/2 calc_mean #> INFO  [2024-12-01 21:41:21.961] Finished execution of steps. #> INFO  [2024-12-01 21:41:21.962] Done. #> [1] 2.5  p <- Pipeline$new(\"myPipe\", data = 1:10) p$add(\"s1\", \\(x = ~data) 2*x, description = \"multiply by 2\") print(p) #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     s1    data [NULL]   FALSE     s1    New print(p, verbose = TRUE) # print all columns #>      step           fun funcName    params depends    out keepOut  group #>    <char>        <list>   <char>    <list>  <list> <list>  <lgcl> <char> #> 1:   data <function[1]> function <list[0]>         [NULL]   FALSE   data #> 2:     s1 <function[1]> function <list[1]>    data [NULL]   FALSE     s1 #>      description                time  state #>           <char>              <POSc> <char> #> 1:               2024-12-01 21:41:21    New #> 2: multiply by 2 2024-12-01 21:41:21    New  p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:5, y = 1:5)) p$add(\"prep_x\", \\(data = ~data) data$x, group = \"prep\") p$add(\"prep_y\", \\(data = ~data) (data$y)^2, group = \"prep\") p$add(\"sum\", \\(x = ~prep_x, y = ~prep_y) x + y) p$run()$collect_out(all = TRUE) #> INFO  [2024-12-01 21:41:21.975] Start run of 'myPipe' pipeline: #> INFO  [2024-12-01 21:41:21.976] Step 1/4 data #> INFO  [2024-12-01 21:41:21.979] Step 2/4 prep_x #> INFO  [2024-12-01 21:41:21.981] Step 3/4 prep_y #> INFO  [2024-12-01 21:41:21.983] Step 4/4 sum #> INFO  [2024-12-01 21:41:21.984] Finished execution of steps. #> INFO  [2024-12-01 21:41:21.985] Done. #> $data #>   x y #> 1 1 1 #> 2 2 2 #> 3 3 3 #> 4 4 4 #> 5 5 5 #>  #> $prep #> $prep$prep_x #> [1] 1 2 3 4 5 #>  #> $prep$prep_y #> [1]  1  4  9 16 25 #>  #>  #> $sum #> [1]  2  6 12 20 30 #>   ## ------------------------------------------------ ## Method `Pipeline$append` ## ------------------------------------------------  # Append pipeline p1 <- Pipeline$new(\"pipe1\") p1$add(\"step1\", \\(x = 1) x) p2 <- Pipeline$new(\"pipe2\") p2$add(\"step2\", \\(y = 1) y) p1$append(p2) #>          step depends    out keepOut  group  state #>        <char>  <list> <list>  <lgcl> <char> <char> #> 1:       data         [NULL]   FALSE   data    New #> 2:      step1         [NULL]   FALSE  step1    New #> 3: data.pipe2         [NULL]   FALSE   data    New #> 4:      step2         [NULL]   FALSE  step2    New  p3 <- Pipeline$new(\"pipe3\") p3$add(\"step1\", \\(z = 1) z) p1$append(p2)$append(p3) #>           step depends    out keepOut  group  state #>         <char>  <list> <list>  <lgcl> <char> <char> #> 1:        data         [NULL]   FALSE   data    New #> 2:       step1         [NULL]   FALSE  step1    New #> 3:  data.pipe2         [NULL]   FALSE   data    New #> 4:       step2         [NULL]   FALSE  step2    New #> 5:  data.pipe3         [NULL]   FALSE   data    New #> 6: step1.pipe3         [NULL]   FALSE  step1    New  ## ------------------------------------------------ ## Method `Pipeline$append_to_step_names` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\") p$add(\"step1\", \\(x = 1) x) p$add(\"step2\", \\(y = 1) y) p$append_to_step_names(\"new\") p #>         step depends    out keepOut  group  state #>       <char>  <list> <list>  <lgcl> <char> <char> #> 1:  data.new         [NULL]   FALSE   data    New #> 2: step1.new         [NULL]   FALSE  step1    New #> 3: step2.new         [NULL]   FALSE  step2    New p$append_to_step_names(\"new\", sep = \"_\") p #>             step depends    out keepOut  group  state #>           <char>  <list> <list>  <lgcl> <char> <char> #> 1:  data.new_new         [NULL]   FALSE   data    New #> 2: step1.new_new         [NULL]   FALSE  step1    New #> 3: step2.new_new         [NULL]   FALSE  step2    New  ## ------------------------------------------------ ## Method `Pipeline$collect_out` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2) p$add(\"step2\", \\(x = ~step1) x + 2, keepOut = TRUE) p$run() #> INFO  [2024-12-01 21:41:22.118] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.119] Step 1/3 data #> INFO  [2024-12-01 21:41:22.121] Step 2/3 step1 #> INFO  [2024-12-01 21:41:22.123] Step 3/3 step2 #> INFO  [2024-12-01 21:41:22.124] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.125] Done. p$collect_out() #> $step2 #> [1] 5 6 #>  p$collect_out(all = TRUE) |> str() #> List of 3 #>  $ data : int [1:2] 1 2 #>  $ step1: num [1:2] 3 4 #>  $ step2: num [1:2] 5 6  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2, group = \"add\") p$add(\"step2\", \\(x = ~step1, y = 2) x + y, group = \"add\") p$add(\"step3\", \\(x = ~data) x * 3, group = \"mult\") p$add(\"step4\", \\(x = ~data, y = 2) x * y, group = \"mult\") p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:  step1    data [NULL]   FALSE    add    New #> 3:  step2   step1 [NULL]   FALSE    add    New #> 4:  step3    data [NULL]   FALSE   mult    New #> 5:  step4    data [NULL]   FALSE   mult    New p$run() #> INFO  [2024-12-01 21:41:22.154] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.155] Step 1/5 data #> INFO  [2024-12-01 21:41:22.157] Step 2/5 step1 #> INFO  [2024-12-01 21:41:22.159] Step 3/5 step2 #> INFO  [2024-12-01 21:41:22.160] Step 4/5 step3 #> INFO  [2024-12-01 21:41:22.162] Step 5/5 step4 #> INFO  [2024-12-01 21:41:22.163] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.164] Done. p$collect_out(all = TRUE) |> str() #> List of 3 #>  $ data: int [1:2] 1 2 #>  $ add :List of 2 #>   ..$ step1: num [1:2] 3 4 #>   ..$ step2: num [1:2] 5 6 #>  $ mult:List of 2 #>   ..$ step3: num [1:2] 3 6 #>   ..$ step4: num [1:2] 2 4  # Grouped by state p$set_params(list(y = 5)) p #>      step depends    out keepOut  group    state #>    <char>  <list> <list>  <lgcl> <char>   <char> #> 1:   data            1,2   FALSE   data     Done #> 2:  step1    data    3,4   FALSE    add     Done #> 3:  step2   step1    5,6   FALSE    add Outdated #> 4:  step3    data    3,6   FALSE   mult     Done #> 5:  step4    data    2,4   FALSE   mult Outdated p$collect_out(groupBy = \"state\", all = TRUE) |> str() #> List of 2 #>  $ Done    :List of 3 #>   ..$ data : int [1:2] 1 2 #>   ..$ step1: num [1:2] 3 4 #>   ..$ step3: num [1:2] 3 6 #>  $ Outdated:List of 2 #>   ..$ step2: num [1:2] 5 6 #>   ..$ step4: num [1:2] 2 4  ## ------------------------------------------------ ## Method `Pipeline$discard_steps` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~add1) x + 2) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$discard_steps(\"mult\") #> step 'mult4' was removed #> step 'mult3' was removed p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New  # Re-add steps p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New #> 4:  mult3    add1 [NULL]   FALSE  mult3    New #> 5:  mult4    add2 [NULL]   FALSE  mult4    New # Discard step 'add1' does'nt work as 'add2' and 'mult3' depend on it try(p$discard_steps(\"add1\")) #> Error in self$remove_step(step, recursive = recursive) :  #>   cannot remove step 'add1' because the following steps depend on it: 'add2', 'mult3' p$discard_steps(\"add1\", recursive = TRUE)   # this works #> Removing step 'add1' and its downstream dependencies: 'add2', 'mult3', 'mult4' #> step 'add1' was removed p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New  # Trying to discard non-existent steps is just ignored p$discard_steps(\"non-existent\")  ## ------------------------------------------------ ## Method `Pipeline$get_data` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$get_data() #> [1] 1 2 p$set_data(3:4) p$get_data() #> [1] 3 4  ## ------------------------------------------------ ## Method `Pipeline$get_depends` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$get_depends() #> $data #> character(0) #>  #> $add1 #>      x  #> \"data\"  #>  #> $add2 #>      x      y  #> \"data\" \"add1\"  #>   ## ------------------------------------------------ ## Method `Pipeline$get_depends_down` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_down(\"add1\") #> [1] \"add2\"  \"mult3\" \"mult4\" p$get_depends_down(\"add1\", recursive = FALSE) #> [1] \"add2\"  \"mult3\"  ## ------------------------------------------------ ## Method `Pipeline$get_depends_up` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_up(\"mult4\") #> [1] \"data\" \"add1\" \"add2\" p$get_depends_up(\"mult4\", recursive = FALSE) #> [1] \"add2\"  ## ------------------------------------------------ ## Method `Pipeline$get_graph` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = ~add1, y = ~add2) x * y) if (require(\"visNetwork\", quietly = TRUE)) {     do.call(visNetwork, args = p$get_graph()) }  {\"x\":{\"nodes\":{\"id\":[1,2,3,4],\"label\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"group\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"shape\":[\"database\",\"box\",\"box\",\"box\"],\"color\":[\"lightblue\",\"lightblue\",\"lightblue\",\"lightblue\"],\"title\":[\"<p><\\/p>\",\"<p><\\/p>\",\"<p><\\/p>\",\"<p><\\/p>\"]},\"edges\":{\"from\":[1,2,2,3],\"to\":[2,3,4,4],\"arrows\":[\"to\",\"to\",\"to\",\"to\"]},\"nodesToDataframe\":true,\"edgesToDataframe\":true,\"options\":{\"width\":\"100%\",\"height\":\"100%\",\"nodes\":{\"shape\":\"dot\"},\"manipulation\":{\"enabled\":false}},\"groups\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"width\":null,\"height\":null,\"idselection\":{\"enabled\":false},\"byselection\":{\"enabled\":false},\"main\":null,\"submain\":null,\"footer\":null,\"background\":\"rgba(0, 0, 0, 0)\"},\"evals\":[],\"jsHooks\":[]} ## ------------------------------------------------ ## Method `Pipeline$get_out` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$run() #> INFO  [2024-12-01 21:41:22.273] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.274] Step 1/3 data #> INFO  [2024-12-01 21:41:22.276] Step 2/3 add1 #> INFO  [2024-12-01 21:41:22.278] Step 3/3 add2 #> INFO  [2024-12-01 21:41:22.279] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.280] Done. p$get_out(\"add1\") #> [1] 2 3 p$get_out(\"add2\") #> [1] 3 5  ## ------------------------------------------------ ## Method `Pipeline$get_params` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params() |> str() #> List of 2 #>  $ add1:List of 1 #>   ..$ x: num 1 #>  $ add2:List of 2 #>   ..$ x: num 1 #>   ..$ y: num 2 p$get_params(ignoreHidden = FALSE) |> str() #> List of 2 #>  $ add1:List of 1 #>   ..$ x: num 1 #>  $ add2:List of 3 #>   ..$ x : num 1 #>   ..$ y : num 2 #>   ..$ .z: num 3  ## ------------------------------------------------ ## Method `Pipeline$get_params_at_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params_at_step(\"add2\") #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  p$get_params_at_step(\"add2\", ignoreHidden = FALSE) #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  #> $.z #> [1] 3 #>  p$get_params_at_step(\"add3\") #> list()  ## ------------------------------------------------ ## Method `Pipeline$get_params_unique` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique() #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  p$get_params_unique(ignoreHidden = FALSE) #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  #> $.z #> [1] 3 #>   ## ------------------------------------------------ ## Method `Pipeline$get_params_unique_json` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique_json() #> [ #>   { #>     \"name\": \"x\", #>     \"value\": 1 #>   }, #>   { #>     \"name\": \"y\", #>     \"value\": 2 #>   } #> ]  p$get_params_unique_json(ignoreHidden = FALSE) #> [ #>   { #>     \"name\": \"x\", #>     \"value\": 1 #>   }, #>   { #>     \"name\": \"y\", #>     \"value\": 2 #>   }, #>   { #>     \"name\": \".z\", #>     \"value\": 3 #>   } #> ]   ## ------------------------------------------------ ## Method `Pipeline$get_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, z = ~add1) x + y + z) p$run() #> INFO  [2024-12-01 21:41:22.322] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.323] Step 1/3 data #> INFO  [2024-12-01 21:41:22.325] Step 2/3 add1 #> INFO  [2024-12-01 21:41:22.326] Step 3/3 add2 #> INFO  [2024-12-01 21:41:22.328] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.328] Done. add1 <- p$get_step(\"add1\") print(add1) #>      step           fun funcName    params depends    out keepOut  group #>    <char>        <list>   <char>    <list>  <list> <list>  <lgcl> <char> #> 1:   add1 <function[1]> function <list[2]>    data    2,3   FALSE   add1 #>    description                time  state #>         <char>              <POSc> <char> #> 1:             2024-12-01 21:41:22   Done add1[[\"params\"]] #> [[1]] #> [[1]]$data #> ~data #> <environment: 0x000002328176fc38> #>  #> [[1]]$x #> [1] 1 #>  #>  add1[[\"out\"]] #> [[1]] #> [1] 2 3 #>  try() #> Error in try() : argument \"expr\" is missing, with no default try(p$get_step(\"foo\")) # error: step 'foo' does not exist #> Error : step 'foo' does not exist  ## ------------------------------------------------ ## Method `Pipeline$get_step_names` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_names() #> [1] \"data\" \"f1\"   \"f2\"    ## ------------------------------------------------ ## Method `Pipeline$get_step_number` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_number(\"f2\") #> [1] 3  ## ------------------------------------------------ ## Method `Pipeline$has_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$has_step(\"f2\") #> [1] TRUE p$has_step(\"foo\") #> [1] FALSE  ## ------------------------------------------------ ## Method `Pipeline$insert_after` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_after(\"f1\", \"f3\", \\(x = ~f1) x) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f3      f1 [NULL]   FALSE     f3    New #> 4:     f2      f1 [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$insert_before` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_before(\"f2\", \"f3\", \\(x = ~f1) x) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f3      f1 [NULL]   FALSE     f3    New #> 4:     f2      f1 [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$length` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$length() #> [1] 3  ## ------------------------------------------------ ## Method `Pipeline$lock_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$run() #> INFO  [2024-12-01 21:41:22.370] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.371] Step 1/3 data #> INFO  [2024-12-01 21:41:22.373] Step 2/3 add1 #> INFO  [2024-12-01 21:41:22.375] Step 3/3 add2 #> INFO  [2024-12-01 21:41:22.376] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.376] Done. p$get_out(\"add1\") #> [1] 2 p$get_out(\"add2\") #> [1] 2 p$lock_step(\"add1\")  p$set_data(3) p$set_params(list(x = 3)) #> skipping setting parameters x at locked step 'add1' p$run() #> INFO  [2024-12-01 21:41:22.396] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.397] Step 1/3 data #> INFO  [2024-12-01 21:41:22.399] Step 2/3 add1 - skip 'locked' step #> INFO  [2024-12-01 21:41:22.400] Step 3/3 add2 #> INFO  [2024-12-01 21:41:22.401] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.402] Done. p$get_out(\"add1\") #> [1] 2 p$get_out(\"add2\") #> [1] 6  ## ------------------------------------------------ ## Method `Pipeline$print` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$print() #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f2         [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$pop_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f2         [NULL]   FALSE     f2    New p$pop_step() # \"f2\" #> [1] \"f2\" p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New  ## ------------------------------------------------ ## Method `Pipeline$pop_steps_after` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_after(\"f1\")  # \"f2\", \"f3\" #> [1] \"f2\" \"f3\" p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New  ## ------------------------------------------------ ## Method `Pipeline$pop_steps_from` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_from(\"f2\")  # \"f2\", \"f3\" #> [1] \"f2\" \"f3\" p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New  ## ------------------------------------------------ ## Method `Pipeline$remove_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = 1, y = ~add2) x * y) p$remove_step(\"mult1\") p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New try(p$remove_step(\"add1\"))  # fails because \"add2\" depends on \"add1\" #> Error in p$remove_step(\"add1\") :  #>   cannot remove step 'add1' because the following steps depend on it: 'add2' p$remove_step(\"add1\", recursive = TRUE)  # removes \"add1\" and \"add2\" #> Removing step 'add1' and its downstream dependencies: 'add2' p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New  ## ------------------------------------------------ ## Method `Pipeline$rename_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New try(p$rename_step(\"add1\", \"add2\"))  # fails because \"add2\" exists #> Error : step 'add2' already exists p$rename_step(\"add1\", \"first_add\")  # Ok p #>         step   depends    out keepOut  group  state #>       <char>    <list> <list>  <lgcl> <char> <char> #> 1:      data           [NULL]   FALSE   data    New #> 2: first_add      data [NULL]   FALSE   add1    New #> 3:      add2 first_add [NULL]   FALSE   add2    New  ## ------------------------------------------------ ## Method `Pipeline$replace_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = 1, y = 2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:41:22.462] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.463] Step 1/4 data #> INFO  [2024-12-01 21:41:22.465] Step 2/4 add1 #> INFO  [2024-12-01 21:41:22.466] Step 3/4 add2 #> INFO  [2024-12-01 21:41:22.468] Step 4/4 mult #> INFO  [2024-12-01 21:41:22.469] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.469] Done. #> $mult #> [1] 2 #>  p$replace_step(\"mult\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:41:22.472] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.473] Step 1/4 data - skip 'done' step #> INFO  [2024-12-01 21:41:22.474] Step 2/4 add1 - skip 'done' step #> INFO  [2024-12-01 21:41:22.475] Step 3/4 add2 - skip 'done' step #> INFO  [2024-12-01 21:41:22.476] Step 4/4 mult #> INFO  [2024-12-01 21:41:22.477] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.477] Done. #> $mult #> [1] 6 #>  try(p$replace_step(\"foo\", \\(x = 1) x))   # step 'foo' does not exist #> Error : step 'foo' does not exist  ## ------------------------------------------------ ## Method `Pipeline$reset` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$run() #> INFO  [2024-12-01 21:41:22.493] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.494] Step 1/3 data #> INFO  [2024-12-01 21:41:22.495] Step 2/3 f1 #> INFO  [2024-12-01 21:41:22.496] Step 3/3 f2 #> INFO  [2024-12-01 21:41:22.498] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.498] Done. p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data            1,2   FALSE   data   Done #> 2:     f1              1   FALSE     f1   Done #> 3:     f2              1   FALSE     f2   Done p$reset() p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f2         [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$run` ## ------------------------------------------------  # Simple pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"final\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:41:22.508] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.509] Step 1/4 data #> INFO  [2024-12-01 21:41:22.511] Step 2/4 add1 #> INFO  [2024-12-01 21:41:22.513] Step 3/4 add2 #> INFO  [2024-12-01 21:41:22.515] Step 4/4 final #> INFO  [2024-12-01 21:41:22.516] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.517] Done. #> $final #> [1] 8 #>  p$set_params(list(z = 4))  # outdates steps add2 and final p #>      step   depends    out keepOut  group    state #>    <char>    <list> <list>  <lgcl> <char>   <char> #> 1:   data                1   FALSE   data     Done #> 2:   add1      data      2   FALSE   add1     Done #> 3:   add2      add1      4   FALSE   add2 Outdated #> 4:  final add1,add2      8    TRUE  final Outdated p$run()$collect_out() #> INFO  [2024-12-01 21:41:22.524] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.525] Step 1/4 data - skip 'done' step #> INFO  [2024-12-01 21:41:22.526] Step 2/4 add1 - skip 'done' step #> INFO  [2024-12-01 21:41:22.526] Step 3/4 add2 #> INFO  [2024-12-01 21:41:22.528] Step 4/4 final #> INFO  [2024-12-01 21:41:22.529] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.530] Done. #> $final #> [1] 12 #>  p$run(cleanUnkept = TRUE)  # clean up temporary results #> INFO  [2024-12-01 21:41:22.531] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.532] Step 1/4 data - skip 'done' step #> INFO  [2024-12-01 21:41:22.533] Step 2/4 add1 - skip 'done' step #> INFO  [2024-12-01 21:41:22.534] Step 3/4 add2 - skip 'done' step #> INFO  [2024-12-01 21:41:22.535] Step 4/4 final - skip 'done' step #> INFO  [2024-12-01 21:41:22.535] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.536] Clean temporary results. #> INFO  [2024-12-01 21:41:22.536] Done. p #>      step   depends    out keepOut  group    state #>    <char>    <list> <list>  <lgcl> <char>   <char> #> 1:   data           [NULL]   FALSE   data Outdated #> 2:   add1      data [NULL]   FALSE   add1 Outdated #> 3:   add2      add1 [NULL]   FALSE   add2 Outdated #> 4:  final add1,add2     12    TRUE  final     Done  # Recursive pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"new_pipe\", \\(x = ~add1) {     pp <- Pipeline$new(\"new_pipe\", data = x)     pp$add(\"add1\", \\(x = ~data) x + 1)     pp$add(\"add2\", \\(x = ~add1) x + 2, keepOut = TRUE)     } ) p$run()$collect_out() #> INFO  [2024-12-01 21:41:22.543] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:22.544] Step 1/3 data #> INFO  [2024-12-01 21:41:22.546] Step 2/3 add1 #> INFO  [2024-12-01 21:41:22.547] Step 3/3 new_pipe #> INFO  [2024-12-01 21:41:22.550] Abort pipeline execution and restart on new. #> INFO  [2024-12-01 21:41:22.551] Start run of 'new_pipe' pipeline: #> INFO  [2024-12-01 21:41:22.552] Step 1/3 data #> INFO  [2024-12-01 21:41:22.554] Step 2/3 add1 #> INFO  [2024-12-01 21:41:22.556] Step 3/3 add2 #> INFO  [2024-12-01 21:41:22.557] Finished execution of steps. #> INFO  [2024-12-01 21:41:22.557] Done. #> $add2 #> [1] 5 #>   # Run pipeline with progress bar p <- Pipeline$new(\"pipe\", data = 1) p$add(\"first step\", \\() Sys.sleep(1)) p$add(\"second step\", \\() Sys.sleep(1)) p$add(\"last step\", \\() Sys.sleep(1)) pb <- txtProgressBar(min = 1, max = p$length(), style = 3) fprogress <- function(value, detail) {    setTxtProgressBar(pb, value) } p$run(progress = fprogress, showLog = FALSE) #>    |                                                                               |                                                                      |   0%   |                                                                               |=======================                                               |  33%   |                                                                               |===============================================                       |  67%   |                                                                               |======================================================================| 100%  ## ------------------------------------------------ ## Method `Pipeline$run_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run_step(\"add2\") #> INFO  [2024-12-01 21:41:25.633] Start step run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.634] Step 1/3 data (upstream) #> INFO  [2024-12-01 21:41:25.636] Step 2/3 add1 (upstream) #> INFO  [2024-12-01 21:41:25.638] Step 3/3 add2 #> INFO  [2024-12-01 21:41:25.640] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.640] Done. p$run_step(\"add2\", downstream = TRUE) #> INFO  [2024-12-01 21:41:25.655] Start step run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.655] Step 1/4 data (upstream) #> INFO  [2024-12-01 21:41:25.658] Step 2/4 add1 (upstream) #> INFO  [2024-12-01 21:41:25.660] Step 3/4 add2 #> INFO  [2024-12-01 21:41:25.664] Step 4/4 mult (downstream) #> INFO  [2024-12-01 21:41:25.665] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.666] Done. p$run_step(\"mult\", upstream = TRUE) #> INFO  [2024-12-01 21:41:25.667] Start step run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.668] Step 1/4 data (upstream) #> INFO  [2024-12-01 21:41:25.670] Step 2/4 add1 (upstream) #> INFO  [2024-12-01 21:41:25.672] Step 3/4 add2 (upstream) #> INFO  [2024-12-01 21:41:25.674] Step 4/4 mult #> INFO  [2024-12-01 21:41:25.675] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.675] Done.  ## ------------------------------------------------ ## Method `Pipeline$set_data` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:41:25.679] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.680] Step 1/2 data #> INFO  [2024-12-01 21:41:25.682] Step 2/2 add1 #> INFO  [2024-12-01 21:41:25.683] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.684] Done. #> $add1 #> [1] 2 #>  p$set_data(3) p$run()$collect_out() #> INFO  [2024-12-01 21:41:25.687] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.688] Step 1/2 data #> INFO  [2024-12-01 21:41:25.690] Step 2/2 add1 #> INFO  [2024-12-01 21:41:25.691] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.692] Done. #> $add1 #> [1] 4 #>   ## ------------------------------------------------ ## Method `Pipeline$set_data_split` ## ------------------------------------------------  # Split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList) p3 #>      step       depends    out keepOut  group    state #>    <char>        <list> <list>  <lgcl> <char>   <char> #> 1: data.a               [NULL]   FALSE      a      New #> 2: add1.a        data.a [NULL]    TRUE      a Outdated #> 3: mult.a data.a,add1.a [NULL]    TRUE      a Outdated #> 4: data.b               [NULL]   FALSE      b      New #> 5: add1.b        data.b [NULL]    TRUE      b Outdated #> 6: mult.b data.b,add1.b [NULL]    TRUE      b Outdated #> 7: data.c               [NULL]   FALSE      c      New #> 8: add1.c        data.c [NULL]    TRUE      c Outdated #> 9: mult.c data.c,add1.c [NULL]    TRUE      c Outdated p3$run()$collect_out() |> str() #> INFO  [2024-12-01 21:41:25.714] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.715] Step 1/9 data.a #> INFO  [2024-12-01 21:41:25.717] Step 2/9 add1.a #> INFO  [2024-12-01 21:41:25.719] Step 3/9 mult.a #> INFO  [2024-12-01 21:41:25.721] Step 4/9 data.b #> INFO  [2024-12-01 21:41:25.723] Step 5/9 add1.b #> INFO  [2024-12-01 21:41:25.725] Step 6/9 mult.b #> INFO  [2024-12-01 21:41:25.726] Step 7/9 data.c #> INFO  [2024-12-01 21:41:25.728] Step 8/9 add1.c #> INFO  [2024-12-01 21:41:25.730] Step 9/9 mult.c #> INFO  [2024-12-01 21:41:25.731] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.732] Done. #> List of 3 #>  $ a:List of 2 #>   ..$ add1.a: num 2 #>   ..$ mult.a: num 2 #>  $ b:List of 2 #>   ..$ add1.b: num 3 #>   ..$ mult.b: num 6 #>  $ c:List of 2 #>   ..$ add1.c: num 4 #>   ..$ mult.c: num 12  # Don't group output by split p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList, groupBySplit = FALSE) p3 #>      step       depends    out keepOut  group    state #>    <char>        <list> <list>  <lgcl> <char>   <char> #> 1: data.a               [NULL]   FALSE data.a      New #> 2: add1.a        data.a [NULL]    TRUE add1.a Outdated #> 3: mult.a data.a,add1.a [NULL]    TRUE mult.a Outdated #> 4: data.b               [NULL]   FALSE data.b      New #> 5: add1.b        data.b [NULL]    TRUE add1.b Outdated #> 6: mult.b data.b,add1.b [NULL]    TRUE mult.b Outdated #> 7: data.c               [NULL]   FALSE data.c      New #> 8: add1.c        data.c [NULL]    TRUE add1.c Outdated #> 9: mult.c data.c,add1.c [NULL]    TRUE mult.c Outdated p3$run()$collect_out() |> str() #> INFO  [2024-12-01 21:41:25.757] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.758] Step 1/9 data.a #> INFO  [2024-12-01 21:41:25.760] Step 2/9 add1.a #> INFO  [2024-12-01 21:41:25.762] Step 3/9 mult.a #> INFO  [2024-12-01 21:41:25.764] Step 4/9 data.b #> INFO  [2024-12-01 21:41:25.766] Step 5/9 add1.b #> INFO  [2024-12-01 21:41:25.767] Step 6/9 mult.b #> INFO  [2024-12-01 21:41:25.769] Step 7/9 data.c #> INFO  [2024-12-01 21:41:25.771] Step 8/9 add1.c #> INFO  [2024-12-01 21:41:25.773] Step 9/9 mult.c #> INFO  [2024-12-01 21:41:25.774] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.775] Done. #> List of 6 #>  $ add1.a: num 2 #>  $ mult.a: num 2 #>  $ add1.b: num 3 #>  $ mult.b: num 6 #>  $ add1.c: num 4 #>  $ mult.c: num 12  ## ------------------------------------------------ ## Method `Pipeline$set_keep_out` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run()$collect_out() #> INFO  [2024-12-01 21:41:25.783] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:25.784] Step 1/4 data #> INFO  [2024-12-01 21:41:25.786] Step 2/4 add1 #> INFO  [2024-12-01 21:41:25.788] Step 3/4 add2 #> INFO  [2024-12-01 21:41:25.790] Step 4/4 mult #> INFO  [2024-12-01 21:41:25.791] Finished execution of steps. #> INFO  [2024-12-01 21:41:25.792] Done. #> $add1 #> [1] 2 #>  p$set_keep_out(\"add1\", keepOut = FALSE) p$set_keep_out(\"mult\", keepOut = TRUE) p$collect_out() #> $mult #> [1] 6 #>   ## ------------------------------------------------ ## Method `Pipeline$set_params` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 1) x + y) p$add(\"mult\", \\(x = 1, z = 1) x * z) p$get_params() #> $add1 #> $add1$y #> [1] 1 #>  #>  #> $add2 #> $add2$y #> [1] 1 #>  #>  #> $mult #> $mult$x #> [1] 1 #>  #> $mult$z #> [1] 1 #>  #>  p$set_params(list(x = 3, y = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #>  #>  #> $add2 #> $add2$y #> [1] 3 #>  #>  #> $mult #> $mult$x #> [1] 3 #>  #> $mult$z #> [1] 1 #>  #>  p$set_params(list(x = 5, z = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #>  #>  #> $add2 #> $add2$y #> [1] 3 #>  #>  #> $mult #> $mult$x #> [1] 5 #>  #> $mult$z #> [1] 3 #>  #>  suppressWarnings(     p$set_params(list(foo = 3))  # warning: trying to set undefined )  ## ------------------------------------------------ ## Method `Pipeline$set_params_at_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1, z = 2) x + y) p$add(\"add2\", \\(x = ~data, y = 1, z = 2) x + y) p$set_params_at_step(\"add1\", list(y = 3, z = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #>  #> $add1$z #> [1] 3 #>  #>  #> $add2 #> $add2$y #> [1] 1 #>  #> $add2$z #> [1] 2 #>  #>  try(p$set_params_at_step(\"add1\", list(foo = 3))) # foo not defined #> Error in p$set_params_at_step(\"add1\", list(foo = 3)) :  #>   Unable to set parameter(s) foo at step add1 - candidates are y, z  ## ------------------------------------------------ ## Method `Pipeline$split` ## ------------------------------------------------  # Example for two independent calculation paths p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = ~data) x) p$add(\"f2\", \\(x = 1) x) p$add(\"f3\", \\(x = ~f1) x) p$add(\"f4\", \\(x = ~f2) x) p$split() #> [[1]] #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1    data [NULL]   FALSE     f1    New #> 3:     f3      f1 [NULL]   FALSE     f3    New #>  #> [[2]] #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:     f2         [NULL]   FALSE     f2    New #> 2:     f4      f2 [NULL]   FALSE     f4    New #>   # Example of split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) pips <- p$set_data_split(dataList)$split()  ## ------------------------------------------------ ## Method `Pipeline$unlock_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$lock_step(\"add1\") p$set_params(list(x = 3)) #> skipping setting parameters x at locked step 'add1' p$get_params() #> $add1 #> $add1$x #> [1] 1 #>  #>  #> $add2 #> $add2$x #> [1] 3 #>  #>  p$unlock_step(\"add1\") p$set_params(list(x = 3)) p$get_params() #> $add1 #> $add1$x #> [1] 3 #>  #>  #> $add2 #> $add2$x #> [1] 3 #>  #>"},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":null,"dir":"Reference","previous_headings":"","what":"Pipeline alias functions — pipelineAliases","title":"Pipeline alias functions — pipelineAliases","text":"Alias functions, one member function Pipeline object.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline alias functions — pipelineAliases","text":"","code":"pipe_add(pip, ...)  pipe_append(pip, ...)  pipe_append_to_step_names(pip, ...)  pipe_clone(pip, ...)  pipe_collect_out(pip, ...)  pipe_discard_steps(pip, ...)  pipe_get_data(pip, ...)  pipe_get_depends(pip, ...)  pipe_get_depends_down(pip, ...)  pipe_get_depends_up(pip, ...)  pipe_get_graph(pip, ...)  pipe_get_out(pip, ...)  pipe_get_params(pip, ...)  pipe_get_params_at_step(pip, ...)  pipe_get_params_unique(pip, ...)  pipe_get_params_unique_json(pip, ...)  pipe_get_step(pip, ...)  pipe_get_step_names(pip, ...)  pipe_get_step_number(pip, ...)  pipe_has_step(pip, ...)  pipe_insert_after(pip, ...)  pipe_insert_before(pip, ...)  pipe_length(pip, ...)  pipe_lock_step(pip, ...)  pipe_new(...)  pipe_print(pip, ...)  pipe_pop_step(pip, ...)  pipe_pop_steps_after(pip, ...)  pipe_pop_steps_from(pip, ...)  pipe_remove_step(pip, ...)  pipe_rename_step(pip, ...)  pipe_replace_step(pip, ...)  pipe_reset(pip, ...)  pipe_run(pip, ...)  pipe_run_step(pip, ...)  pipe_set_data(pip, ...)  pipe_set_data_split(pip, ...)  pipe_set_keep_out(pip, ...)  pipe_set_params(pip, ...)  pipe_set_params_at_step(pip, ...)  pipe_split(pip, ...)  pipe_unlock_step(pip, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline alias functions — pipelineAliases","text":"pip pipeline object ... Arguments passed respective pipeline method","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Pipeline alias functions — pipelineAliases","text":"result respective pipeline method","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":null,"dir":"Reference","previous_headings":"","what":"Set pipeflow log layout — set_log_layout","title":"Set pipeflow log layout — set_log_layout","text":"Set pipeflow log layout","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set pipeflow log layout — set_log_layout","text":"","code":"set_log_layout(layout)"},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set pipeflow log layout — set_log_layout","text":"layout Layout name","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set pipeflow log layout — set_log_layout","text":"invisibly returns logger object","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set pipeflow log layout — set_log_layout","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$run() #> INFO  [2024-12-01 21:41:29.126] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:29.127] Step 1/2 data #> INFO  [2024-12-01 21:41:29.129] Step 2/2 add1 #> INFO  [2024-12-01 21:41:29.130] Finished execution of steps. #> INFO  [2024-12-01 21:41:29.131] Done.  lg <- set_log_layout(\"json\") print(lg) #> <Logger> [info] pipeflow #>  #> appenders: #>   [[1]]: <AppenderConsole> [all] -> console  p$run() #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Start run of 'pipe' pipeline:\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Step 1/2 data - skip 'done' step\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Step 2/2 add1 - skip 'done' step\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Finished execution of steps.\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:41:29+1:00\",\"message\":\"Done.\"}  set_log_layout(\"text\") p$run() #> INFO  [2024-12-01 21:41:29.169] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:41:29.170] Step 1/2 data - skip 'done' step #> INFO  [2024-12-01 21:41:29.171] Step 2/2 add1 - skip 'done' step #> INFO  [2024-12-01 21:41:29.171] Finished execution of steps. #> INFO  [2024-12-01 21:41:29.172] Done."}]
+[{"path":"https://github.com/rpahl/pipeflow/articles/collect-output.html","id":"flag-output-steps","dir":"Articles","previous_headings":"","what":"Flag output steps","title":"Collecting output","text":"Output steps flagged settting keepOut argument TRUE adding step pipeline. following example, want keep output steps data_summary, model_summary, model_plot. Looking pipeline, see steps data_summary, model-summary, model_plot flagged accordingly (see column keepOut). Graphically, steps flagged keepOut = TRUE displayed circle shape “normal” steps shown rectangle boxes. Now let’s run collect output flagged steps using collect_out method, returns list output flagged steps. expected, output list contains output flagged steps.","code":"library(pipeflow) library(ggplot2)  pip <- pipe_new(         \"my-pipeline\",         data = airquality     ) |>      pipe_add(         \"data_prep\",         function(data = ~data) {             replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)         }     ) |>      pipe_add(         \"data_summary\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             data[, c(xVar, yVar)]         },         keepOut = TRUE              # <- keep this     ) |>      pipe_add(         \"model_fit\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"model_summary\",         function(             fit = ~model_fit         ) {             summary(fit)         },         keepOut = TRUE              # <- keep this     ) |>      pipe_add(         \"model_plot\",         function(             model = ~model_fit,             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\",             title = \"Linear model fit\"         ) {             coeffs <- coefficients(model)             ggplot(data) +                 geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) +                 geom_abline(intercept = coeffs[1], slope = coeffs[2]) +                 labs(title = title)         },         keepOut = TRUE              # <- keep this     ) pip #             step             depends    out keepOut         group  state #           <char>              <list> <list>  <lgcl>        <char> <char> # 1:          data                     [NULL]   FALSE          data    New # 2:     data_prep                data [NULL]   FALSE     data_prep    New # 3:  data_summary           data_prep [NULL]    TRUE  data_summary    New # 4:     model_fit           data_prep [NULL]   FALSE     model_fit    New # 5: model_summary           model_fit [NULL]    TRUE model_summary    New # 6:    model_plot model_fit,data_prep [NULL]    TRUE    model_plot    New pip$run() # INFO  [2024-12-01 21:46:27.511] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:27.539] Step 1/6 data # INFO  [2024-12-01 21:46:27.548] Step 2/6 data_prep # INFO  [2024-12-01 21:46:27.572] Step 3/6 data_summary # INFO  [2024-12-01 21:46:27.573] Step 4/6 model_fit # INFO  [2024-12-01 21:46:27.577] Step 5/6 model_summary # INFO  [2024-12-01 21:46:27.580] Step 6/6 model_plot # INFO  [2024-12-01 21:46:27.591] Finished execution of steps. # INFO  [2024-12-01 21:46:27.591] Done.  out <- pip$collect_out()  names(out) # [1] \"data_summary\"  \"model_summary\" \"model_plot\" str(out, max.level = 1) # List of 3 #  $ data_summary :'data.frame':    153 obs. of  2 variables: #  $ model_summary:List of 12 #   ..- attr(*, \"class\")= chr \"summary.lm\" #  $ model_plot   :List of 11 #   ..- attr(*, \"class\")= chr [1:2] \"gg\" \"ggplot\""},{"path":"https://github.com/rpahl/pipeflow/articles/collect-output.html","id":"grouping-output-steps","dir":"Articles","previous_headings":"","what":"Grouping output steps","title":"Collecting output","text":"Often certain output steps related grouped together. can achieved conveniently setting group argument adding step pipeline. Let’s illustrate slightly modifying previous example. Looking pipeline, defined groups shown group column. see, default, group identical step name, , step represents trivial case one-sized group. , run pipeline collect output. can see, output related modelling grouped one sublist named Model.","code":"pip <- Pipeline$new(\"my-pipeline\", data = airquality) |>      pipe_add(         \"data_prep\",         function(data = ~data) {             replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)         }     ) |>      pipe_add(         \"used_data\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             data[, c(xVar, yVar)]         },         keepOut = TRUE,         group = \"Data\"                 # <- define 'Data' group here     ) |>      pipe_add(         \"model_fit\",         function(             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"model_summary\",         function(             fit = ~model_fit         ) {             summary(fit)         },         keepOut = TRUE,         group = \"Model\"                # <- define 'Model' group here     ) |>      pipe_add(         \"model_plot\",         function(             model = ~model_fit,             data = ~data_prep,             xVar = \"Temp.Celsius\",             yVar = \"Ozone\",             title = \"Linear model fit\"         ) {             coeffs <- coefficients(model)             ggplot(data) +                 geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) +                 geom_abline(intercept = coeffs[1], slope = coeffs[2]) +                 labs(title = title)         },         keepOut = TRUE,         group = \"Model\"                # <- define 'Model' group here     ) pip #             step             depends    out keepOut     group  state #           <char>              <list> <list>  <lgcl>    <char> <char> # 1:          data                     [NULL]   FALSE      data    New # 2:     data_prep                data [NULL]   FALSE data_prep    New # 3:     used_data           data_prep [NULL]    TRUE      Data    New # 4:     model_fit           data_prep [NULL]   FALSE model_fit    New # 5: model_summary           model_fit [NULL]    TRUE     Model    New # 6:    model_plot model_fit,data_prep [NULL]    TRUE     Model    New pip$run() # INFO  [2024-12-01 21:46:27.822] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:27.823] Step 1/6 data # INFO  [2024-12-01 21:46:27.826] Step 2/6 data_prep # INFO  [2024-12-01 21:46:27.829] Step 3/6 used_data # INFO  [2024-12-01 21:46:27.830] Step 4/6 model_fit # INFO  [2024-12-01 21:46:27.853] Step 5/6 model_summary # INFO  [2024-12-01 21:46:27.855] Step 6/6 model_plot # INFO  [2024-12-01 21:46:27.860] Finished execution of steps. # INFO  [2024-12-01 21:46:27.861] Done.  out <- pip$collect_out()  names(out) # [1] \"Data\"  \"Model\" str(out, max.level = 2) # List of 2 #  $ Data :'data.frame':    153 obs. of  2 variables: #   ..$ Temp.Celsius: num [1:153] 19.4 22.2 23.3 16.7 13.3 ... #   ..$ Ozone       : int [1:153] 41 36 12 18 NA 28 23 19 8 NA ... #  $ Model:List of 2 #   ..$ model_summary:List of 12 #   .. ..- attr(*, \"class\")= chr \"summary.lm\" #   ..$ model_plot   :List of 11 #   .. ..- attr(*, \"class\")= chr [1:2] \"gg\" \"ggplot\""},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"define-two-pipelines","dir":"Articles","previous_headings":"","what":"Define two pipelines","title":"Combining pipelines","text":"Let’s define one pipeline used data_preprocessing one modeling. Data preprocessing pipeline: Modeling pipeline: Graphically, two pipelines look follows:","code":"library(pipeflow) library(ggplot2)  pip1 <- pipe_new(         \"preprocessing\",         data = airquality     ) |>      pipe_add(         \"data_prep\",         function(data = ~data) {             replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)         }     ) |>      pipe_add(         \"standardize\",         function(             data = ~data_prep,             yVar = \"Ozone\"         ) {             data[, yVar] <- scale(data[, yVar])             data         }     ) pip1 #           step   depends    out keepOut       group  state #         <char>    <list> <list>  <lgcl>      <char> <char> # 1:        data           [NULL]   FALSE        data    New # 2:   data_prep      data [NULL]   FALSE   data_prep    New # 3: standardize data_prep [NULL]   FALSE standardize    New pip2 <- pipe_new(         \"modeling\",         data = airquality     ) |>      pipe_add(         \"fit\",         function(             data = ~data,             xVar = \"Temp\",             yVar = \"Ozone\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"plot\",         function(             model = ~fit,             data = ~data,             xVar = \"Temp\",             yVar = \"Ozone\",             title = \"Linear model fit\"         ) {             coeffs <- coefficients(model)             ggplot(data) +                 geom_point(aes(.data[[xVar]], .data[[yVar]])) +                 geom_abline(intercept = coeffs[1], slope = coeffs[2]) +                 labs(title = title)         }     ) pip2 #      step  depends    out keepOut  group  state #    <char>   <list> <list>  <lgcl> <char> <char> # 1:   data          [NULL]   FALSE   data    New # 2:    fit     data [NULL]   FALSE    fit    New # 3:   plot fit,data [NULL]   FALSE   plot    New"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"combine-pipelines","dir":"Articles","previous_headings":"","what":"Combine pipelines","title":"Combining pipelines","text":"Next combine two pipelines. can using append function. First , note data step second pipeline appended name second pipeline. particular, first step second pipeline renamed data data.modeling (line 4 step column) likewise dependencies second pipeline updated (see lines 5-6 depends column). , appending two pipelines, pipeflow ensures step names remain unique resulting combined pipeline therefore automatically renames duplicated step names necessary. Now, can also seen graphical representation pipeline, two pipelines yet connected. make actual use combined pipeline, therefore want use output first pipeline input second pipeline, , want use output standardize step data parameter input data.modeling step. One way achieve use replace function described earlier vignette modify pipeline, example:","code":"pip <- pip1$append(pip2)  pip #             step           depends    out keepOut       group  state #           <char>            <list> <list>  <lgcl>      <char> <char> # 1:          data                   [NULL]   FALSE        data    New # 2:     data_prep              data [NULL]   FALSE   data_prep    New # 3:   standardize         data_prep [NULL]   FALSE standardize    New # 4: data.modeling                   [NULL]   FALSE        data    New # 5:           fit     data.modeling [NULL]   FALSE         fit    New # 6:          plot fit,data.modeling [NULL]   FALSE        plot    New pip$replace_step(\"data.modeling\", function(data = ~standardize) data)  pip #             step           depends    out keepOut         group    state #           <char>            <list> <list>  <lgcl>        <char>   <char> # 1:          data                   [NULL]   FALSE          data      New # 2:     data_prep              data [NULL]   FALSE     data_prep      New # 3:   standardize         data_prep [NULL]   FALSE   standardize      New # 4: data.modeling       standardize [NULL]   FALSE data.modeling      New # 5:           fit     data.modeling [NULL]   FALSE           fit Outdated # 6:          plot fit,data.modeling [NULL]   FALSE          plot Outdated"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"relative-indexing","dir":"Articles","previous_headings":"Combine pipelines","what":"Relative indexing","title":"Combining pipelines","text":"Since name last step might always known1, pipeflow package also provides relative position indexing mechanism, allows rewrite command follows: Generally speaking, relative indexing mechanism allows refer steps positioned current step. index ~-1 can interpreted “go one step back”, ~-2 “go two steps back”, . Since scenario connecting two pipelines common avoid replacement steps manually, append function actually provides argument outAsIn allows appending “connecting” pipelines one go: inspect data.modeling step, see “hood” original step indeed replaced output last step first pipeline using relative indexing mechanism manually .","code":"pip$replace_step(\"data.modeling\", function(data = ~-1) data)  pip #             step           depends    out keepOut         group    state #           <char>            <list> <list>  <lgcl>        <char>   <char> # 1:          data                   [NULL]   FALSE          data      New # 2:     data_prep              data [NULL]   FALSE     data_prep      New # 3:   standardize         data_prep [NULL]   FALSE   standardize      New # 4: data.modeling       standardize [NULL]   FALSE data.modeling      New # 5:           fit     data.modeling [NULL]   FALSE           fit Outdated # 6:          plot fit,data.modeling [NULL]   FALSE          plot Outdated pip <- pip1$append(pip2, outAsIn = TRUE)  pip #             step           depends    out keepOut         group    state #           <char>            <list> <list>  <lgcl>        <char>   <char> # 1:          data                   [NULL]   FALSE          data      New # 2:     data_prep              data [NULL]   FALSE     data_prep      New # 3:   standardize         data_prep [NULL]   FALSE   standardize      New # 4: data.modeling       standardize [NULL]   FALSE data.modeling      New # 5:           fit     data.modeling [NULL]   FALSE           fit Outdated # 6:          plot fit,data.modeling [NULL]   FALSE          plot Outdated pip$get_step(\"data.modeling\")[[\"fun\"]] # [[1]] # function (data = ~-1)  # data # <bytecode: 0x0000015b1cf930e8> # <environment: 0x0000015b18bae928>"},{"path":"https://github.com/rpahl/pipeflow/articles/combine-pipelines.html","id":"run-combined-pipeline","dir":"Articles","previous_headings":"","what":"Run combined pipeline","title":"Combining pipelines","text":"Let’s now run combined pipeline inspect plot.  can see, plot shows linear model fit using standardized data. can now go ahead example change x-variable model rerun pipeline.  creating pipelines, usually lot steps calculating intermediate results steps contain final results interested . example, interested final plot output. real-world scenario, pipeline contain many steps interest us. see conveniently tag, collect possibly group output final steps, see next vignette Collecting output.","code":"pip$run() # INFO  [2024-12-01 21:46:32.182] Start run of 'preprocessing.modeling' pipeline: # INFO  [2024-12-01 21:46:32.255] Step 1/6 data # INFO  [2024-12-01 21:46:32.263] Step 2/6 data_prep # INFO  [2024-12-01 21:46:32.280] Step 3/6 standardize # INFO  [2024-12-01 21:46:32.283] Step 4/6 data.modeling # INFO  [2024-12-01 21:46:32.285] Step 5/6 fit # INFO  [2024-12-01 21:46:32.289] Step 6/6 plot # INFO  [2024-12-01 21:46:32.302] Finished execution of steps. # INFO  [2024-12-01 21:46:32.303] Done. pip$get_out(\"plot\") # Warning: Removed 37 rows containing missing values or values outside the scale range # (`geom_point()`). pip$set_params(list(xVar = \"Temp.Celsius\")) pip$run() # INFO  [2024-12-01 21:46:32.852] Start run of 'preprocessing.modeling' pipeline: # INFO  [2024-12-01 21:46:32.853] Step 1/6 data - skip 'done' step # INFO  [2024-12-01 21:46:32.854] Step 2/6 data_prep - skip 'done' step # INFO  [2024-12-01 21:46:32.855] Step 3/6 standardize - skip 'done' step # INFO  [2024-12-01 21:46:32.856] Step 4/6 data.modeling - skip 'done' step # INFO  [2024-12-01 21:46:32.857] Step 5/6 fit # INFO  [2024-12-01 21:46:32.861] Step 6/6 plot # INFO  [2024-12-01 21:46:32.878] Finished execution of steps. # INFO  [2024-12-01 21:46:32.879] Done. pip$get_out(\"plot\") # Warning: Removed 37 rows containing missing values or values outside the scale range # (`geom_point()`)."},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"a-simple-example-to-get-started","dir":"Articles","previous_headings":"","what":"A simple example to get started","title":"Get started with pipeflow","text":"example, ’ll use base R’s airquality dataset. goal create analysis pipeline performs following steps: add new data column Temp.Celsius containing temperature degrees Celsius fit linear model data plot data model fit. following, ’ll show define run pipeline, inspect output specifics steps, finally re-run pipeline different parameter settings, one selling points using pipeline.","code":"head(airquality) #   Ozone Solar.R Wind Temp Month Day # 1    41     190  7.4   67     5   1 # 2    36     118  8.0   72     5   2 # 3    12     149 12.6   74     5   3 # 4    18     313 11.5   62     5   4 # 5    NA      NA 14.3   56     5   5 # 6    28      NA 14.9   66     5   6"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"define-the-pipeline","dir":"Articles","previous_headings":"A simple example to get started","what":"Define the pipeline","title":"Get started with pipeflow","text":"easier understanding, go step step. First, create new pipeline name “-pipeline” airquality dataset input data. Let’s view pipeline ’s table form initialization. , step represented one row table step column lists name step state column shows current state step. now, don’t worry columns - explained later. can see, data always first step pipeline initially state New, means pipeline run yet. Next, add step called data_prep, consists function takes output data step first argument, adds new column returns modified data output. refer output earlier pipeline step, always use tilde (~) operator. , let’s inspect pipeline. second step called data_prep added depends data step can seen column depends, list dependencies step. Next, add step called model_fit, fits linear model data. function takes output data_prep defines parameter xVar, used specify variable used predictor linear model. Lastly, add step called model_plot, plots data linear model fit. function uses output model_fit data_prep step. also defines xVar parameter parameter title, used title plot. completes pipeline, now looks follows: last line, see model_plot step depends model_fit data_prep step. visualize pipeline, can use get_graph() method, returns list arguments can passed visNetwork function visNetwork package.","code":"library(pipeflow) library(ggplot2)  pip <- Pipeline$new(\"my-pipeline\", data = airquality) pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data         [NULL]   FALSE   data    New pip$add(     \"data_prep\",     function(data = ~data) {         replace(data, \"Temp.Celsius\", (data[, \"Temp\"] - 32) * 5/9)     } ) pip #         step depends    out keepOut     group  state #       <char>  <list> <list>  <lgcl>    <char> <char> # 1:      data         [NULL]   FALSE      data    New # 2: data_prep    data [NULL]   FALSE data_prep    New pip$add(     \"model_fit\",     function(         data = ~data_prep,         xVar = \"Temp.Celsius\"     ) {         lm(paste(\"Ozone ~\", xVar), data = data)     } ) pip$add(     \"model_plot\",     function(         model = ~model_fit,         data = ~data_prep,         xVar = \"Temp.Celsius\",         title = \"Linear model fit\"     ) {         coeffs <- coefficients(model)         ggplot(data) +             geom_point(aes(.data[[xVar]], .data[[\"Ozone\"]])) +             geom_abline(intercept = coeffs[1], slope = coeffs[2]) +             labs(title = title)     } ) pip #          step             depends    out keepOut      group  state #        <char>              <list> <list>  <lgcl>     <char> <char> # 1:       data                     [NULL]   FALSE       data    New # 2:  data_prep                data [NULL]   FALSE  data_prep    New # 3:  model_fit           data_prep [NULL]   FALSE  model_fit    New # 4: model_plot model_fit,data_prep [NULL]   FALSE model_plot    New library(visNetwork) do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\")"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"run-the-pipeline-and-inspect-the-output","dir":"Articles","previous_headings":"A simple example to get started","what":"Run the pipeline and inspect the output","title":"Get started with pipeflow","text":"run pipeline, simply call run() method pipeline, produce following output: Let’s inspect pipeline . can see state steps changed New Done, graphically represented color change blue green. addition, output added column. can inspect output specific step calling get_out() method pipeline.","code":"pip$run() # INFO  [2024-12-01 21:46:37.075] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:37.101] Step 1/4 data # INFO  [2024-12-01 21:46:37.112] Step 2/4 data_prep # INFO  [2024-12-01 21:46:37.134] Step 3/4 model_fit # INFO  [2024-12-01 21:46:37.138] Step 4/4 model_plot # INFO  [2024-12-01 21:46:37.149] Finished execution of steps. # INFO  [2024-12-01 21:46:37.150] Done. pip #          step             depends                 out keepOut      group  state #        <char>              <list>              <list>  <lgcl>     <char> <char> # 1:       data                     <data.frame[153x6]>   FALSE       data   Done # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep   Done # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit   Done # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot   Done pip$get_out(\"model_fit\") #  # Call: # lm(formula = paste(\"Ozone ~\", xVar), data = data) #  # Coefficients: #  (Intercept)  Temp.Celsius   #      -69.277         4.372 pip$get_out(\"model_plot\")"},{"path":"https://github.com/rpahl/pipeflow/articles/get-started.html","id":"change-parameters-of-the-pipeline","dir":"Articles","previous_headings":"A simple example to get started","what":"Change parameters of the pipeline","title":"Get started with pipeflow","text":"parameter pipeline bound specific step (.e. defined tilde (~) operator) can changed calling set_params() method pipeline. Let’s first list unbound parameters: Now let’s assume want change xVar entire pipeline \"Solar.R\". can simply follows: Changing parameters pipeline least effect steps depend changed parameter. can see model_fit model_plot steps now state Outdated (graphically indicated orange color), means need rerun. can calling run() method pipeline . outdated steps re-run expected output updated accordingly.  just change title plot, obviously model_plot step needs rerun.  instead change input data, since steps depend , expect steps rerun.  Next, let’s see modify pipeline.","code":"pip$get_params() # $model_fit # $model_fit$xVar # [1] \"Temp.Celsius\" #  #  # $model_plot # $model_plot$xVar # [1] \"Temp.Celsius\" #  # $model_plot$title # [1] \"Linear model fit\" pip$set_params(list(xVar = \"Solar.R\")) pip$get_params() # $model_fit # $model_fit$xVar # [1] \"Solar.R\" #  #  # $model_plot # $model_plot$xVar # [1] \"Solar.R\" #  # $model_plot$title # [1] \"Linear model fit\" pip #          step             depends                 out keepOut      group    state #        <char>              <list>              <list>  <lgcl>     <char>   <char> # 1:       data                     <data.frame[153x6]>   FALSE       data     Done # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep     Done # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit Outdated # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot Outdated pip$run() # INFO  [2024-12-01 21:46:38.057] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:38.058] Step 1/4 data - skip 'done' step # INFO  [2024-12-01 21:46:38.059] Step 2/4 data_prep - skip 'done' step # INFO  [2024-12-01 21:46:38.060] Step 3/4 model_fit # INFO  [2024-12-01 21:46:38.064] Step 4/4 model_plot # INFO  [2024-12-01 21:46:38.074] Finished execution of steps. # INFO  [2024-12-01 21:46:38.074] Done. pip$get_out(\"model_plot\") pip$set_params(list(title = \"Some new title\")) pip #          step             depends                 out keepOut      group    state #        <char>              <list>              <list>  <lgcl>     <char>   <char> # 1:       data                     <data.frame[153x6]>   FALSE       data     Done # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep     Done # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit     Done # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot Outdated pip$run()$get_out(\"model_plot\") # INFO  [2024-12-01 21:46:38.428] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:38.429] Step 1/4 data - skip 'done' step # INFO  [2024-12-01 21:46:38.430] Step 2/4 data_prep - skip 'done' step # INFO  [2024-12-01 21:46:38.431] Step 3/4 model_fit - skip 'done' step # INFO  [2024-12-01 21:46:38.432] Step 4/4 model_plot # INFO  [2024-12-01 21:46:38.440] Finished execution of steps. # INFO  [2024-12-01 21:46:38.440] Done. pip$set_data(airquality[1:10, ]) pip #          step             depends                 out keepOut      group    state #        <char>              <list>              <list>  <lgcl>     <char>   <char> # 1:       data                                  [NULL]   FALSE       data      New # 2:  data_prep                data <data.frame[153x7]>   FALSE  data_prep Outdated # 3:  model_fit           data_prep            <lm[13]>   FALSE  model_fit Outdated # 4: model_plot model_fit,data_prep            <gg[11]>   FALSE model_plot Outdated pip$run()$get_out(\"model_plot\") # INFO  [2024-12-01 21:46:38.710] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:38.711] Step 1/4 data # INFO  [2024-12-01 21:46:38.714] Step 2/4 data_prep # INFO  [2024-12-01 21:46:38.718] Step 3/4 model_fit # INFO  [2024-12-01 21:46:38.720] Step 4/4 model_plot # INFO  [2024-12-01 21:46:38.726] Finished execution of steps. # INFO  [2024-12-01 21:46:38.726] Done."},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"existing-pipeline","dir":"Articles","previous_headings":"","what":"Existing pipeline","title":"Modifying existing pipelines","text":"Let’s start left Get started pipeflow vignette, , following pipeline following set data","code":"pip #          step             depends                out keepOut      group  state #        <char>              <list>             <list>  <lgcl>     <char> <char> # 1:       data                     <data.frame[10x6]>   FALSE       data   Done # 2:  data_prep                data <data.frame[10x7]>   FALSE  data_prep   Done # 3:  model_fit           data_prep           <lm[13]>   FALSE  model_fit   Done # 4: model_plot model_fit,data_prep           <gg[11]>   FALSE model_plot   Done pip$get_data() |> head(3) #   Ozone Solar.R Wind Temp Month Day # 1    41     190  7.4   67     5   1 # 2    36     118  8.0   72     5   2 # 3    12     149 12.6   74     5   3"},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"insert-new-step","dir":"Articles","previous_headings":"","what":"Insert new step","title":"Modifying existing pipelines","text":"Let’s say want insert new step data_prep step standardizes y-variable. can see, standardize step now part pipeline, far used step.","code":"pip$insert_after(     afterStep = \"data_prep\",     step = \"standardize\",     function(         data = ~`data_prep`,         yVar = \"Ozone\"     ) {         data[, yVar] <- scale(data[, yVar])         data     } ) pip #           step             depends                out keepOut       group  state #         <char>              <list>             <list>  <lgcl>      <char> <char> # 1:        data                     <data.frame[10x6]>   FALSE        data   Done # 2:   data_prep                data <data.frame[10x7]>   FALSE   data_prep   Done # 3: standardize           data_prep             [NULL]   FALSE standardize    New # 4:   model_fit           data_prep           <lm[13]>   FALSE   model_fit   Done # 5:  model_plot model_fit,data_prep           <gg[11]>   FALSE  model_plot   Done library(visNetwork) do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\")"},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"replace-existing-steps","dir":"Articles","previous_headings":"","what":"Replace existing steps","title":"Modifying existing pipelines","text":"Let’s revisit function definition model_fit step use standardized data, need change data dependency refers standardize step. Also instead fixed y-variable model, want pass paramter. model_plot step needs updated similar way. updated pipeline now looks follows. see model_fit model_plot steps now use standardized data. Let’s re-run pipeline inspect output.","code":"pip$get_step(\"model_fit\")[[\"fun\"]] # [[1]] # function (data = ~data_prep, xVar = \"Temp.Celsius\")  # { #     lm(paste(\"Ozone ~\", xVar), data = data) # } pip$replace_step(     \"model_fit\",     function(         data = ~standardize,        # <- changed data reference         xVar = \"Temp.Celsius\",         yVar = \"Ozone\"              # <- new y-variable     ) {         lm(paste(yVar, \"~\", xVar), data = data)     } ) pip$replace_step(     \"model_plot\",     function(         model = ~model_fit,         data = ~standardize,         # <- changed data reference         xVar = \"Temp.Celsius\",         yVar = \"Ozone\",              # <- new y-variable         title = \"Linear model fit\"     ) {         coeffs <- coefficients(model)         ggplot(data) +             geom_point(aes(.data[[xVar]], .data[[yVar]])) +             geom_abline(intercept = coeffs[1], slope = coeffs[2]) +             labs(title = title)     } ) pip #           step               depends                out keepOut       group  state #         <char>                <list>             <list>  <lgcl>      <char> <char> # 1:        data                       <data.frame[10x6]>   FALSE        data   Done # 2:   data_prep                  data <data.frame[10x7]>   FALSE   data_prep   Done # 3: standardize             data_prep             [NULL]   FALSE standardize    New # 4:   model_fit           standardize             [NULL]   FALSE   model_fit    New # 5:  model_plot model_fit,standardize             [NULL]   FALSE  model_plot    New pip$set_params(list(xVar = \"Solar.R\", yVar = \"Wind\")) pip$run() # INFO  [2024-12-01 21:46:43.011] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:43.012] Step 1/5 data - skip 'done' step # INFO  [2024-12-01 21:46:43.013] Step 2/5 data_prep - skip 'done' step # INFO  [2024-12-01 21:46:43.014] Step 3/5 standardize # INFO  [2024-12-01 21:46:43.016] Step 4/5 model_fit # INFO  [2024-12-01 21:46:43.019] Step 5/5 model_plot # INFO  [2024-12-01 21:46:43.024] Finished execution of steps. # INFO  [2024-12-01 21:46:43.025] Done. pip$get_out(\"model_fit\") |> coefficients() #  (Intercept)      Solar.R  #  0.979672739 -0.006625601 pip$get_out(\"model_plot\") # Warning: Removed 2 rows containing missing values or values outside the scale range # (`geom_point()`)."},{"path":"https://github.com/rpahl/pipeflow/articles/modify-pipeline.html","id":"removing-steps","dir":"Articles","previous_headings":"","what":"Removing steps","title":"Modifying existing pipelines","text":"Let’s see pipeline . trying remove step, pipeflow default checks step used step. , raise error. enforce removing step together downstream dependencies, can use recursive argument. Naturally, last step never downstream dependencies, can removed without issues. another way just remove last step. Replacing steps pipeline shown vignette allow re-use existing pipelines adapt programmatically new requirements. Another way re-using pipelines combine , shown Combining pipelines vignette.","code":"pip #           step               depends                out keepOut       group  state #         <char>                <list>             <list>  <lgcl>      <char> <char> # 1:        data                       <data.frame[10x6]>   FALSE        data   Done # 2:   data_prep                  data <data.frame[10x7]>   FALSE   data_prep   Done # 3: standardize             data_prep <data.frame[10x7]>   FALSE standardize   Done # 4:   model_fit           standardize           <lm[13]>   FALSE   model_fit   Done # 5:  model_plot model_fit,standardize           <gg[11]>   FALSE  model_plot   Done try(pip$remove_step(\"standardize\")) # Error in pip$remove_step(\"standardize\") :  #   cannot remove step 'standardize' because the following steps depend on it: 'model_fit', 'model_plot' pip$remove_step(\"standardize\", recursive = TRUE) # Removing step 'standardize' and its downstream dependencies: 'model_fit', 'model_plot' pip #         step depends                out keepOut     group  state #       <char>  <list>             <list>  <lgcl>    <char> <char> # 1:      data         <data.frame[10x6]>   FALSE      data   Done # 2: data_prep    data <data.frame[10x7]>   FALSE data_prep   Done pip$pop_step() # [1] \"data_prep\" pip #      step depends                out keepOut  group  state #    <char>  <list>             <list>  <lgcl> <char> <char> # 1:   data         <data.frame[10x6]>   FALSE   data   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"changing-pipeline-parameters-at-runtime","dir":"Articles","previous_headings":"","what":"Changing pipeline parameters at runtime","title":"Advanced: How pipelines can modify themselves at runtime","text":"Let’s first define pipeline, fits linear model, checks ’s residuals normality using Shapiro-Wilk test, plots residuals. pipeline looks like can run like  Now let’s imagine, want change color points plot depending Shapiro-Wilk test result. obvious way change plot step passing test result plot step function change color . However, interested another way keep plot function unchanged. example, run pipeline second time follows:  mentioned another vignette, solution ideal, requires run additional code outside pipeline framework. solve issue, therefore basically set parameter within pipeline execution. , make pipeline aware , can done passing pipeline object parameter. Let’s update residual_shapiro_p_value step example. Now just make sure set .self parameter.  simple “trick” opens wide range possibilities pipeline modifications runtime. show next section, limited changing parameters can also used modify pipeline structure .","code":"library(pipeflow) library(ggplot2)  pip <- pipe_new(\"my-pipeline\") |>      pipe_add(         \"fit\",         function(data = ~data, xVar = \"x\", yVar = \"y\")         {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"residual_shapiro_p_value\",         function(fit = ~fit)         {             residuals <- residuals(fit)             p <- shapiro.test(residuals)$p.value             p         },         keepOut = TRUE     ) |>      pipe_add(         \"plot\",         function(fit = ~fit, pointColor = \"black\")         {             data <- data.frame(                 fitted = predict(fit),                 residuals = residuals(fit)             )              ggplot(data, aes(x = fitted, y = residuals)) +                 geom_point(shape = 21, color = pointColor) +                 geom_hline(yintercept = 0, linetype = \"dashed\") +                 theme_minimal()         },         keepOut = TRUE     ) pip #                        step depends    out keepOut                    group  state #                      <char>  <list> <list>  <lgcl>                   <char> <char> # 1:                     data         [NULL]   FALSE                     data    New # 2:                      fit    data [NULL]   FALSE                      fit    New # 3: residual_shapiro_p_value     fit [NULL]    TRUE residual_shapiro_p_value    New # 4:                     plot     fit [NULL]    TRUE                     plot    New pip$set_data(airquality) pip$set_params(list(xVar = \"Ozone\", yVar = \"Temp\")) pip$run()$collect_out() # INFO  [2024-12-01 21:46:46.851] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:46.880] Step 1/4 data # INFO  [2024-12-01 21:46:46.895] Step 2/4 fit # INFO  [2024-12-01 21:46:46.912] Step 3/4 residual_shapiro_p_value # INFO  [2024-12-01 21:46:46.914] Step 4/4 plot # INFO  [2024-12-01 21:46:46.930] Finished execution of steps. # INFO  [2024-12-01 21:46:46.931] Done. # $residual_shapiro_p_value # [1] 0.00022598 #  # $plot if (pip$get_out(\"residual_shapiro_p_value\") < 0.05) {     pip$set_params(list(pointColor = \"red\"))     pip$run()$collect_out() } # INFO  [2024-12-01 21:46:47.325] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:47.326] Step 1/4 data - skip 'done' step # INFO  [2024-12-01 21:46:47.327] Step 2/4 fit - skip 'done' step # INFO  [2024-12-01 21:46:47.328] Step 3/4 residual_shapiro_p_value - skip 'done' step # INFO  [2024-12-01 21:46:47.329] Step 4/4 plot # INFO  [2024-12-01 21:46:47.342] Finished execution of steps. # INFO  [2024-12-01 21:46:47.342] Done. # $residual_shapiro_p_value # [1] 0.00022598 #  # $plot pip$replace_step(     \"residual_shapiro_p_value\",     function(         fit = ~fit,         .self = NULL     ) {         residuals <- residuals(fit)         p <- shapiro.test(residuals)$p.value          if (!is.null(.self) && p < 0.05) {             .self$set_params(list(pointColor = \"red\"))         }          p     },     keepOut = TRUE ) pip$set_data(airquality) pip$set_params(list(xVar = \"Ozone\", yVar = \"Temp\", .self = pip)) pip$run()$collect_out() # INFO  [2024-12-01 21:46:47.638] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:47.640] Step 1/4 data # INFO  [2024-12-01 21:46:47.642] Step 2/4 fit # INFO  [2024-12-01 21:46:47.646] Step 3/4 residual_shapiro_p_value # INFO  [2024-12-01 21:46:47.650] Step 4/4 plot # INFO  [2024-12-01 21:46:47.657] Finished execution of steps. # INFO  [2024-12-01 21:46:47.658] Done. # $residual_shapiro_p_value # [1] 0.00022598 #  # $plot"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"changing-the-pipeline-structure-at-runtime","dir":"Articles","previous_headings":"","what":"Changing the pipeline structure at runtime","title":"Advanced: How pipelines can modify themselves at runtime","text":"Subsequently, pipeline steps comprised basic functions order keep examples simple. focus pipeline structure can modified runtime. pipeline just adds 1, 2, 3 input data, respectively. column table shows output step. Now let’s modify last step pipeline input greater 10, pipeline replace last step new step now instead f2 references f1 subtracts 3 input.","code":"pip <- pipe_new(\"my-pipeline\") |>      pipe_add(         \"f1\",         function(x = ~data) {             x + 1         }     ) |>      pipe_add(         \"f2\",         function(x = ~f1) {             x + 2         }     ) |>      pipe_add(         \"f3\",         function(x = ~f2) {             x + 3         }     ) pip$set_data(1)$run() # INFO  [2024-12-01 21:46:47.901] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:47.902] Step 1/4 data # INFO  [2024-12-01 21:46:47.904] Step 2/4 f1 # INFO  [2024-12-01 21:46:47.907] Step 3/4 f2 # INFO  [2024-12-01 21:46:47.908] Step 4/4 f3 # INFO  [2024-12-01 21:46:47.910] Finished execution of steps. # INFO  [2024-12-01 21:46:47.910] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data              1   FALSE   data   Done # 2:     f1    data      2   FALSE     f1   Done # 3:     f2      f1      4   FALSE     f2   Done # 4:     f3      f2      7   FALSE     f3   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"modify-a-step","dir":"Articles","previous_headings":"Changing the pipeline structure at runtime","what":"Modify a step","title":"Advanced: How pipelines can modify themselves at runtime","text":"run pipeline , nothing change. Now let’s try input 10. see output pipeline dependencies last step changed.","code":"pip <- pipe_new(\"my-pipeline\") |>      pipe_add(         \"f1\",         function(x = ~data) {             x + 1         }     ) |>      pipe_add(         \"f2\",         function(x = ~f1, .self = NULL)         {             if (x > 10 && !is.null(.self))             {                 .self$replace_step(                     \"f3\",                     function(x = ~f1) {                         x - 3                     }                 )             }             x + 2         }     ) |>      pipe_add(         \"f3\",         function(x = ~f2) {             x + 3         }     ) pip$set_params(list(.self = pip)) pip$set_data(1)$run() # INFO  [2024-12-01 21:46:48.028] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:48.029] Step 1/4 data # INFO  [2024-12-01 21:46:48.031] Step 2/4 f1 # INFO  [2024-12-01 21:46:48.034] Step 3/4 f2 # INFO  [2024-12-01 21:46:48.035] Step 4/4 f3 # INFO  [2024-12-01 21:46:48.037] Finished execution of steps. # INFO  [2024-12-01 21:46:48.037] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data              1   FALSE   data   Done # 2:     f1    data      2   FALSE     f1   Done # 3:     f2      f1      4   FALSE     f2   Done # 4:     f3      f2      7   FALSE     f3   Done pip$set_data(10)$run() # INFO  [2024-12-01 21:46:48.096] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:48.097] Step 1/4 data # INFO  [2024-12-01 21:46:48.099] Step 2/4 f1 # INFO  [2024-12-01 21:46:48.102] Step 3/4 f2 # INFO  [2024-12-01 21:46:48.108] Step 4/4 f3 # INFO  [2024-12-01 21:46:48.109] Finished execution of steps. # INFO  [2024-12-01 21:46:48.110] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data             10   FALSE   data   Done # 2:     f1    data     11   FALSE     f1   Done # 3:     f2      f1     13   FALSE     f2   Done # 4:     f3      f1      8   FALSE     f3   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/self-modify-pipeline.html","id":"insert-and-remove-steps","dir":"Articles","previous_headings":"Changing the pipeline structure at runtime","what":"Insert and remove steps","title":"Advanced: How pipelines can modify themselves at runtime","text":"last example, instead just replacing, go bit insert remove steps. pipeline definition follows: Basically, input greater 10, insert two new steps f1, remove f2, replace f3 new step adds 30 input. Also note return pipeline object case. important, pipeline’s run function argument recursive, default set TRUE means step returns pipeline, run current pipeline aborted returned pipeline re-run. Let’s see pipeline structure running . now let’s run input 10. log output shows abort re-run pipeline. Let’s see final structure step outputs. final structure expected new steps inserted old step removed. mentioned , just simple example show possibilities. leave user come sensible complex use cases.","code":"pip <- pipe_new(         \"my-pipeline\"     ) |>      pipe_add(         \"f1\",         function(x = ~data) {             x + 1         }     ) |>      pipe_add(         \"f2\",         function(x = ~f1, .self = NULL)         {             if (x > 10 && !is.null(.self)) {                 .self$insert_after(                     afterStep = \"f1\",                     step = \"f2a\",                     function(x = ~f1) {                         x + 21                     }                 )                 .self$insert_after(                     afterStep = \"f2a\",                     step = \"f2b\",                     function(x = ~f2a) {                         x + 22                     }                 )                 .self$replace_step(                     \"f3\",                     function(x = ~f2b) {                         x + 30                     }                 )                 .self$remove_step(\"f2\")                 return(.self)             }             x + 2         }     ) |>      pipe_add(         \"f3\",         function(x = ~f2) {             x + 3         }     ) pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data         [NULL]   FALSE   data    New # 2:     f1    data [NULL]   FALSE     f1    New # 3:     f2      f1 [NULL]   FALSE     f2    New # 4:     f3      f2 [NULL]   FALSE     f3    New pip$set_params(list(.self = pip)) pip$set_data(10)$run() # INFO  [2024-12-01 21:46:48.282] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:48.283] Step 1/4 data # INFO  [2024-12-01 21:46:48.285] Step 2/4 f1 # INFO  [2024-12-01 21:46:48.287] Step 3/4 f2 # INFO  [2024-12-01 21:46:48.300] Abort pipeline execution and restart on new. # INFO  [2024-12-01 21:46:48.301] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:48.302] Step 1/5 data - skip 'done' step # INFO  [2024-12-01 21:46:48.302] Step 2/5 f1 - skip 'done' step # INFO  [2024-12-01 21:46:48.303] Step 3/5 f2a # INFO  [2024-12-01 21:46:48.308] Step 4/5 f2b # INFO  [2024-12-01 21:46:48.310] Step 5/5 f3 # INFO  [2024-12-01 21:46:48.312] Finished execution of steps. # INFO  [2024-12-01 21:46:48.312] Done. pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data             10   FALSE   data   Done # 2:     f1    data     11   FALSE     f1   Done # 3:    f2a      f1     32   FALSE    f2a   Done # 4:    f2b     f2a     54   FALSE    f2b   Done # 5:     f3     f2b     84   FALSE     f3   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"motivation","dir":"Articles","previous_headings":"","what":"Motivation","title":"Advanced: How to use pipeflow with split data sets","text":"common scenario split data set subsets apply analysis part. context pipelines, means like apply pipeline multiple times data subset. additon, may want combine parts individual output. see, pipeflow provides built-function handle scenario.","code":""},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"define-pipeline","dir":"Articles","previous_headings":"","what":"Define pipeline","title":"Advanced: How to use pipeflow with split data sets","text":"Let’s first define pipeline, , keep matters simple, just fits linear model outputs model coefficients. pipeline looks like : graphically: use iris data set working example. First, apply pipeline whole data set.","code":"library(pipeflow)  pip <- pipe_new(         \"my-pipeline\"     ) |>      pipe_add(         \"fit\",         function(             data = ~data,             xVar = \"x\",             yVar = \"y\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"coefs\",         function(             fit = ~fit         ) {             coefficients(fit)         },         keepOut = TRUE     ) pip #      step depends    out keepOut  group  state #    <char>  <list> <list>  <lgcl> <char> <char> # 1:   data         [NULL]   FALSE   data    New # 2:    fit    data [NULL]   FALSE    fit    New # 3:  coefs     fit [NULL]    TRUE  coefs    New library(visNetwork) do.call(visNetwork, args = c(pip$get_graph(), list(height = 100))) |>     visHierarchicalLayout(direction = \"LR\") head(iris) #   Sepal.Length Sepal.Width Petal.Length Petal.Width Species # 1          5.1         3.5          1.4         0.2  setosa # 2          4.9         3.0          1.4         0.2  setosa # 3          4.7         3.2          1.3         0.2  setosa # 4          4.6         3.1          1.5         0.2  setosa # 5          5.0         3.6          1.4         0.2  setosa # 6          5.4         3.9          1.7         0.4  setosa pip$set_data(iris) pip$set_params(list(xVar = \"Sepal.Length\", yVar = \"Sepal.Width\")) pip$run() # INFO  [2024-12-01 21:46:51.463] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:51.487] Step 1/3 data # INFO  [2024-12-01 21:46:51.498] Step 2/3 fit # INFO  [2024-12-01 21:46:51.517] Step 3/3 coefs # INFO  [2024-12-01 21:46:51.519] Finished execution of steps. # INFO  [2024-12-01 21:46:51.519] Done. pip$collect_out() # $coefs #  (Intercept) Sepal.Length  #    3.4189468   -0.0618848"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"split-data-set","dir":"Articles","previous_headings":"","what":"Split data set","title":"Advanced: How to use pipeflow with split data sets","text":"Next, want apply pipeline species separately. One way use R’s split function. can split Species column run pipeline subset. example: Unfortunately, approach create additional code run outside pipeline framework. Let’s now see handle scenario within pipeline framework. reminder, pipeline looks like : deal split data sets, use built-function set_data_split. function actually transforms pipeline: can see, pipeline now replicated data subset. Note set_data_split accepts list data frames, just output split. Now let’s run pipeline. can see, output now obtained without need write additional code outside pipeline framework. Finally, side note another built-function named split, can used split pipeline independent parts. works pipeline, working example, naturally split pipeline parts defined data split. function especially useful want separate parts pipeline code order run parallel.","code":"run_pipeline <- function(data) {     pip$set_data(data)     pip$run()     pip$collect_out() }  results <- lapply(split(iris, iris$Species), FUN = run_pipeline) # INFO  [2024-12-01 21:46:51.654] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:51.655] Step 1/3 data # INFO  [2024-12-01 21:46:51.657] Step 2/3 fit # INFO  [2024-12-01 21:46:51.661] Step 3/3 coefs # INFO  [2024-12-01 21:46:51.665] Finished execution of steps. # INFO  [2024-12-01 21:46:51.666] Done. # INFO  [2024-12-01 21:46:51.683] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:51.684] Step 1/3 data # INFO  [2024-12-01 21:46:51.686] Step 2/3 fit # INFO  [2024-12-01 21:46:51.688] Step 3/3 coefs # INFO  [2024-12-01 21:46:51.690] Finished execution of steps. # INFO  [2024-12-01 21:46:51.690] Done. # INFO  [2024-12-01 21:46:51.692] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:51.693] Step 1/3 data # INFO  [2024-12-01 21:46:51.695] Step 2/3 fit # INFO  [2024-12-01 21:46:51.698] Step 3/3 coefs # INFO  [2024-12-01 21:46:51.699] Finished execution of steps. # INFO  [2024-12-01 21:46:51.699] Done. results # $setosa # $setosa$coefs #  (Intercept) Sepal.Length  #   -0.5694327    0.7985283  #  #  # $versicolor # $versicolor$coefs #  (Intercept) Sepal.Length  #    0.8721460    0.3197193  #  #  # $virginica # $virginica$coefs #  (Intercept) Sepal.Length  #    1.4463054    0.2318905 pip #      step depends                 out keepOut  group  state #    <char>  <list>              <list>  <lgcl> <char> <char> # 1:   data          <data.frame[50x5]>   FALSE   data   Done # 2:    fit    data            <lm[12]>   FALSE    fit   Done # 3:  coefs     fit 1.4463054,0.2318905    TRUE  coefs   Done splitData <- split(iris, iris$Species) pip$set_data_split(splitData) pip #                step         depends                 out keepOut      group    state #              <char>          <list>              <list>  <lgcl>     <char>   <char> # 1:      data.setosa                              [NULL]   FALSE     setosa      New # 2:       fit.setosa     data.setosa            <lm[12]>   FALSE     setosa Outdated # 3:     coefs.setosa      fit.setosa 1.4463054,0.2318905    TRUE     setosa Outdated # 4:  data.versicolor                              [NULL]   FALSE versicolor      New # 5:   fit.versicolor data.versicolor            <lm[12]>   FALSE versicolor Outdated # 6: coefs.versicolor  fit.versicolor 1.4463054,0.2318905    TRUE versicolor Outdated # 7:   data.virginica                              [NULL]   FALSE  virginica      New # 8:    fit.virginica  data.virginica            <lm[12]>   FALSE  virginica Outdated # 9:  coefs.virginica   fit.virginica 1.4463054,0.2318905    TRUE  virginica Outdated do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\") pip$run() # INFO  [2024-12-01 21:46:52.167] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:52.168] Step 1/9 data.setosa # INFO  [2024-12-01 21:46:52.170] Step 2/9 fit.setosa # INFO  [2024-12-01 21:46:52.172] Step 3/9 coefs.setosa # INFO  [2024-12-01 21:46:52.174] Step 4/9 data.versicolor # INFO  [2024-12-01 21:46:52.176] Step 5/9 fit.versicolor # INFO  [2024-12-01 21:46:52.179] Step 6/9 coefs.versicolor # INFO  [2024-12-01 21:46:52.180] Step 7/9 data.virginica # INFO  [2024-12-01 21:46:52.182] Step 8/9 fit.virginica # INFO  [2024-12-01 21:46:52.185] Step 9/9 coefs.virginica # INFO  [2024-12-01 21:46:52.186] Finished execution of steps. # INFO  [2024-12-01 21:46:52.187] Done. pip$collect_out() # $setosa #  (Intercept) Sepal.Length  #   -0.5694327    0.7985283  #  # $versicolor #  (Intercept) Sepal.Length  #    0.8721460    0.3197193  #  # $virginica #  (Intercept) Sepal.Length  #    1.4463054    0.2318905 pip$split() # [[1]] #            step     depends                   out keepOut  group  state #          <char>      <list>                <list>  <lgcl> <char> <char> # 1:  data.setosa                <data.frame[50x5]>   FALSE setosa   Done # 2:   fit.setosa data.setosa              <lm[12]>   FALSE setosa   Done # 3: coefs.setosa  fit.setosa -0.5694327, 0.7985283    TRUE setosa   Done #  # [[2]] #                step         depends                 out keepOut      group  state #              <char>          <list>              <list>  <lgcl>     <char> <char> # 1:  data.versicolor                  <data.frame[50x5]>   FALSE versicolor   Done # 2:   fit.versicolor data.versicolor            <lm[12]>   FALSE versicolor   Done # 3: coefs.versicolor  fit.versicolor 0.8721460,0.3197193    TRUE versicolor   Done #  # [[3]] #               step        depends                 out keepOut     group  state #             <char>         <list>              <list>  <lgcl>    <char> <char> # 1:  data.virginica                 <data.frame[50x5]>   FALSE virginica   Done # 2:   fit.virginica data.virginica            <lm[12]>   FALSE virginica   Done # 3: coefs.virginica  fit.virginica 1.4463054,0.2318905    TRUE virginica   Done"},{"path":"https://github.com/rpahl/pipeflow/articles/split-and-combine.html","id":"combine-output","dir":"Articles","previous_headings":"","what":"Combine output","title":"Advanced: How to use pipeflow with split data sets","text":"cases, may want (re-)combine output pipeline parts. example, may want combine coefficients linear models one table. Let’s matter extend pipeline one step end. pipeline looks now. Next want apply set_data_split function, basically need make sure pipeline split except last step combines everything. achieve using toStep parameter, basically tells pipeline split certain step. see last step replicated data subset now contains list dependencies, namely: Graphically becomes even clear: Finally, let’s see plays run pipeline.","code":"pip <- pipe_new(         \"my-pipeline\"     ) |>      pipe_add(         \"fit\",         function(             data = ~data,             xVar = \"x\",             yVar = \"y\"         ) {             lm(paste(yVar, \"~\", xVar), data = data)         }     ) |>      pipe_add(         \"coefs\",         function(             fit = ~fit         ) {             coefficients(fit)         }     )  |>      pipe_add(         \"combine_coefs\",         function(             coefs = ~coefs         ) {             coefs |> do.call(rbind, args = _) |> as.data.frame()         },         keepOut = TRUE     ) pip #             step depends    out keepOut         group  state #           <char>  <list> <list>  <lgcl>        <char> <char> # 1:          data         [NULL]   FALSE          data    New # 2:           fit    data [NULL]   FALSE           fit    New # 3:         coefs     fit [NULL]   FALSE         coefs    New # 4: combine_coefs   coefs [NULL]    TRUE combine_coefs    New pip$set_data_split(split(iris, iris$Species), toStep = \"coefs\") pip #                 step         depends    out keepOut         group    state #               <char>          <list> <list>  <lgcl>        <char>   <char> #  1:      data.setosa                 [NULL]   FALSE        setosa      New #  2:       fit.setosa     data.setosa [NULL]   FALSE        setosa Outdated #  3:     coefs.setosa      fit.setosa [NULL]   FALSE        setosa Outdated #  4:  data.versicolor                 [NULL]   FALSE    versicolor      New #  5:   fit.versicolor data.versicolor [NULL]   FALSE    versicolor Outdated #  6: coefs.versicolor  fit.versicolor [NULL]   FALSE    versicolor Outdated #  7:   data.virginica                 [NULL]   FALSE     virginica      New #  8:    fit.virginica  data.virginica [NULL]   FALSE     virginica Outdated #  9:  coefs.virginica   fit.virginica [NULL]   FALSE     virginica Outdated # 10:    combine_coefs       <list[1]> [NULL]    TRUE combine_coefs      New pip$get_depends()[[\"combine_coefs\"]] # $coefs # [1] \"coefs.setosa\"     \"coefs.versicolor\" \"coefs.virginica\" do.call(visNetwork, args = pip$get_graph()) |>     visHierarchicalLayout(direction = \"LR\", sortMethod = \"directed\") pip$set_params(list(xVar = \"Sepal.Length\", yVar = \"Sepal.Width\")) pip$run() # INFO  [2024-12-01 21:46:52.680] Start run of 'my-pipeline' pipeline: # INFO  [2024-12-01 21:46:52.681] Step 1/10 data.setosa # INFO  [2024-12-01 21:46:52.683] Step 2/10 fit.setosa # INFO  [2024-12-01 21:46:52.686] Step 3/10 coefs.setosa # INFO  [2024-12-01 21:46:52.688] Step 4/10 data.versicolor # INFO  [2024-12-01 21:46:52.691] Step 5/10 fit.versicolor # INFO  [2024-12-01 21:46:52.694] Step 6/10 coefs.versicolor # INFO  [2024-12-01 21:46:52.696] Step 7/10 data.virginica # INFO  [2024-12-01 21:46:52.698] Step 8/10 fit.virginica # INFO  [2024-12-01 21:46:52.701] Step 9/10 coefs.virginica # INFO  [2024-12-01 21:46:52.702] Step 10/10 combine_coefs # INFO  [2024-12-01 21:46:52.704] Finished execution of steps. # INFO  [2024-12-01 21:46:52.705] Done. pip$collect_out() # $combine_coefs #                  (Intercept) Sepal.Length # coefs.setosa      -0.5694327    0.7985283 # coefs.versicolor   0.8721460    0.3197193 # coefs.virginica    1.4463054    0.2318905"},{"path":"https://github.com/rpahl/pipeflow/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Roman Pahl. Author, maintainer.","code":""},{"path":"https://github.com/rpahl/pipeflow/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Pahl R (2024). pipeflow: Implement Data Analysis Workflows Pipelines. R package version 0.1.1, https://github.com/rpahl/pipeflow, https://rpahl.github.io/pipeflow.","code":"@Manual{,   title = {pipeflow: Implement Data Analysis Workflows with Pipelines},   author = {Roman Pahl},   year = {2024},   note = {R package version 0.1.1, https://github.com/rpahl/pipeflow},   url = {https://rpahl.github.io/pipeflow}, }"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"pipeflow-","dir":"","previous_headings":"","what":"Implement Data Analysis Workflows with Pipelines","title":"Implement Data Analysis Workflows with Pipelines","text":"pipeflow intuitive yet powerful framework building running data processing pipelines. Whether ’re working scientific computing, machine learning (AI), statistical reporting, pipeflow adapts seamlessly scenario—small exploratory analyses complex production workflows. Since pipeflow pipelines essentially sequences R functions, ’re easy learn can used interactively typical R users. Advanced features like dynamic branching self-modifying pipelines provide additional flexibility complex workflows.","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"why-use-pipeflow","dir":"","previous_headings":"","what":"Why use pipeflow","title":"Implement Data Analysis Workflows with Pipelines","text":"Easy learn yet powerful complex workflows Automatically manages function dependencies Promotes standardized, reproducible analysis Simplifies error handling, debugging, reusability","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"key-features","dir":"","previous_headings":"","what":"Key features","title":"Implement Data Analysis Workflows with Pipelines","text":"Flexible Application: Use interactively programmatically R Dependency Management: Dependencies checked definition, ensuring reliable workflows Comprehensive Logging: Logs step, customizable logger options Parameter Control: Easily view adjust function parameters one place Modular Composition: Modify, extend, combine pipelines effortlessly Intelligent Execution: Skip steps already --date, similar make Visualization: View pipelines tabular graphical formats","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"advanced-features","dir":"","previous_headings":"","what":"Advanced features","title":"Implement Data Analysis Workflows with Pipelines","text":"Dynamic Branching: Apply pipeline multiple datasets seamlessly Self-Modifying: Pipelines can adapt modify runtime","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Implement Data Analysis Workflows with Pipelines","text":"","code":"# Install release version from CRAN install.packages(\"pipeflow\")  # Install development version from GitHub devtools::install_github(\"rpahl/pipeflow\")"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"usage","dir":"","previous_headings":"","what":"Usage","title":"Implement Data Analysis Workflows with Pipelines","text":"","code":"library(pipeflow)"},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"getting-started","dir":"","previous_headings":"","what":"Getting Started","title":"Implement Data Analysis Workflows with Pipelines","text":"recommended read vignettes order listed : Get started pipeflow Modifying existing pipelines Combining pipelines Collecting output","code":""},{"path":"https://github.com/rpahl/pipeflow/index.html","id":"advanced-topics","dir":"","previous_headings":"","what":"Advanced topics","title":"Implement Data Analysis Workflows with Pipelines","text":"use pipeflow split data sets pipelines can modify runtime","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":null,"dir":"Reference","previous_headings":"","what":"Pipeline Class — Pipeline","title":"Pipeline Class — Pipeline","text":"class implements analysis pipeline. pipeline consists sequence analysis steps, can added one one. added step may may depend one previous steps. pipeline keeps track dependencies among steps ensure dependencies met creation pipeline, , pipeline run. pipeline run, output stored pipeline along step can accessed later. Different pipelines can bound together preserving dependencies within pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Pipeline Class — Pipeline","text":"Roman Pahl","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"public-fields","dir":"Reference","previous_headings":"","what":"Public fields","title":"Pipeline Class — Pipeline","text":"name string name pipeline pipeline data.table pipeline row represents one step.","code":""},{"path":[]},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"public-methods","dir":"Reference","previous_headings":"","what":"Public methods","title":"Pipeline Class — Pipeline","text":"Pipeline$new() Pipeline$add() Pipeline$append() Pipeline$append_to_step_names() Pipeline$collect_out() Pipeline$discard_steps() Pipeline$get_data() Pipeline$get_depends() Pipeline$get_depends_down() Pipeline$get_depends_up() Pipeline$get_graph() Pipeline$get_out() Pipeline$get_params() Pipeline$get_params_at_step() Pipeline$get_params_unique() Pipeline$get_params_unique_json() Pipeline$get_step() Pipeline$get_step_names() Pipeline$get_step_number() Pipeline$has_step() Pipeline$insert_after() Pipeline$insert_before() Pipeline$length() Pipeline$lock_step() Pipeline$print() Pipeline$pop_step() Pipeline$pop_steps_after() Pipeline$pop_steps_from() Pipeline$remove_step() Pipeline$rename_step() Pipeline$replace_step() Pipeline$reset() Pipeline$run() Pipeline$run_step() Pipeline$set_data() Pipeline$set_data_split() Pipeline$set_keep_out() Pipeline$set_params() Pipeline$set_params_at_step() Pipeline$split() Pipeline$unlock_step() Pipeline$clone()","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-new-","dir":"Reference","previous_headings":"","what":"Method new()","title":"Pipeline Class — Pipeline","text":"constructor","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$new(name, data = NULL, logger = NULL)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"name name Pipeline data optional data used start pipeline. data also can set later using set_data function. logger custom logger used logging. logger provided, default logger used, sufficient use cases. want use custom log function, need provide function obeys following form: function(level, msg, ...) { custom logging code } level argument string one info, warn, error. msg argument string containing message logged. ... argument list named parameters, can used add additional information log message. Currently, used add context case step giving warning error. Note default logger, log layout can altered time via set_log_layout().","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:8)) p  # Passing custom logger my_logger <- function(level, msg, ...) {    cat(level, msg, \"\\n\") } p <- Pipeline$new(\"myPipe\", logger = my_logger)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-add-","dir":"Reference","previous_headings":"","what":"Method add()","title":"Pipeline Class — Pipeline","text":"Add pipeline step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-1","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$add(   step,   fun,   params = list(),   description = \"\",   group = step,   keepOut = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-1","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step. step name must unique. fun function name function applied step. existing lambda/anonymous functions can used. params list list parameters set overwrite parameters passed function. description string optional description step group string output collected pipeline execution (see function collect_out) grouped defined group names. default, name step, comes handy pipeline copy-appended multiple times keep results function/step grouped one place. keepOut logical FALSE (default) output step collected calling collect_out pipeline run. option used keep results matter skip intermediate results needed. See also function collect_out details.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-1","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-1","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Add steps with lambda functions p <- Pipeline$new(\"myPipe\", data = 1) p$add(\"s1\", \\(x = ~data) 2*x)  # use input data p$add(\"s2\", \\(x = ~data, y = ~s1) x * y) try(p$add(\"s2\", \\(z = 3) 3)) # error: step 's2' exists already try(p$add(\"s3\", \\(z = ~foo) 3)) # dependency 'foo' not found p  p <- Pipeline$new(\"myPipe\", data = c(1, 2, NA, 3, 4)) p$add(\"calc_mean\", mean, params = list(x = ~data, na.rm = TRUE)) p$run()$get_out(\"calc_mean\")  p <- Pipeline$new(\"myPipe\", data = 1:10) p$add(\"s1\", \\(x = ~data) 2*x, description = \"multiply by 2\") print(p) print(p, verbose = TRUE) # print all columns  p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:5, y = 1:5)) p$add(\"prep_x\", \\(data = ~data) data$x, group = \"prep\") p$add(\"prep_y\", \\(data = ~data) (data$y)^2, group = \"prep\") p$add(\"sum\", \\(x = ~prep_x, y = ~prep_y) x + y) p$run()$collect_out(all = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-append-","dir":"Reference","previous_headings":"","what":"Method append()","title":"Pipeline Class — Pipeline","text":"Append another pipeline. append takes care name clashes dependencies, changed append.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-2","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$append(p, outAsIn = FALSE, tryAutofixNames = TRUE, sep = \".\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-2","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"p Pipeline object appended. outAsIn logical TRUE, output first pipeline used input second pipeline. tryAutofixNames logical TRUE, name clashes tried automatically resolved appending 2nd pipeline's name. sep string separator used auto-resolving step names","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-2","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns new combined Pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-2","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Append pipeline p1 <- Pipeline$new(\"pipe1\") p1$add(\"step1\", \\(x = 1) x) p2 <- Pipeline$new(\"pipe2\") p2$add(\"step2\", \\(y = 1) y) p1$append(p2)  p3 <- Pipeline$new(\"pipe3\") p3$add(\"step1\", \\(z = 1) z) p1$append(p2)$append(p3)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-append-to-step-names-","dir":"Reference","previous_headings":"","what":"Method append_to_step_names()","title":"Pipeline Class — Pipeline","text":"Append string step names. Also takes care updating dependencies accordingly.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-3","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$append_to_step_names(postfix, sep = \".\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-3","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"postfix string appended step name. sep string separator step name postfix.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-3","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-3","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\") p$add(\"step1\", \\(x = 1) x) p$add(\"step2\", \\(y = 1) y) p$append_to_step_names(\"new\") p p$append_to_step_names(\"new\", sep = \"_\") p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-collect-out-","dir":"Reference","previous_headings":"","what":"Method collect_out()","title":"Pipeline Class — Pipeline","text":"Collect output afer pipeline run, default, steps keepOut set TRUE. output grouped group names (see group parameter function add) set explicitly corresponds step names.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-4","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$collect_out(groupBy = \"group\", all = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-4","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"groupBy string column pipeline group output. logical TRUE output collected regardless keepOut flag. can useful debugging.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-4","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list containing output, named groups, , default, steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-4","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2) p$add(\"step2\", \\(x = ~step1) x + 2, keepOut = TRUE) p$run() p$collect_out() p$collect_out(all = TRUE) |> str()  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2, group = \"add\") p$add(\"step2\", \\(x = ~step1, y = 2) x + y, group = \"add\") p$add(\"step3\", \\(x = ~data) x * 3, group = \"mult\") p$add(\"step4\", \\(x = ~data, y = 2) x * y, group = \"mult\") p p$run() p$collect_out(all = TRUE) |> str()  # Grouped by state p$set_params(list(y = 5)) p p$collect_out(groupBy = \"state\", all = TRUE) |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-discard-steps-","dir":"Reference","previous_headings":"","what":"Method discard_steps()","title":"Pipeline Class — Pipeline","text":"Discard steps match given pattern.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-5","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$discard_steps(pattern, recursive = FALSE, fixed = TRUE, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-5","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"pattern string containing regular expression (character string fixed = TRUE) matched. recursive logical TRUE step removed together downstream dependencies. fixed logical TRUE, pattern string matched . Overrides conflicting arguments. ... arguments passed grep().","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-5","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-5","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~add1) x + 2) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$discard_steps(\"mult\") p  # Re-add steps p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p # Discard step 'add1' does'nt work as 'add2' and 'mult3' depend on it try(p$discard_steps(\"add1\")) p$discard_steps(\"add1\", recursive = TRUE)   # this works p  # Trying to discard non-existent steps is just ignored p$discard_steps(\"non-existent\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-data-","dir":"Reference","previous_headings":"","what":"Method get_data()","title":"Pipeline Class — Pipeline","text":"Get data","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-6","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_data()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-6","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"output defined data step, default first step pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-6","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$get_data() p$set_data(3:4) p$get_data()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-","dir":"Reference","previous_headings":"","what":"Method get_depends()","title":"Pipeline Class — Pipeline","text":"Get dependencies defined pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-7","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-7","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"named list dependencies step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-7","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$get_depends()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-down-","dir":"Reference","previous_headings":"","what":"Method get_depends_down()","title":"Pipeline Class — Pipeline","text":"Get downstream dependencies given step, default descending recursively.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-8","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends_down(step, recursive = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-6","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step recursive logical TRUE, dependencies dependencies also returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-8","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list downstream dependencies","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-8","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_down(\"add1\") p$get_depends_down(\"add1\", recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-depends-up-","dir":"Reference","previous_headings":"","what":"Method get_depends_up()","title":"Pipeline Class — Pipeline","text":"Get upstream dependencies given step, default descending recursively.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-9","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_depends_up(step, recursive = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-7","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step recursive logical TRUE, dependencies dependencies also returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-9","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list upstream dependencies","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-9","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_up(\"mult4\") p$get_depends_up(\"mult4\", recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-graph-","dir":"Reference","previous_headings":"","what":"Method get_graph()","title":"Pipeline Class — Pipeline","text":"Visualize pipeline graph.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-10","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_graph(groups = NULL)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-8","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"groups character NULL, steps belonging given groups considered.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-10","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"two data frames, one nodes one edges ready used visNetwork package.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-10","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = ~add1, y = ~add2) x * y) if (require(\"visNetwork\", quietly = TRUE)) {     do.call(visNetwork, args = p$get_graph()) }"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-out-","dir":"Reference","previous_headings":"","what":"Method get_out()","title":"Pipeline Class — Pipeline","text":"Get output given step pipeline run.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-11","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_out(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-9","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-11","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"output given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-11","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$run() p$get_out(\"add1\") p$get_out(\"add2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-","dir":"Reference","previous_headings":"","what":"Method get_params()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) function parameters defined pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-12","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-10","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-12","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list parameters, sorted named step. Steps parameters filtered .","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-12","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params() |> str() p$get_params(ignoreHidden = FALSE) |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-at-step-","dir":"Reference","previous_headings":"","what":"Method get_params_at_step()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) given step name.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-13","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_at_step(step, ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-11","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-13","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list parameters defined given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-13","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params_at_step(\"add2\") p$get_params_at_step(\"add2\", ignoreHidden = FALSE) p$get_params_at_step(\"add3\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-unique-","dir":"Reference","previous_headings":"","what":"Method get_params_unique()","title":"Pipeline Class — Pipeline","text":"Get unbound (.e. referring steps) parameters defined pipeline, list parameter . values parameters, values first step parameter defined. particularly useful parameters set using set_params function, set value steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-14","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_unique(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-12","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-14","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list unique parameters","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-14","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique() p$get_params_unique(ignoreHidden = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-params-unique-json-","dir":"Reference","previous_headings":"","what":"Method get_params_unique_json()","title":"Pipeline Class — Pipeline","text":"Get unique function parameters json format.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-15","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_params_unique_json(ignoreHidden = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-13","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"ignoreHidden logical TRUE, hidden parameters (.e. names starting dot) ignored thus returned.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-15","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list flat unnamed json list unique function parameters","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-15","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique_json() p$get_params_unique_json(ignoreHidden = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-","dir":"Reference","previous_headings":"","what":"Method get_step()","title":"Pipeline Class — Pipeline","text":"Get step pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-16","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-14","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-16","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"data.table row containing step. step found, error given.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-16","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, z = ~add1) x + y + z) p$run() add1 <- p$get_step(\"add1\") print(add1) add1[[\"params\"]] add1[[\"out\"]] try() try(p$get_step(\"foo\")) # error: step 'foo' does not exist"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-names-","dir":"Reference","previous_headings":"","what":"Method get_step_names()","title":"Pipeline Class — Pipeline","text":"Get step names pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-17","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step_names()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-17","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector step names","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-17","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_names()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-get-step-number-","dir":"Reference","previous_headings":"","what":"Method get_step_number()","title":"Pipeline Class — Pipeline","text":"Get step number","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-18","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$get_step_number(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-15","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-18","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"step number pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-18","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_number(\"f2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-has-step-","dir":"Reference","previous_headings":"","what":"Method has_step()","title":"Pipeline Class — Pipeline","text":"Determine whether pipeline given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-19","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$has_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-16","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-19","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"logical whether step exists","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-19","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$has_step(\"f2\") p$has_step(\"foo\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-insert-after-","dir":"Reference","previous_headings":"","what":"Method insert_after()","title":"Pipeline Class — Pipeline","text":"Insert step certain step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-20","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$insert_after(afterStep, step, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-17","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"afterStep string name step insert step string name step insert ... arguments passed add method pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-20","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-20","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_after(\"f1\", \"f3\", \\(x = ~f1) x) p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-insert-before-","dir":"Reference","previous_headings":"","what":"Method insert_before()","title":"Pipeline Class — Pipeline","text":"Insert step certain step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-21","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$insert_before(beforeStep, step, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-18","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"beforeStep string name step insert step string name step insert ... arguments passed add method pipeline","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-21","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-21","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_before(\"f2\", \"f3\", \\(x = ~f1) x) p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-length-","dir":"Reference","previous_headings":"","what":"Method length()","title":"Pipeline Class — Pipeline","text":"Length pipeline aka number pipeline steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-22","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$length()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-22","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"numeric length pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-22","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$length()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-lock-step-","dir":"Reference","previous_headings":"","what":"Method lock_step()","title":"Pipeline Class — Pipeline","text":"Locking step means parameters output (given output) locked. output, parameters locked. Locking step useful step happens share parameter names steps affected parameters set commonly entire pipeline (see function set_params ).","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-23","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$lock_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-19","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-23","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-23","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$run() p$get_out(\"add1\") p$get_out(\"add2\") p$lock_step(\"add1\")  p$set_data(3) p$set_params(list(x = 3)) p$run() p$get_out(\"add1\") p$get_out(\"add2\")"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-print-","dir":"Reference","previous_headings":"","what":"Method print()","title":"Pipeline Class — Pipeline","text":"Print pipeline table.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-24","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$print(verbose = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-20","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"verbose logical TRUE, print columns pipeline, otherwise subset columns printed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-24","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-24","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$print()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-step-","dir":"Reference","previous_headings":"","what":"Method pop_step()","title":"Pipeline Class — Pipeline","text":"Remove last step pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-25","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_step()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-25","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"string name step removed","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-25","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p p$pop_step() # \"f2\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-steps-after-","dir":"Reference","previous_headings":"","what":"Method pop_steps_after()","title":"Pipeline Class — Pipeline","text":"Remove steps given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-26","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_steps_after(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-21","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-26","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector steps removed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-26","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_after(\"f1\")  # \"f2\", \"f3\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-pop-steps-from-","dir":"Reference","previous_headings":"","what":"Method pop_steps_from()","title":"Pipeline Class — Pipeline","text":"Remove steps including given step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-27","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$pop_steps_from(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-22","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-27","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"character vector steps removed.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-27","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_from(\"f2\")  # \"f2\", \"f3\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-remove-step-","dir":"Reference","previous_headings":"","what":"Method remove_step()","title":"Pipeline Class — Pipeline","text":"Remove certain step pipeline. step exist, error given. steps depend step removed, error given, unless recursive = TRUE.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-28","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$remove_step(step, recursive = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-23","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step removed. recursive logical TRUE step removed together downstream dependencies.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-28","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-28","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = 1, y = ~add2) x * y) p$remove_step(\"mult1\") p try(p$remove_step(\"add1\"))  # fails because \"add2\" depends on \"add1\" p$remove_step(\"add1\", recursive = TRUE)  # removes \"add1\" and \"add2\" p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-rename-step-","dir":"Reference","previous_headings":"","what":"Method rename_step()","title":"Pipeline Class — Pipeline","text":"Safely rename step pipeline. new step name result name clash, error given.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-29","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$rename_step(from, to)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-24","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"string name step renamed. string new name step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-29","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-29","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p try(p$rename_step(\"add1\", \"add2\"))  # fails because \"add2\" exists p$rename_step(\"add1\", \"first_add\")  # Ok p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-replace-step-","dir":"Reference","previous_headings":"","what":"Method replace_step()","title":"Pipeline Class — Pipeline","text":"Replace pipeline step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-30","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$replace_step(   step,   fun,   params = list(),   description = \"\",   group = step,   keepOut = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-25","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step replaced. Step must exist. fun string function operation applied step. existing lambda/anonymous functions can used. params list list parameters overwrite default parameters existing functions. description string optional description step group string grouping information (default name step. output collected later (see function collect_out default put together group names. , example, comes handy pipeline copy-appended multiple times keep results function/step one place. keepOut logical FALSE output function cleaned end whole pipeline execution. option used keep results matter.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-30","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-30","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = 1, y = 2) x * y, keepOut = TRUE) p$run()$collect_out() p$replace_step(\"mult\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() try(p$replace_step(\"foo\", \\(x = 1) x))   # step 'foo' does not exist"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-reset-","dir":"Reference","previous_headings":"","what":"Method reset()","title":"Pipeline Class — Pipeline","text":"Resets pipeline state run. means output removed state steps reset 'New'.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-31","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$reset()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-31","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-31","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$run() p p$reset() p"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-run-","dir":"Reference","previous_headings":"","what":"Method run()","title":"Pipeline Class — Pipeline","text":"Run new /outdated pipeline steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-32","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$run(   force = FALSE,   recursive = TRUE,   cleanUnkept = FALSE,   progress = NULL,   showLog = TRUE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-26","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"force logical TRUE steps run regardless whether outdated . recursive logical TRUE step returns new pipeline, run current pipeline aborted new pipeline run recursively. cleanUnkept logical TRUE output marked kept removed pipeline run. option can useful temporary results require lot memory. progress function parameter can used provide custom progress function form function(value, detail), show progress pipeline run step, value current step number detail name step. showLog logical steps logged pipeline run?","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-32","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-32","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Simple pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"final\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() p$set_params(list(z = 4))  # outdates steps add2 and final p p$run()$collect_out() p$run(cleanUnkept = TRUE)  # clean up temporary results p  # Recursive pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"new_pipe\", \\(x = ~add1) {     pp <- Pipeline$new(\"new_pipe\", data = x)     pp$add(\"add1\", \\(x = ~data) x + 1)     pp$add(\"add2\", \\(x = ~add1) x + 2, keepOut = TRUE)     } ) p$run()$collect_out()  # Run pipeline with progress bar p <- Pipeline$new(\"pipe\", data = 1) p$add(\"first step\", \\() Sys.sleep(1)) p$add(\"second step\", \\() Sys.sleep(1)) p$add(\"last step\", \\() Sys.sleep(1)) pb <- txtProgressBar(min = 1, max = p$length(), style = 3) fprogress <- function(value, detail) {    setTxtProgressBar(pb, value) } p$run(progress = fprogress, showLog = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-run-step-","dir":"Reference","previous_headings":"","what":"Method run_step()","title":"Pipeline Class — Pipeline","text":"Run given pipeline step possibly together upstream downstream dependencies.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-33","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$run_step(   step,   upstream = TRUE,   downstream = FALSE,   cleanUnkept = FALSE )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-27","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step upstream logical TRUE, run dependent upstream steps first. downstream logical TRUE, run depdendent downstream afterwards. cleanUnkept logical TRUE output marked kept removed pipeline run. option can useful temporary results require lot memory.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-33","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-33","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run_step(\"add2\") p$run_step(\"add2\", downstream = TRUE) p$run_step(\"mult\", upstream = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-data-","dir":"Reference","previous_headings":"","what":"Method set_data()","title":"Pipeline Class — Pipeline","text":"Set data first step pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-34","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_data(data)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-28","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"data data.frame initial data set.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-34","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-34","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$run()$collect_out() p$set_data(3) p$run()$collect_out()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-data-split-","dir":"Reference","previous_headings":"","what":"Method set_data_split()","title":"Pipeline Class — Pipeline","text":"Split-copy pipeline list data sets. sub-pipeline one data sets set input data. step names sub-pipelines original step names plus name data set.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-35","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_data_split(   dataList,   toStep = utils::tail(self$get_step_names(), 1),   groupBySplit = TRUE,   sep = \".\" )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-29","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"dataList list data sets toStep string step name marking optional subset pipeline, data split applied . groupBySplit logical whether set step groups according data split. sep string separator used step name data set name creating new step names.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-35","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"new combined Pipeline sub-pipeline set one data sets.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-35","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList) p3 p3$run()$collect_out() |> str()  # Don't group output by split p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList, groupBySplit = FALSE) p3 p3$run()$collect_out() |> str()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-keep-out-","dir":"Reference","previous_headings":"","what":"Method set_keep_out()","title":"Pipeline Class — Pipeline","text":"Change keepOut flag given pipeline step, determines whether output step collected calling collect_out() pipeline run.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-36","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_keep_out(step, keepOut = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-30","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step keepOut logical whether keep output step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-36","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-36","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run()$collect_out() p$set_keep_out(\"add1\", keepOut = FALSE) p$set_keep_out(\"mult\", keepOut = TRUE) p$collect_out()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-params-","dir":"Reference","previous_headings":"","what":"Method set_params()","title":"Pipeline Class — Pipeline","text":"Set parameters pipeline. parameter occurs several steps, parameter set commonly steps.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-37","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_params(params, warnUndefined = TRUE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-31","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"params list parameters set warnUndefined logical whether give warning parameter defined pipeline.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-37","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-37","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 1) x + y) p$add(\"mult\", \\(x = 1, z = 1) x * z) p$get_params() p$set_params(list(x = 3, y = 3)) p$get_params() p$set_params(list(x = 5, z = 3)) p$get_params() suppressWarnings(     p$set_params(list(foo = 3))  # warning: trying to set undefined )"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-set-params-at-step-","dir":"Reference","previous_headings":"","what":"Method set_params_at_step()","title":"Pipeline Class — Pipeline","text":"Set unbound parameter values given pipeline step.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-38","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$set_params_at_step(step, params)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-32","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step params list parameters set","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-38","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"returns Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-38","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1, z = 2) x + y) p$add(\"add2\", \\(x = ~data, y = 1, z = 2) x + y) p$set_params_at_step(\"add1\", list(y = 3, z = 3)) p$get_params() try(p$set_params_at_step(\"add1\", list(foo = 3))) # foo not defined"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-split-","dir":"Reference","previous_headings":"","what":"Method split()","title":"Pipeline Class — Pipeline","text":"Splits pipeline independent parts.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-39","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$split()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-39","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"list Pipeline objects","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-39","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"# Example for two independent calculation paths p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = ~data) x) p$add(\"f2\", \\(x = 1) x) p$add(\"f3\", \\(x = ~f1) x) p$add(\"f4\", \\(x = ~f2) x) p$split()  # Example of split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) pips <- p$set_data_split(dataList)$split()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-unlock-step-","dir":"Reference","previous_headings":"","what":"Method unlock_step()","title":"Pipeline Class — Pipeline","text":"Unlock previously locked step. step locked, command ignored.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-40","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$unlock_step(step)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-33","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"step string name step","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"returns-40","dir":"Reference","previous_headings":"","what":"Returns","title":"Pipeline Class — Pipeline","text":"Pipeline object invisibly","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"examples-40","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$lock_step(\"add1\") p$set_params(list(x = 3)) p$get_params() p$unlock_step(\"add1\") p$set_params(list(x = 3)) p$get_params()"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"method-clone-","dir":"Reference","previous_headings":"","what":"Method clone()","title":"Pipeline Class — Pipeline","text":"objects class cloneable method.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"usage-41","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline Class — Pipeline","text":"","code":"Pipeline$clone(deep = FALSE)"},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"arguments-34","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline Class — Pipeline","text":"deep Whether make deep clone.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/Pipeline.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Pipeline Class — Pipeline","text":"","code":"## ------------------------------------------------ ## Method `Pipeline$new` ## ------------------------------------------------  p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:8)) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New  # Passing custom logger my_logger <- function(level, msg, ...) {    cat(level, msg, \"\\n\") } p <- Pipeline$new(\"myPipe\", logger = my_logger)  ## ------------------------------------------------ ## Method `Pipeline$add` ## ------------------------------------------------  # Add steps with lambda functions p <- Pipeline$new(\"myPipe\", data = 1) p$add(\"s1\", \\(x = ~data) 2*x)  # use input data p$add(\"s2\", \\(x = ~data, y = ~s1) x * y) try(p$add(\"s2\", \\(z = 3) 3)) # error: step 's2' exists already #> Error : step 's2' already exists try(p$add(\"s3\", \\(z = ~foo) 3)) # dependency 'foo' not found #> Error : step 's3': dependency 'foo' not found p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     s1    data [NULL]   FALSE     s1    New #> 3:     s2 data,s1 [NULL]   FALSE     s2    New  p <- Pipeline$new(\"myPipe\", data = c(1, 2, NA, 3, 4)) p$add(\"calc_mean\", mean, params = list(x = ~data, na.rm = TRUE)) p$run()$get_out(\"calc_mean\") #> INFO  [2024-12-01 21:46:17.666] Start run of 'myPipe' pipeline: #> INFO  [2024-12-01 21:46:17.694] Step 1/2 data #> INFO  [2024-12-01 21:46:17.703] Step 2/2 calc_mean #> INFO  [2024-12-01 21:46:17.723] Finished execution of steps. #> INFO  [2024-12-01 21:46:17.724] Done. #> [1] 2.5  p <- Pipeline$new(\"myPipe\", data = 1:10) p$add(\"s1\", \\(x = ~data) 2*x, description = \"multiply by 2\") print(p) #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     s1    data [NULL]   FALSE     s1    New print(p, verbose = TRUE) # print all columns #>      step           fun funcName    params depends    out keepOut  group #>    <char>        <list>   <char>    <list>  <list> <list>  <lgcl> <char> #> 1:   data <function[1]> function <list[0]>         [NULL]   FALSE   data #> 2:     s1 <function[1]> function <list[1]>    data [NULL]   FALSE     s1 #>      description                time  state #>           <char>              <POSc> <char> #> 1:               2024-12-01 21:46:17    New #> 2: multiply by 2 2024-12-01 21:46:17    New  p <- Pipeline$new(\"myPipe\", data = data.frame(x = 1:5, y = 1:5)) p$add(\"prep_x\", \\(data = ~data) data$x, group = \"prep\") p$add(\"prep_y\", \\(data = ~data) (data$y)^2, group = \"prep\") p$add(\"sum\", \\(x = ~prep_x, y = ~prep_y) x + y) p$run()$collect_out(all = TRUE) #> INFO  [2024-12-01 21:46:17.739] Start run of 'myPipe' pipeline: #> INFO  [2024-12-01 21:46:17.740] Step 1/4 data #> INFO  [2024-12-01 21:46:17.742] Step 2/4 prep_x #> INFO  [2024-12-01 21:46:17.744] Step 3/4 prep_y #> INFO  [2024-12-01 21:46:17.746] Step 4/4 sum #> INFO  [2024-12-01 21:46:17.747] Finished execution of steps. #> INFO  [2024-12-01 21:46:17.748] Done. #> $data #>   x y #> 1 1 1 #> 2 2 2 #> 3 3 3 #> 4 4 4 #> 5 5 5 #>  #> $prep #> $prep$prep_x #> [1] 1 2 3 4 5 #>  #> $prep$prep_y #> [1]  1  4  9 16 25 #>  #>  #> $sum #> [1]  2  6 12 20 30 #>   ## ------------------------------------------------ ## Method `Pipeline$append` ## ------------------------------------------------  # Append pipeline p1 <- Pipeline$new(\"pipe1\") p1$add(\"step1\", \\(x = 1) x) p2 <- Pipeline$new(\"pipe2\") p2$add(\"step2\", \\(y = 1) y) p1$append(p2) #>          step depends    out keepOut  group  state #>        <char>  <list> <list>  <lgcl> <char> <char> #> 1:       data         [NULL]   FALSE   data    New #> 2:      step1         [NULL]   FALSE  step1    New #> 3: data.pipe2         [NULL]   FALSE   data    New #> 4:      step2         [NULL]   FALSE  step2    New  p3 <- Pipeline$new(\"pipe3\") p3$add(\"step1\", \\(z = 1) z) p1$append(p2)$append(p3) #>           step depends    out keepOut  group  state #>         <char>  <list> <list>  <lgcl> <char> <char> #> 1:        data         [NULL]   FALSE   data    New #> 2:       step1         [NULL]   FALSE  step1    New #> 3:  data.pipe2         [NULL]   FALSE   data    New #> 4:       step2         [NULL]   FALSE  step2    New #> 5:  data.pipe3         [NULL]   FALSE   data    New #> 6: step1.pipe3         [NULL]   FALSE  step1    New  ## ------------------------------------------------ ## Method `Pipeline$append_to_step_names` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\") p$add(\"step1\", \\(x = 1) x) p$add(\"step2\", \\(y = 1) y) p$append_to_step_names(\"new\") p #>         step depends    out keepOut  group  state #>       <char>  <list> <list>  <lgcl> <char> <char> #> 1:  data.new         [NULL]   FALSE   data    New #> 2: step1.new         [NULL]   FALSE  step1    New #> 3: step2.new         [NULL]   FALSE  step2    New p$append_to_step_names(\"new\", sep = \"_\") p #>             step depends    out keepOut  group  state #>           <char>  <list> <list>  <lgcl> <char> <char> #> 1:  data.new_new         [NULL]   FALSE   data    New #> 2: step1.new_new         [NULL]   FALSE  step1    New #> 3: step2.new_new         [NULL]   FALSE  step2    New  ## ------------------------------------------------ ## Method `Pipeline$collect_out` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2) p$add(\"step2\", \\(x = ~step1) x + 2, keepOut = TRUE) p$run() #> INFO  [2024-12-01 21:46:17.887] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:17.892] Step 1/3 data #> INFO  [2024-12-01 21:46:17.894] Step 2/3 step1 #> INFO  [2024-12-01 21:46:17.896] Step 3/3 step2 #> INFO  [2024-12-01 21:46:17.897] Finished execution of steps. #> INFO  [2024-12-01 21:46:17.898] Done. p$collect_out() #> $step2 #> [1] 5 6 #>  p$collect_out(all = TRUE) |> str() #> List of 3 #>  $ data : int [1:2] 1 2 #>  $ step1: num [1:2] 3 4 #>  $ step2: num [1:2] 5 6  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"step1\", \\(x = ~data) x + 2, group = \"add\") p$add(\"step2\", \\(x = ~step1, y = 2) x + y, group = \"add\") p$add(\"step3\", \\(x = ~data) x * 3, group = \"mult\") p$add(\"step4\", \\(x = ~data, y = 2) x * y, group = \"mult\") p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:  step1    data [NULL]   FALSE    add    New #> 3:  step2   step1 [NULL]   FALSE    add    New #> 4:  step3    data [NULL]   FALSE   mult    New #> 5:  step4    data [NULL]   FALSE   mult    New p$run() #> INFO  [2024-12-01 21:46:17.924] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:17.925] Step 1/5 data #> INFO  [2024-12-01 21:46:17.928] Step 2/5 step1 #> INFO  [2024-12-01 21:46:17.930] Step 3/5 step2 #> INFO  [2024-12-01 21:46:17.931] Step 4/5 step3 #> INFO  [2024-12-01 21:46:17.933] Step 5/5 step4 #> INFO  [2024-12-01 21:46:17.934] Finished execution of steps. #> INFO  [2024-12-01 21:46:17.935] Done. p$collect_out(all = TRUE) |> str() #> List of 3 #>  $ data: int [1:2] 1 2 #>  $ add :List of 2 #>   ..$ step1: num [1:2] 3 4 #>   ..$ step2: num [1:2] 5 6 #>  $ mult:List of 2 #>   ..$ step3: num [1:2] 3 6 #>   ..$ step4: num [1:2] 2 4  # Grouped by state p$set_params(list(y = 5)) p #>      step depends    out keepOut  group    state #>    <char>  <list> <list>  <lgcl> <char>   <char> #> 1:   data            1,2   FALSE   data     Done #> 2:  step1    data    3,4   FALSE    add     Done #> 3:  step2   step1    5,6   FALSE    add Outdated #> 4:  step3    data    3,6   FALSE   mult     Done #> 5:  step4    data    2,4   FALSE   mult Outdated p$collect_out(groupBy = \"state\", all = TRUE) |> str() #> List of 2 #>  $ Done    :List of 3 #>   ..$ data : int [1:2] 1 2 #>   ..$ step1: num [1:2] 3 4 #>   ..$ step3: num [1:2] 3 6 #>  $ Outdated:List of 2 #>   ..$ step2: num [1:2] 5 6 #>   ..$ step4: num [1:2] 2 4  ## ------------------------------------------------ ## Method `Pipeline$discard_steps` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~add1) x + 2) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$discard_steps(\"mult\") #> step 'mult4' was removed #> step 'mult3' was removed p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New  # Re-add steps p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New #> 4:  mult3    add1 [NULL]   FALSE  mult3    New #> 5:  mult4    add2 [NULL]   FALSE  mult4    New # Discard step 'add1' does'nt work as 'add2' and 'mult3' depend on it try(p$discard_steps(\"add1\")) #> Error in self$remove_step(step, recursive = recursive) :  #>   cannot remove step 'add1' because the following steps depend on it: 'add2', 'mult3' p$discard_steps(\"add1\", recursive = TRUE)   # this works #> Removing step 'add1' and its downstream dependencies: 'add2', 'mult3', 'mult4' #> step 'add1' was removed p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New  # Trying to discard non-existent steps is just ignored p$discard_steps(\"non-existent\")  ## ------------------------------------------------ ## Method `Pipeline$get_data` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$get_data() #> [1] 1 2 p$set_data(3:4) p$get_data() #> [1] 3 4  ## ------------------------------------------------ ## Method `Pipeline$get_depends` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$get_depends() #> $data #> character(0) #>  #> $add1 #>      x  #> \"data\"  #>  #> $add2 #>      x      y  #> \"data\" \"add1\"  #>   ## ------------------------------------------------ ## Method `Pipeline$get_depends_down` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_down(\"add1\") #> [1] \"add2\"  \"mult3\" \"mult4\" p$get_depends_down(\"add1\", recursive = FALSE) #> [1] \"add2\"  \"mult3\"  ## ------------------------------------------------ ## Method `Pipeline$get_depends_up` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$add(\"mult3\", \\(x = ~add1) x * 3) p$add(\"mult4\", \\(x = ~add2) x * 4) p$get_depends_up(\"mult4\") #> [1] \"data\" \"add1\" \"add2\" p$get_depends_up(\"mult4\", recursive = FALSE) #> [1] \"add2\"  ## ------------------------------------------------ ## Method `Pipeline$get_graph` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = ~add1, y = ~add2) x * y) if (require(\"visNetwork\", quietly = TRUE)) {     do.call(visNetwork, args = p$get_graph()) }  {\"x\":{\"nodes\":{\"id\":[1,2,3,4],\"label\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"group\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"shape\":[\"database\",\"box\",\"box\",\"box\"],\"color\":[\"lightblue\",\"lightblue\",\"lightblue\",\"lightblue\"],\"title\":[\"<p><\\/p>\",\"<p><\\/p>\",\"<p><\\/p>\",\"<p><\\/p>\"]},\"edges\":{\"from\":[1,2,2,3],\"to\":[2,3,4,4],\"arrows\":[\"to\",\"to\",\"to\",\"to\"]},\"nodesToDataframe\":true,\"edgesToDataframe\":true,\"options\":{\"width\":\"100%\",\"height\":\"100%\",\"nodes\":{\"shape\":\"dot\"},\"manipulation\":{\"enabled\":false}},\"groups\":[\"data\",\"add1\",\"add2\",\"mult1\"],\"width\":null,\"height\":null,\"idselection\":{\"enabled\":false},\"byselection\":{\"enabled\":false},\"main\":null,\"submain\":null,\"footer\":null,\"background\":\"rgba(0, 0, 0, 0)\"},\"evals\":[],\"jsHooks\":[]} ## ------------------------------------------------ ## Method `Pipeline$get_out` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(x = ~data) x + 1) p$add(\"add2\", \\(x = ~data, y = ~add1) x + y) p$run() #> INFO  [2024-12-01 21:46:18.047] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.048] Step 1/3 data #> INFO  [2024-12-01 21:46:18.050] Step 2/3 add1 #> INFO  [2024-12-01 21:46:18.052] Step 3/3 add2 #> INFO  [2024-12-01 21:46:18.053] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.053] Done. p$get_out(\"add1\") #> [1] 2 3 p$get_out(\"add2\") #> [1] 3 5  ## ------------------------------------------------ ## Method `Pipeline$get_params` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params() |> str() #> List of 2 #>  $ add1:List of 1 #>   ..$ x: num 1 #>  $ add2:List of 2 #>   ..$ x: num 1 #>   ..$ y: num 2 p$get_params(ignoreHidden = FALSE) |> str() #> List of 2 #>  $ add1:List of 1 #>   ..$ x: num 1 #>  $ add2:List of 3 #>   ..$ x : num 1 #>   ..$ y : num 2 #>   ..$ .z: num 3  ## ------------------------------------------------ ## Method `Pipeline$get_params_at_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"add3\", \\() 1 + 2) p$get_params_at_step(\"add2\") #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  p$get_params_at_step(\"add2\", ignoreHidden = FALSE) #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  #> $.z #> [1] 3 #>  p$get_params_at_step(\"add3\") #> list()  ## ------------------------------------------------ ## Method `Pipeline$get_params_unique` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique() #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  p$get_params_unique(ignoreHidden = FALSE) #> $x #> [1] 1 #>  #> $y #> [1] 2 #>  #> $.z #> [1] 3 #>   ## ------------------------------------------------ ## Method `Pipeline$get_params_unique_json` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, .z = 3) x + y + .z) p$add(\"mult1\", \\(x = 1, y = 2, .z = 3, b = ~add2) x * y * b) p$get_params_unique_json() #> [ #>   { #>     \"name\": \"x\", #>     \"value\": 1 #>   }, #>   { #>     \"name\": \"y\", #>     \"value\": 2 #>   } #> ]  p$get_params_unique_json(ignoreHidden = FALSE) #> [ #>   { #>     \"name\": \"x\", #>     \"value\": 1 #>   }, #>   { #>     \"name\": \"y\", #>     \"value\": 2 #>   }, #>   { #>     \"name\": \".z\", #>     \"value\": 3 #>   } #> ]   ## ------------------------------------------------ ## Method `Pipeline$get_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = 2, z = ~add1) x + y + z) p$run() #> INFO  [2024-12-01 21:46:18.092] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.093] Step 1/3 data #> INFO  [2024-12-01 21:46:18.095] Step 2/3 add1 #> INFO  [2024-12-01 21:46:18.097] Step 3/3 add2 #> INFO  [2024-12-01 21:46:18.098] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.099] Done. add1 <- p$get_step(\"add1\") print(add1) #>      step           fun funcName    params depends    out keepOut  group #>    <char>        <list>   <char>    <list>  <list> <list>  <lgcl> <char> #> 1:   add1 <function[1]> function <list[2]>    data    2,3   FALSE   add1 #>    description                time  state #>         <char>              <POSc> <char> #> 1:             2024-12-01 21:46:18   Done add1[[\"params\"]] #> [[1]] #> [[1]]$data #> ~data #> <environment: 0x00000296abb38438> #>  #> [[1]]$x #> [1] 1 #>  #>  add1[[\"out\"]] #> [[1]] #> [1] 2 3 #>  try() #> Error in try() : argument \"expr\" is missing, with no default try(p$get_step(\"foo\")) # error: step 'foo' does not exist #> Error : step 'foo' does not exist  ## ------------------------------------------------ ## Method `Pipeline$get_step_names` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_names() #> [1] \"data\" \"f1\"   \"f2\"    ## ------------------------------------------------ ## Method `Pipeline$get_step_number` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$get_step_number(\"f2\") #> [1] 3  ## ------------------------------------------------ ## Method `Pipeline$has_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$has_step(\"f2\") #> [1] TRUE p$has_step(\"foo\") #> [1] FALSE  ## ------------------------------------------------ ## Method `Pipeline$insert_after` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_after(\"f1\", \"f3\", \\(x = ~f1) x) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f3      f1 [NULL]   FALSE     f3    New #> 4:     f2      f1 [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$insert_before` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(x = ~f1) x) p$insert_before(\"f2\", \"f3\", \\(x = ~f1) x) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f3      f1 [NULL]   FALSE     f3    New #> 4:     f2      f1 [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$length` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$length() #> [1] 3  ## ------------------------------------------------ ## Method `Pipeline$lock_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$run() #> INFO  [2024-12-01 21:46:18.141] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.142] Step 1/3 data #> INFO  [2024-12-01 21:46:18.144] Step 2/3 add1 #> INFO  [2024-12-01 21:46:18.146] Step 3/3 add2 #> INFO  [2024-12-01 21:46:18.147] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.147] Done. p$get_out(\"add1\") #> [1] 2 p$get_out(\"add2\") #> [1] 2 p$lock_step(\"add1\")  p$set_data(3) p$set_params(list(x = 3)) #> skipping setting parameters x at locked step 'add1' p$run() #> INFO  [2024-12-01 21:46:18.168] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.169] Step 1/3 data #> INFO  [2024-12-01 21:46:18.171] Step 2/3 add1 - skip 'locked' step #> INFO  [2024-12-01 21:46:18.172] Step 3/3 add2 #> INFO  [2024-12-01 21:46:18.173] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.173] Done. p$get_out(\"add1\") #> [1] 2 p$get_out(\"add2\") #> [1] 6  ## ------------------------------------------------ ## Method `Pipeline$print` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$print() #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f2         [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$pop_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f2         [NULL]   FALSE     f2    New p$pop_step() # \"f2\" #> [1] \"f2\" p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New  ## ------------------------------------------------ ## Method `Pipeline$pop_steps_after` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_after(\"f1\")  # \"f2\", \"f3\" #> [1] \"f2\" \"f3\" p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New  ## ------------------------------------------------ ## Method `Pipeline$pop_steps_from` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$add(\"f3\", \\(z = 1) z) p$pop_steps_from(\"f2\")  # \"f2\", \"f3\" #> [1] \"f2\" \"f3\" p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New  ## ------------------------------------------------ ## Method `Pipeline$remove_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p$add(\"mult1\", \\(x = 1, y = ~add2) x * y) p$remove_step(\"mult1\") p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New try(p$remove_step(\"add1\"))  # fails because \"add2\" depends on \"add1\" #> Error in p$remove_step(\"add1\") :  #>   cannot remove step 'add1' because the following steps depend on it: 'add2' p$remove_step(\"add1\", recursive = TRUE)  # removes \"add1\" and \"add2\" #> Removing step 'add1' and its downstream dependencies: 'add2' p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New  ## ------------------------------------------------ ## Method `Pipeline$rename_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$add(\"add2\", \\(x = 1, y = ~add1) x + y) p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:   add1    data [NULL]   FALSE   add1    New #> 3:   add2    add1 [NULL]   FALSE   add2    New try(p$rename_step(\"add1\", \"add2\"))  # fails because \"add2\" exists #> Error : step 'add2' already exists p$rename_step(\"add1\", \"first_add\")  # Ok p #>         step   depends    out keepOut  group  state #>       <char>    <list> <list>  <lgcl> <char> <char> #> 1:      data           [NULL]   FALSE   data    New #> 2: first_add      data [NULL]   FALSE   add1    New #> 3:      add2 first_add [NULL]   FALSE   add2    New  ## ------------------------------------------------ ## Method `Pipeline$replace_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = 1, y = 2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:46:18.233] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.234] Step 1/4 data #> INFO  [2024-12-01 21:46:18.236] Step 2/4 add1 #> INFO  [2024-12-01 21:46:18.237] Step 3/4 add2 #> INFO  [2024-12-01 21:46:18.239] Step 4/4 mult #> INFO  [2024-12-01 21:46:18.240] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.240] Done. #> $mult #> [1] 2 #>  p$replace_step(\"mult\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:46:18.243] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.244] Step 1/4 data - skip 'done' step #> INFO  [2024-12-01 21:46:18.245] Step 2/4 add1 - skip 'done' step #> INFO  [2024-12-01 21:46:18.246] Step 3/4 add2 - skip 'done' step #> INFO  [2024-12-01 21:46:18.246] Step 4/4 mult #> INFO  [2024-12-01 21:46:18.248] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.248] Done. #> $mult #> [1] 6 #>  try(p$replace_step(\"foo\", \\(x = 1) x))   # step 'foo' does not exist #> Error : step 'foo' does not exist  ## ------------------------------------------------ ## Method `Pipeline$reset` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"f1\", \\(x = 1) x) p$add(\"f2\", \\(y = 1) y) p$run() #> INFO  [2024-12-01 21:46:18.264] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.265] Step 1/3 data #> INFO  [2024-12-01 21:46:18.266] Step 2/3 f1 #> INFO  [2024-12-01 21:46:18.268] Step 3/3 f2 #> INFO  [2024-12-01 21:46:18.269] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.269] Done. p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data            1,2   FALSE   data   Done #> 2:     f1              1   FALSE     f1   Done #> 3:     f2              1   FALSE     f2   Done p$reset() p #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1         [NULL]   FALSE     f1    New #> 3:     f2         [NULL]   FALSE     f2    New  ## ------------------------------------------------ ## Method `Pipeline$run` ## ------------------------------------------------  # Simple pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"final\", \\(x = ~add1, y = ~add2) x * y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:46:18.280] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.280] Step 1/4 data #> INFO  [2024-12-01 21:46:18.283] Step 2/4 add1 #> INFO  [2024-12-01 21:46:18.285] Step 3/4 add2 #> INFO  [2024-12-01 21:46:18.287] Step 4/4 final #> INFO  [2024-12-01 21:46:18.288] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.288] Done. #> $final #> [1] 8 #>  p$set_params(list(z = 4))  # outdates steps add2 and final p #>      step   depends    out keepOut  group    state #>    <char>    <list> <list>  <lgcl> <char>   <char> #> 1:   data                1   FALSE   data     Done #> 2:   add1      data      2   FALSE   add1     Done #> 3:   add2      add1      4   FALSE   add2 Outdated #> 4:  final add1,add2      8    TRUE  final Outdated p$run()$collect_out() #> INFO  [2024-12-01 21:46:18.295] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.296] Step 1/4 data - skip 'done' step #> INFO  [2024-12-01 21:46:18.297] Step 2/4 add1 - skip 'done' step #> INFO  [2024-12-01 21:46:18.298] Step 3/4 add2 #> INFO  [2024-12-01 21:46:18.299] Step 4/4 final #> INFO  [2024-12-01 21:46:18.301] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.301] Done. #> $final #> [1] 12 #>  p$run(cleanUnkept = TRUE)  # clean up temporary results #> INFO  [2024-12-01 21:46:18.303] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.303] Step 1/4 data - skip 'done' step #> INFO  [2024-12-01 21:46:18.304] Step 2/4 add1 - skip 'done' step #> INFO  [2024-12-01 21:46:18.305] Step 3/4 add2 - skip 'done' step #> INFO  [2024-12-01 21:46:18.306] Step 4/4 final - skip 'done' step #> INFO  [2024-12-01 21:46:18.306] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.307] Clean temporary results. #> INFO  [2024-12-01 21:46:18.307] Done. p #>      step   depends    out keepOut  group    state #>    <char>    <list> <list>  <lgcl> <char>   <char> #> 1:   data           [NULL]   FALSE   data Outdated #> 2:   add1      data [NULL]   FALSE   add1 Outdated #> 3:   add2      add1 [NULL]   FALSE   add2 Outdated #> 4:  final add1,add2     12    TRUE  final     Done  # Recursive pipeline p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"new_pipe\", \\(x = ~add1) {     pp <- Pipeline$new(\"new_pipe\", data = x)     pp$add(\"add1\", \\(x = ~data) x + 1)     pp$add(\"add2\", \\(x = ~add1) x + 2, keepOut = TRUE)     } ) p$run()$collect_out() #> INFO  [2024-12-01 21:46:18.314] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:18.315] Step 1/3 data #> INFO  [2024-12-01 21:46:18.317] Step 2/3 add1 #> INFO  [2024-12-01 21:46:18.319] Step 3/3 new_pipe #> INFO  [2024-12-01 21:46:18.322] Abort pipeline execution and restart on new. #> INFO  [2024-12-01 21:46:18.322] Start run of 'new_pipe' pipeline: #> INFO  [2024-12-01 21:46:18.323] Step 1/3 data #> INFO  [2024-12-01 21:46:18.325] Step 2/3 add1 #> INFO  [2024-12-01 21:46:18.327] Step 3/3 add2 #> INFO  [2024-12-01 21:46:18.328] Finished execution of steps. #> INFO  [2024-12-01 21:46:18.329] Done. #> $add2 #> [1] 5 #>   # Run pipeline with progress bar p <- Pipeline$new(\"pipe\", data = 1) p$add(\"first step\", \\() Sys.sleep(1)) p$add(\"second step\", \\() Sys.sleep(1)) p$add(\"last step\", \\() Sys.sleep(1)) pb <- txtProgressBar(min = 1, max = p$length(), style = 3) fprogress <- function(value, detail) {    setTxtProgressBar(pb, value) } p$run(progress = fprogress, showLog = FALSE) #>    |                                                                               |                                                                      |   0%   |                                                                               |=======================                                               |  33%   |                                                                               |===============================================                       |  67%   |                                                                               |======================================================================| 100%  ## ------------------------------------------------ ## Method `Pipeline$run_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~add1, z = 2) x + z) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run_step(\"add2\") #> INFO  [2024-12-01 21:46:21.408] Start step run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.409] Step 1/3 data (upstream) #> INFO  [2024-12-01 21:46:21.411] Step 2/3 add1 (upstream) #> INFO  [2024-12-01 21:46:21.413] Step 3/3 add2 #> INFO  [2024-12-01 21:46:21.414] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.415] Done. p$run_step(\"add2\", downstream = TRUE) #> INFO  [2024-12-01 21:46:21.432] Start step run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.432] Step 1/4 data (upstream) #> INFO  [2024-12-01 21:46:21.434] Step 2/4 add1 (upstream) #> INFO  [2024-12-01 21:46:21.436] Step 3/4 add2 #> INFO  [2024-12-01 21:46:21.438] Step 4/4 mult (downstream) #> INFO  [2024-12-01 21:46:21.439] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.439] Done. p$run_step(\"mult\", upstream = TRUE) #> INFO  [2024-12-01 21:46:21.441] Start step run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.441] Step 1/4 data (upstream) #> INFO  [2024-12-01 21:46:21.444] Step 2/4 add1 (upstream) #> INFO  [2024-12-01 21:46:21.446] Step 3/4 add2 (upstream) #> INFO  [2024-12-01 21:46:21.448] Step 4/4 mult #> INFO  [2024-12-01 21:46:21.449] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.449] Done.  ## ------------------------------------------------ ## Method `Pipeline$set_data` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$run()$collect_out() #> INFO  [2024-12-01 21:46:21.453] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.454] Step 1/2 data #> INFO  [2024-12-01 21:46:21.456] Step 2/2 add1 #> INFO  [2024-12-01 21:46:21.457] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.457] Done. #> $add1 #> [1] 2 #>  p$set_data(3) p$run()$collect_out() #> INFO  [2024-12-01 21:46:21.460] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.461] Step 1/2 data #> INFO  [2024-12-01 21:46:21.464] Step 2/2 add1 #> INFO  [2024-12-01 21:46:21.465] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.465] Done. #> $add1 #> [1] 4 #>   ## ------------------------------------------------ ## Method `Pipeline$set_data_split` ## ------------------------------------------------  # Split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList) p3 #>      step       depends    out keepOut  group    state #>    <char>        <list> <list>  <lgcl> <char>   <char> #> 1: data.a               [NULL]   FALSE      a      New #> 2: add1.a        data.a [NULL]    TRUE      a Outdated #> 3: mult.a data.a,add1.a [NULL]    TRUE      a Outdated #> 4: data.b               [NULL]   FALSE      b      New #> 5: add1.b        data.b [NULL]    TRUE      b Outdated #> 6: mult.b data.b,add1.b [NULL]    TRUE      b Outdated #> 7: data.c               [NULL]   FALSE      c      New #> 8: add1.c        data.c [NULL]    TRUE      c Outdated #> 9: mult.c data.c,add1.c [NULL]    TRUE      c Outdated p3$run()$collect_out() |> str() #> INFO  [2024-12-01 21:46:21.488] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.488] Step 1/9 data.a #> INFO  [2024-12-01 21:46:21.490] Step 2/9 add1.a #> INFO  [2024-12-01 21:46:21.492] Step 3/9 mult.a #> INFO  [2024-12-01 21:46:21.494] Step 4/9 data.b #> INFO  [2024-12-01 21:46:21.496] Step 5/9 add1.b #> INFO  [2024-12-01 21:46:21.498] Step 6/9 mult.b #> INFO  [2024-12-01 21:46:21.500] Step 7/9 data.c #> INFO  [2024-12-01 21:46:21.502] Step 8/9 add1.c #> INFO  [2024-12-01 21:46:21.503] Step 9/9 mult.c #> INFO  [2024-12-01 21:46:21.505] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.505] Done. #> List of 3 #>  $ a:List of 2 #>   ..$ add1.a: num 2 #>   ..$ mult.a: num 2 #>  $ b:List of 2 #>   ..$ add1.b: num 3 #>   ..$ mult.b: num 6 #>  $ c:List of 2 #>   ..$ add1.c: num 4 #>   ..$ mult.c: num 12  # Don't group output by split p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) p3 <- p$set_data_split(dataList, groupBySplit = FALSE) p3 #>      step       depends    out keepOut  group    state #>    <char>        <list> <list>  <lgcl> <char>   <char> #> 1: data.a               [NULL]   FALSE data.a      New #> 2: add1.a        data.a [NULL]    TRUE add1.a Outdated #> 3: mult.a data.a,add1.a [NULL]    TRUE mult.a Outdated #> 4: data.b               [NULL]   FALSE data.b      New #> 5: add1.b        data.b [NULL]    TRUE add1.b Outdated #> 6: mult.b data.b,add1.b [NULL]    TRUE mult.b Outdated #> 7: data.c               [NULL]   FALSE data.c      New #> 8: add1.c        data.c [NULL]    TRUE add1.c Outdated #> 9: mult.c data.c,add1.c [NULL]    TRUE mult.c Outdated p3$run()$collect_out() |> str() #> INFO  [2024-12-01 21:46:21.531] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.532] Step 1/9 data.a #> INFO  [2024-12-01 21:46:21.534] Step 2/9 add1.a #> INFO  [2024-12-01 21:46:21.536] Step 3/9 mult.a #> INFO  [2024-12-01 21:46:21.537] Step 4/9 data.b #> INFO  [2024-12-01 21:46:21.539] Step 5/9 add1.b #> INFO  [2024-12-01 21:46:21.541] Step 6/9 mult.b #> INFO  [2024-12-01 21:46:21.543] Step 7/9 data.c #> INFO  [2024-12-01 21:46:21.545] Step 8/9 add1.c #> INFO  [2024-12-01 21:46:21.546] Step 9/9 mult.c #> INFO  [2024-12-01 21:46:21.548] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.548] Done. #> List of 6 #>  $ add1.a: num 2 #>  $ mult.a: num 2 #>  $ add1.b: num 3 #>  $ mult.b: num 6 #>  $ add1.c: num 4 #>  $ mult.c: num 12  ## ------------------------------------------------ ## Method `Pipeline$set_keep_out` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y, keepOut = TRUE) p$add(\"add2\", \\(x = ~data, y = 2) x + y) p$add(\"mult\", \\(x = ~add1, y = ~add2) x * y) p$run()$collect_out() #> INFO  [2024-12-01 21:46:21.557] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:21.558] Step 1/4 data #> INFO  [2024-12-01 21:46:21.560] Step 2/4 add1 #> INFO  [2024-12-01 21:46:21.562] Step 3/4 add2 #> INFO  [2024-12-01 21:46:21.566] Step 4/4 mult #> INFO  [2024-12-01 21:46:21.567] Finished execution of steps. #> INFO  [2024-12-01 21:46:21.568] Done. #> $add1 #> [1] 2 #>  p$set_keep_out(\"add1\", keepOut = FALSE) p$set_keep_out(\"mult\", keepOut = TRUE) p$collect_out() #> $mult #> [1] 6 #>   ## ------------------------------------------------ ## Method `Pipeline$set_params` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1) x + y) p$add(\"add2\", \\(x = ~data, y = 1) x + y) p$add(\"mult\", \\(x = 1, z = 1) x * z) p$get_params() #> $add1 #> $add1$y #> [1] 1 #>  #>  #> $add2 #> $add2$y #> [1] 1 #>  #>  #> $mult #> $mult$x #> [1] 1 #>  #> $mult$z #> [1] 1 #>  #>  p$set_params(list(x = 3, y = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #>  #>  #> $add2 #> $add2$y #> [1] 3 #>  #>  #> $mult #> $mult$x #> [1] 3 #>  #> $mult$z #> [1] 1 #>  #>  p$set_params(list(x = 5, z = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #>  #>  #> $add2 #> $add2$y #> [1] 3 #>  #>  #> $mult #> $mult$x #> [1] 5 #>  #> $mult$z #> [1] 3 #>  #>  suppressWarnings(     p$set_params(list(foo = 3))  # warning: trying to set undefined )  ## ------------------------------------------------ ## Method `Pipeline$set_params_at_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = ~data, y = 1, z = 2) x + y) p$add(\"add2\", \\(x = ~data, y = 1, z = 2) x + y) p$set_params_at_step(\"add1\", list(y = 3, z = 3)) p$get_params() #> $add1 #> $add1$y #> [1] 3 #>  #> $add1$z #> [1] 3 #>  #>  #> $add2 #> $add2$y #> [1] 1 #>  #> $add2$z #> [1] 2 #>  #>  try(p$set_params_at_step(\"add1\", list(foo = 3))) # foo not defined #> Error in p$set_params_at_step(\"add1\", list(foo = 3)) :  #>   Unable to set parameter(s) foo at step add1 - candidates are y, z  ## ------------------------------------------------ ## Method `Pipeline$split` ## ------------------------------------------------  # Example for two independent calculation paths p <- Pipeline$new(\"pipe\", data = 1) p$add(\"f1\", \\(x = ~data) x) p$add(\"f2\", \\(x = 1) x) p$add(\"f3\", \\(x = ~f1) x) p$add(\"f4\", \\(x = ~f2) x) p$split() #> [[1]] #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:   data         [NULL]   FALSE   data    New #> 2:     f1    data [NULL]   FALSE     f1    New #> 3:     f3      f1 [NULL]   FALSE     f3    New #>  #> [[2]] #>      step depends    out keepOut  group  state #>    <char>  <list> <list>  <lgcl> <char> <char> #> 1:     f2         [NULL]   FALSE     f2    New #> 2:     f4      f2 [NULL]   FALSE     f4    New #>   # Example of split by three data sets dataList <- list(a = 1, b = 2, c = 3) p <- Pipeline$new(\"pipe\") p$add(\"add1\", \\(x = ~data) x + 1, keepOut = TRUE) p$add(\"mult\", \\(x = ~data, y = ~add1) x * y, keepOut = TRUE) pips <- p$set_data_split(dataList)$split()  ## ------------------------------------------------ ## Method `Pipeline$unlock_step` ## ------------------------------------------------  p <- Pipeline$new(\"pipe\", data = 1) p$add(\"add1\", \\(x = 1, data = ~data) x + data) p$add(\"add2\", \\(x = 1, data = ~data) x + data) p$lock_step(\"add1\") p$set_params(list(x = 3)) #> skipping setting parameters x at locked step 'add1' p$get_params() #> $add1 #> $add1$x #> [1] 1 #>  #>  #> $add2 #> $add2$x #> [1] 3 #>  #>  p$unlock_step(\"add1\") p$set_params(list(x = 3)) p$get_params() #> $add1 #> $add1$x #> [1] 3 #>  #>  #> $add2 #> $add2$x #> [1] 3 #>  #>"},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":null,"dir":"Reference","previous_headings":"","what":"Pipeline alias functions — pipelineAliases","title":"Pipeline alias functions — pipelineAliases","text":"Alias functions, one member function Pipeline object.","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Pipeline alias functions — pipelineAliases","text":"","code":"pipe_add(pip, ...)  pipe_append(pip, ...)  pipe_append_to_step_names(pip, ...)  pipe_clone(pip, ...)  pipe_collect_out(pip, ...)  pipe_discard_steps(pip, ...)  pipe_get_data(pip, ...)  pipe_get_depends(pip, ...)  pipe_get_depends_down(pip, ...)  pipe_get_depends_up(pip, ...)  pipe_get_graph(pip, ...)  pipe_get_out(pip, ...)  pipe_get_params(pip, ...)  pipe_get_params_at_step(pip, ...)  pipe_get_params_unique(pip, ...)  pipe_get_params_unique_json(pip, ...)  pipe_get_step(pip, ...)  pipe_get_step_names(pip, ...)  pipe_get_step_number(pip, ...)  pipe_has_step(pip, ...)  pipe_insert_after(pip, ...)  pipe_insert_before(pip, ...)  pipe_length(pip, ...)  pipe_lock_step(pip, ...)  pipe_new(...)  pipe_print(pip, ...)  pipe_pop_step(pip, ...)  pipe_pop_steps_after(pip, ...)  pipe_pop_steps_from(pip, ...)  pipe_remove_step(pip, ...)  pipe_rename_step(pip, ...)  pipe_replace_step(pip, ...)  pipe_reset(pip, ...)  pipe_run(pip, ...)  pipe_run_step(pip, ...)  pipe_set_data(pip, ...)  pipe_set_data_split(pip, ...)  pipe_set_keep_out(pip, ...)  pipe_set_params(pip, ...)  pipe_set_params_at_step(pip, ...)  pipe_split(pip, ...)  pipe_unlock_step(pip, ...)"},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Pipeline alias functions — pipelineAliases","text":"pip pipeline object ... Arguments passed respective pipeline method","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/pipelineAliases.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Pipeline alias functions — pipelineAliases","text":"result respective pipeline method","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":null,"dir":"Reference","previous_headings":"","what":"Set pipeflow log layout — set_log_layout","title":"Set pipeflow log layout — set_log_layout","text":"Set pipeflow log layout","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set pipeflow log layout — set_log_layout","text":"","code":"set_log_layout(layout)"},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set pipeflow log layout — set_log_layout","text":"layout Layout name","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set pipeflow log layout — set_log_layout","text":"invisibly returns logger object","code":""},{"path":"https://github.com/rpahl/pipeflow/reference/set_log_layout.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set pipeflow log layout — set_log_layout","text":"","code":"p <- Pipeline$new(\"pipe\", data = 1:2) p$add(\"add1\", \\(data = ~data, x = 1) x + data) p$run() #> INFO  [2024-12-01 21:46:24.778] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:24.779] Step 1/2 data #> INFO  [2024-12-01 21:46:24.781] Step 2/2 add1 #> INFO  [2024-12-01 21:46:24.782] Finished execution of steps. #> INFO  [2024-12-01 21:46:24.783] Done.  lg <- set_log_layout(\"json\") print(lg) #> <Logger> [info] pipeflow #>  #> appenders: #>   [[1]]: <AppenderConsole> [all] -> console  p$run() #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:46:24+1:00\",\"message\":\"Start run of 'pipe' pipeline:\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:46:24+1:00\",\"message\":\"Step 1/2 data - skip 'done' step\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:46:24+1:00\",\"message\":\"Step 2/2 add1 - skip 'done' step\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:46:24+1:00\",\"message\":\"Finished execution of steps.\"} #> {\"application\":\"pipeflow\",\"level\":\"info\",\"time\":\"2024-12-01T21:46:24+1:00\",\"message\":\"Done.\"}  set_log_layout(\"text\") p$run() #> INFO  [2024-12-01 21:46:24.821] Start run of 'pipe' pipeline: #> INFO  [2024-12-01 21:46:24.822] Step 1/2 data - skip 'done' step #> INFO  [2024-12-01 21:46:24.823] Step 2/2 add1 - skip 'done' step #> INFO  [2024-12-01 21:46:24.823] Finished execution of steps. #> INFO  [2024-12-01 21:46:24.824] Done."}]